// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once #include #include #include #include #include #include #include #include #include #include #include #include "glog/logging.h" #include "paddle/cinn/hlir/framework/op.h" #include "paddle/cinn/hlir/framework/pir/utils.h" #include "paddle/cinn/utils/string.h" #include "paddle/common/enforce.h" #include "paddle/phi/core/enforce.h" #include "paddle/pir/include/core/builtin_attribute.h" #include "paddle/pir/include/core/builtin_type.h" #include "paddle/pir/include/dialect/control_flow/ir/cf_op.h" #include "paddle/pir/include/dialect/shape/utils/shape_analysis.h" namespace cinn::fusion { using OpPatternKind = cinn::hlir::framework::OpPatternKind; static OpPatternKind GetOpPatternKind(const ::pir::Operation* op) { return hlir::framework::pir::CompatibleInfo::OpKind(*op); } static std::string GetNewTmpId(std::string origin_id) { if (origin_id.find('_tmp') == std::string::npos) { return origin_id + "_tmp_0"; } else { int ith = std::stoi(origin_id.substr(origin_id.size() - 1)); return origin_id.substr(0, origin_id.size() - 1) + std::to_string(ith + 1); } } static size_t GetRank(pir::Value value) { PADDLE_ENFORCE_EQ(value.type().isa(), true, ::common::errors::InvalidArgument( "The type of value should be a DenseTensorType.")); return value.type().dyn_cast().dims().size(); } // FIXME(Aurelius84): 0D Tensor is not compatible with other rank. // So we need to add a special case for 0D Tensor. static size_t GetCompatibleRank(pir::Value value) { size_t rank = GetRank(value); return rank == 0 ? 1 : rank; } std::vector GetInt64ArrayAttributeData( const ::pir::Attribute& attr_val); std::vector GetInt32ArrayAttributeData( const ::pir::Attribute& attr_val); std::vector GetReduceAxisIdx(pir::Operation* reduce_op); std::pair, bool> GetSliceAxis(pir::Operation* slice_op); bool GetReduceOpKeepDims(pir::Operation* reduce_op); std::optional> GetBroadcastOpInputOutputValue( pir::Operation* op); std::vector> GetNonBroadCastDims(pir::Operation* op); static std::string OpsDebugStr(std::vector ops) { std::stringstream ss; pir::IrPrinter printer(ss); for (const auto* op : ops) { printer.PrintOperation(*op); ss << "(" << op << ")" << "\n"; } return ss.str(); } std::unordered_set GetGroupOutputOps( const std::vector& ops); template void RemoveFromVector(std::vector* vec, T item) { auto iter = std::find(vec->begin(), vec->end(), item); if (iter != vec->end()) { vec->erase(iter); } } template std::vector ConcatVector(const std::vector& first, const std::vector& second) { std::vector result = first; result.insert(result.end(), second.begin(), second.end()); return result; } template std::vector ReverseVector(const std::vector& as) { std::vector result = as; std::reverse(result.begin(), result.end()); return result; } template std::vector ConcatAll(const std::vector>& all) { std::vector result; for (const auto& vec : all) { result = ConcatVector(result, vec); } return result; } template std::vector FilterVector(const std::vector& first, const F& func) { std::vector result; for (const auto& i : first) { if (func(i)) { result.push_back(i); } } return result; } template > bool VectorEqual(const std::vector& first, const std::vector& second, const F& func = nullptr) { if (first.size() != second.size()) { return false; } for (size_t i = 0; i < first.size(); ++i) { if (func) { if (!func(first[i], second[i])) { return false; } } else { if (first[i] != second[i]) { return false; } } } return true; } template std::vector MapVector(const std::vector& as, const std::function& func) { std::vector res; for (const auto& a : as) { res.push_back(func(a)); } return res; } template std::vector MapVectorIfTrue(const std::vector& as, const std::function& func, const std::function& pred) { std::vector res; for (const auto& a : as) { if (pred(a)) { res.push_back(func(a)); } } return res; } template std::vector> Enumerate(const std::vector& inputs) { std::vector> res; int idx = 0; for (const auto& a : inputs) { res.push_back(std::make_pair(a, idx)); idx++; } return res; } template std::set ToSet(const std::vector& input) { std::set result(input.begin(), input.end()); return result; } template std::unordered_set ToUnorderedSet(const std::vector& input) { std::unordered_set result(input.begin(), input.end()); return result; } template std::vector SetToVector(const std::set& input) { std::vector result(input.begin(), input.end()); return result; } template std::vector SetToVector(const std::unordered_set& input) { std::vector result(input.begin(), input.end()); return result; } template std::vector MapKeyToVector(const std::map& input) { std::vector result; for (const auto& pair : input) { result.push_back(pair.first); } return result; } template std::vector MapKeyToVector(const std::unordered_map& input) { std::vector result; for (const auto& pair : input) { result.push_back(pair.first); } return result; } template std::vector GatherMapValue(const std::map& input, const std::vector& keys) { std::vector result; for (const auto& key : keys) { if (input.count(key)) { result.push_back(input.at(key)); } } return result; } template std::vector GatherMapValue(const std::unordered_map& input, const std::vector& keys) { std::vector result; for (const auto& key : keys) { if (input.count(key)) { result.push_back(input.at(key)); } } return result; } template Set SetUnion(const Set& A, const Set& B) { Set result; std::set_union(A.begin(), A.end(), B.begin(), B.end(), std::inserter(result, result.begin())); return result; } template Set SetIntersection(const Set& A, const Set& B) { Set result; std::set_intersection(A.begin(), A.end(), B.begin(), B.end(), std::inserter(result, result.begin())); return result; } template Set SetDifference(const Set& A, const Set& B) { Set result; std::set_difference(A.begin(), A.end(), B.begin(), B.end(), std::inserter(result, result.begin())); return result; } template bool IsAnyFirstInSecond(const std::vector& first, const std::vector& second) { const auto& second_set = ToSet(second); for (const auto& ele : first) { if (second_set.count(ele)) { return true; } } return false; } template std::pair, std::vector> SplitFirstWhetherInSecond( const std::vector& first, const std::vector& second) { std::vector used; std::vector unused; for (size_t i = 0; i < first.size(); ++i) { if (std::find(second.begin(), second.end(), first[i]) != second.end()) { used.emplace_back(first[i]); } else { unused.emplace_back(first[i]); } } return {used, unused}; } template std::pair, std::vector> GatherFirstNotInSecond( const std::vector& first, const std::vector& second) { std::vector pos; std::vector result; for (size_t i = 0; i < first.size(); ++i) { if (std::find(second.begin(), second.end(), first[i]) == second.end()) { result.emplace_back(first[i]); pos.emplace_back(i); } } return {result, pos}; } template std::vector UniqueVectorBySet(const std::vector& v) { std::unordered_set unique(v.begin(), v.end()); return std::vector(unique.begin(), unique.end()); } template void ExtendVector(std::vector* first, const std::vector& second) { std::unordered_set visited = std::unordered_set(first->begin(), first->end()); for (auto iter = second.begin(); iter != second.end(); iter++) { if (visited.find(*iter) == visited.end()) { visited.emplace(*iter); first->emplace_back(*iter); } } } template std::vector UniqueConcatVector(const std::vector& first, const std::vector& second) { std::vector result = std::vector(first); ExtendVector(&result, second); return result; } template std::vector ArangeVector(Int start, Int end, Int step = 1) { std::vector res; for (Int i = start; i < end; i += step) { res.push_back(i); } return res; } template std::vector CastVector(const std::vector& vec) { std::vector res; for (const auto& item : vec) { res.push_back(static_cast(item)); } return res; } template std::vector GetTransposePerm(const std::vector& source, const std::vector& target) { PADDLE_ENFORCE_EQ(source.size(), target.size(), ::common::errors::InvalidArgument( "The size of source and target should be equal.")); std::vector perm; for (size_t i = 0; i < source.size(); ++i) { auto iter = std::find(source.begin(), source.end(), target[i]); PADDLE_ENFORCE_NE(iter, source.end(), ::common::errors::InvalidArgument( "The target should contain all elements in source.")); perm.emplace_back(iter - source.begin()); } return perm; } template std::vector GetReversePerm(const std::vector& perm) { return GetTransposePerm(perm, ArangeVector(0, perm.size())); } template std::vector TransposeVector(const std::vector& v, const std::vector& perm) { PADDLE_ENFORCE_GE( v.size(), perm.size(), ::common::errors::InvalidArgument( "The size of transpose vector and perm should be equal.")); std::vector result; for (size_t i = 0; i < perm.size(); ++i) { result.emplace_back(v[perm[i]]); } for (size_t i = perm.size(); i < v.size(); ++i) { result.emplace_back(v[i]); } return result; } template std::vector GatherVector(const std::vector& inp, std::vector gathers) { std::vector result; for (auto i : gathers) { result.push_back(inp.at(i)); } return result; } template std::vector ExcludeIndex(int n, std::vector excludes) { std::vector result; for (int i = 0; i < n; ++i) { if (std::find(excludes.begin(), excludes.end(), i) == excludes.end()) { result.push_back(i); } } return result; } template std::vector GatherVectorExcept(const std::vector& source, const std::vector& idx) { std::vector result; for (U i = 0; i < source.size(); i++) { if (std::find(idx.begin(), idx.end(), i) == idx.end()) { result.emplace_back(source[i]); } } return result; } template std::vector SliceVector(const std::vector& inp, int start, int end) { if (start < 0) { start = inp.size() + start; } if (end < 0) { end = inp.size() + end; } std::vector result; for (int i = start; i < end; ++i) { result.push_back(inp.at(i)); } return result; } template std::vector VectorFlatMap( const std::vector& inp, const std::function(const T&)>& func) { std::vector result; for (const auto& i : inp) { result = ConcatVector(result, func(i)); } return result; } template bool AnyFirstInSecond(const std::vector& first, const std::vector& second) { std::unordered_set pool = ToUnorderedSet(second); for (const auto& item : first) { if (pool.find(item) != pool.end()) { return true; } } return false; } template bool AllFirstInSecond(const std::vector& first, const std::vector& second) { std::unordered_set pool = ToUnorderedSet(second); for (const auto& item : first) { if (pool.find(item) == pool.end()) { return false; } } return true; } template std::pair, std::vector> SplitVector(const std::vector& vec, int pos) { return {SliceVector(vec, 0, pos), SliceVector(vec, pos, vec.size())}; } template std::vector FindPosInVector(const std::vector& vec, const T& item) { std::vector result; for (size_t i = 0; i < vec.size(); ++i) { if (vec[i] == item) { result.emplace_back(i); } } return result; } static std::vector FindDownstreamOps(pir::Operation* op) { std::vector result; for (int i = 0; i < op->num_results(); i++) { auto v = op->result(i); for (auto consumer_it = v.use_begin(); consumer_it != v.use_end(); ++consumer_it) { result.emplace_back(consumer_it->owner()); } } return result; } static const size_t GetUsageIdx(const pir::Value& v, pir::Operation* op) { size_t i = 0; for (auto consumer_it = v.use_begin(); consumer_it != v.use_end(); ++consumer_it, ++i) { if (consumer_it->owner() == op) { return i; } } PADDLE_THROW(::common::errors::NotFound( "Can not find the usage of value %s in op %s", v.impl(), op->name())); } static const size_t GetOperandIdx(const pir::Value& v, pir::Operation* op) { for (size_t i = 0; i < op->num_operands(); i++) { if (op->operand(i).source() == v) { return i; } } PADDLE_THROW(::common::errors::NotFound( "Can not find the value %s as operand of op %s", v.impl(), op->name())); } static const size_t GetResultIdx(const pir::Value& v, pir::Operation* op) { size_t i = 0; for (size_t i = 0; i < op->num_results(); i++) { if (op->result(i) == v) { return i; } } PADDLE_THROW(::common::errors::NotFound( "Can not find the value %s as result of op %s", v.impl(), op->name())); } static std::vector FindUserOp( const std::vector& candidates, const pir::Value& value) { std::vector results; for (auto consumer_it = value.use_begin(); consumer_it != value.use_end(); ++consumer_it) { pir::Operation* user_op = consumer_it.owner(); auto iter = std::find(candidates.begin(), candidates.end(), user_op); if (iter != candidates.end()) { results.emplace_back(*iter); } } return results; } static bool IsDirectUpstream(const pir::Operation* upstream, const pir::Operation* downstream) { for (const auto& value : upstream->results()) { for (const auto& operand : downstream->operands()) { if (value == operand.source()) { return true; } } } return false; } inline std::vector GetInputsValue( const std::vector& ops) { // include middle value. std::function(pir::Operation* const&)> get_inputs = [](const pir::Operation* const& in) { return in->operands_source(); }; const auto& all_inputs = VectorFlatMap(ops, get_inputs); return UniqueVectorBySet(all_inputs); } inline std::vector GetOutputsValue( const std::vector& ops) { // include middle value. std::function(pir::Operation* const&)> get_outputs = [](const pir::Operation* const& in) { return in->results(); }; const auto& all_outputs = VectorFlatMap(ops, get_outputs); return UniqueVectorBySet(all_outputs); } template std::vector VectorDiff(const std::vector& left, const std::vector& right) { const auto& set = ToSet(right); std::vector res; for (const auto& v : left) { if (!set.count(v)) res.push_back(v); } return res; } inline bool All(const std::vector a) { bool res = true; for (bool i : a) { res &= i; } return res; } inline bool Any(const std::vector a) { bool res = false; for (bool i : a) { res |= i; } return res; } std::shared_ptr GetShapeAnalysisFromValue( const pir::Value& value); template std::vector GetValueDims(const pir::Value& value, std::vector axes) { auto shape_analysis = GetShapeAnalysisFromValue(value); const auto rank = GetRank(value); std::vector dims; for (const auto& axis : axes) { PADDLE_ENFORCE_LT( axis, rank, ::common::errors::InvalidArgument("Given axis out of range.")); dims.push_back( shape_analysis->GetProductDimExpr(value, {static_cast(axis)})); } return dims; } std::vector GetValueAllDims(const pir::Value& value); std::vector GetCompatibleValueAllDims(const pir::Value& value); symbol::DimExpr GetShapeProduct(const std::vector& shape, int start, int end); inline symbol::DimExpr GetShapeProduct( const std::vector& shape) { return GetShapeProduct(shape, 0, shape.size()); } bool ShapeProductEqual(const std::vector& in_shape, const std::vector& out_shape, int in_start, int in_end, int out_start, int out_end); bool ShapeProductEqual(const std::vector& in_shape, const std::vector& out_shape); std::vector> PartitionReshapeAxes( const std::vector& in_shape, const std::vector& out_shape); } // namespace cinn::fusion