/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ /*! * \file tvm/tirx/stmt_functor.h * * \brief Functors for tirx stmts * utility functions to call common functors. */ #ifndef TVM_TIRX_STMT_FUNCTOR_H_ #define TVM_TIRX_STMT_FUNCTOR_H_ #include #include #include #include #include #include #include #include namespace tvm { namespace tirx { /*! * \brief Same as ExprFunctor except it is applied on statements * \tparam FType The function signature. * \sa ExprFunctor */ template class StmtFunctor; #define STMT_FUNCTOR_DEFAULT \ { \ return VisitStmtDefault_(op, std::forward(args)...); \ } #define IR_STMT_FUNCTOR_DISPATCH(OP) \ vtable.template set_dispatch([](const ffi::ObjectRef& n, TSelf* self, Args... args) { \ return self->VisitStmt_(static_cast(n.get()), std::forward(args)...); \ }); template class StmtFunctor { private: using TSelf = StmtFunctor; using FType = NodeFunctor; public: /*! \brief the result type of this functor */ using result_type = R; /*! \brief virtual destructor */ virtual ~StmtFunctor() {} /*! * \brief Same as call. * \param n The stmt node. * \param args Additional arguments. * \return The result of the call */ R operator()(const Stmt& n, Args... args) { return VisitStmt(n, std::forward(args)...); } /*! * \brief The functor call. * \param n The stmt node. * \param args Additional arguments. * \return The result of the call */ virtual R VisitStmt(const Stmt& n, Args... args) { static FType vtable = InitVTable(); return vtable(n, this, std::forward(args)...); } // Functions that can be overriden by subclass virtual R VisitStmt_(const BindNode* op, Args... args) STMT_FUNCTOR_DEFAULT; virtual R VisitStmt_(const AttrStmtNode* op, Args... args) STMT_FUNCTOR_DEFAULT; virtual R VisitStmt_(const IfThenElseNode* op, Args... args) STMT_FUNCTOR_DEFAULT; virtual R VisitStmt_(const ForNode* op, Args... args) STMT_FUNCTOR_DEFAULT; virtual R VisitStmt_(const WhileNode* op, Args... args) STMT_FUNCTOR_DEFAULT; virtual R VisitStmt_(const BreakNode* op, Args... args) STMT_FUNCTOR_DEFAULT; virtual R VisitStmt_(const ContinueNode* op, Args... args) STMT_FUNCTOR_DEFAULT; virtual R VisitStmt_(const AllocBufferNode* op, Args... args) STMT_FUNCTOR_DEFAULT; virtual R VisitStmt_(const DeclBufferNode* op, Args... args) STMT_FUNCTOR_DEFAULT; virtual R VisitStmt_(const BufferStoreNode* op, Args... args) STMT_FUNCTOR_DEFAULT; virtual R VisitStmt_(const AssertStmtNode* op, Args... args) STMT_FUNCTOR_DEFAULT; virtual R VisitStmt_(const SeqStmtNode* op, Args... args) STMT_FUNCTOR_DEFAULT; virtual R VisitStmt_(const EvaluateNode* op, Args... args) STMT_FUNCTOR_DEFAULT; virtual R VisitStmt_(const SBlockNode* op, Args... args) STMT_FUNCTOR_DEFAULT; virtual R VisitStmt_(const SBlockRealizeNode* op, Args... args) STMT_FUNCTOR_DEFAULT; virtual R VisitStmt_(const ScopeIdDefStmtNode* op, Args... args) STMT_FUNCTOR_DEFAULT; virtual R VisitStmt_(const tirx::TilePrimitiveCallNode* op, Args... args) STMT_FUNCTOR_DEFAULT; virtual R VisitStmtDefault_(const ffi::Object* op, Args...) { TVM_FFI_THROW(InternalError) << "Do not have a default for " << op->GetTypeKey(); TVM_FFI_UNREACHABLE(); } private: // initialize the vtable. static FType InitVTable() { FType vtable; IR_STMT_FUNCTOR_DISPATCH(BindNode); IR_STMT_FUNCTOR_DISPATCH(AttrStmtNode); IR_STMT_FUNCTOR_DISPATCH(IfThenElseNode); IR_STMT_FUNCTOR_DISPATCH(ForNode); IR_STMT_FUNCTOR_DISPATCH(WhileNode); IR_STMT_FUNCTOR_DISPATCH(BreakNode); IR_STMT_FUNCTOR_DISPATCH(ContinueNode); IR_STMT_FUNCTOR_DISPATCH(AllocBufferNode); IR_STMT_FUNCTOR_DISPATCH(DeclBufferNode); IR_STMT_FUNCTOR_DISPATCH(AssertStmtNode); IR_STMT_FUNCTOR_DISPATCH(SeqStmtNode); IR_STMT_FUNCTOR_DISPATCH(EvaluateNode); IR_STMT_FUNCTOR_DISPATCH(BufferStoreNode); IR_STMT_FUNCTOR_DISPATCH(SBlockNode); IR_STMT_FUNCTOR_DISPATCH(SBlockRealizeNode); IR_STMT_FUNCTOR_DISPATCH(ScopeIdDefStmtNode); IR_STMT_FUNCTOR_DISPATCH(tirx::TilePrimitiveCallNode); vtable.Finalize(); return vtable; } }; #undef IR_STMT_FUNCTOR_DISPATCH #undef STMT_FUNCTOR_DEFAULT /*! * \brief StmtVisitor. */ class TVM_DLL StmtVisitor : protected StmtFunctor { public: using StmtFunctor::operator(); protected: using StmtFunctor::VisitStmt; /*! * \brief Visitor to Exprs, can be overriden * to do recursive changes to Exprs. * \note A common pattern is to call ExprVisitor here, * or have a class sub-class both StmtVisitor and ExprVisitor * and redirect Visit to ExprMutator::VisitExpr(Expr) */ virtual void VisitExpr(const Expr& e) {} /*! * \brief Visit buffer at definition site (AllocBuffer, DeclBuffer, SBlock alloc_buffers). * Visits buffer shape, strides, elem_offset via VisitExpr. * \param buffer The buffer being defined. * \param alloc_data If true, the buffer's data pointer is a new allocation (AllocBuffer); * if false, data references an existing variable (DeclBuffer). */ virtual void VisitBufferDef(const Buffer& buffer, bool alloc_data); /*! * \brief Visit buffer at use site (BufferStore, BufferLoad, SBlock reads/writes). * By default, this is a no-op, as buffer fields (shape, strides, elem_offset) * are visited at their definition site. */ virtual void VisitBufferUse(const Buffer& buffer); // statement visitor void VisitStmt_(const BindNode* op) override; void VisitStmt_(const AttrStmtNode* op) override; void VisitStmt_(const IfThenElseNode* op) override; void VisitStmt_(const ForNode* op) override; void VisitStmt_(const WhileNode* op) override; void VisitStmt_(const BreakNode* op) override; void VisitStmt_(const ContinueNode* op) override; void VisitStmt_(const AllocBufferNode* op) override; void VisitStmt_(const DeclBufferNode* op) override; void VisitStmt_(const BufferStoreNode* op) override; void VisitStmt_(const AssertStmtNode* op) override; void VisitStmt_(const SeqStmtNode* op) override; void VisitStmt_(const EvaluateNode* op) override; void VisitStmt_(const SBlockNode* op) override; void VisitStmt_(const SBlockRealizeNode* op) override; void VisitStmt_(const ScopeIdDefStmtNode* op) override; void VisitStmt_(const tirx::TilePrimitiveCallNode* op) override; }; /*! * \brief StmtMutator that mutates the statements. */ class TVM_DLL StmtMutator : protected StmtFunctor { public: /*! * \brief Mutate stmt. * \param stmt The input statement to be mutated. * \return The result of the call * \note It is important that stmt is passed by value. * so copy on write can be triggered correctly. * do mutator(std::move(stmt)) or when copy elison is triggered. */ Stmt operator()(Stmt stmt) { allow_copy_on_write_ = true; return VisitStmt(stmt); } protected: /*! \brief Map from old buffer to new buffer, populated by VisitBufferDef. */ ffi::Map buffer_remap_; // We perform copy on write optimizations on the StmtMutator // so that an unique copy of parent can be mutated inplace // when some of its children changed. // We only do such optimization for Stmt nests(instead of Exprs) for now // as Stmt's parent state is more likely remain unchanged when one of // its child block changes. /*! * \brief Internal state to indicate whether copy on write is enabled. * COW is enabled iff all the parents of the node are unique. */ bool allow_copy_on_write_{false}; /*! * \brief Perform copy on write on node. * * If CopyOnWrite is allowed, directly return * a strong reference to the node container. * Otherwise, return a copy of the node. * * \return The result object pointer. */ template ffi::ObjectPtr CopyOnWrite(const TNode* node) { static_assert(std::is_base_of::value, "StmtMutator:: CopyOnWrite requires us to track uniqueness of all parent " "nodes during the recursion. Because the child classes do not necessarily " "check the Array, Expr and other structures during the visit, it is only safe to " "call this function with StmtNodes for now. " "Please create a new node directly in other cases."); if (allow_copy_on_write_) { // return the old node. return ffi::GetObjectPtr(const_cast(node)); } else { // Make a new copy of the node. // need to rely on the default copy constructor return ffi::make_object(*node); } } /*! * \brief Internal mutator that everyone calls. * \note To override mutate's behavior, override VisitExpr instead. * \param stmt The input stmt. * \return The mutated results. */ Stmt VisitStmt(const Stmt& stmt) override { if (allow_copy_on_write_ && !stmt.unique()) { allow_copy_on_write_ = false; Stmt ret = StmtFunctor::VisitStmt(stmt); allow_copy_on_write_ = true; return ret; } else { return StmtFunctor::VisitStmt(stmt); } } /*! * \brief Visitor to Exprs, can be overriden * to do recursive changes to Exprs. * \note A common pattern is to call ExprMutator here, * or have a class sub-class both StmtMutator and ExprMutator * and redirect Mutate to ExprMutator::Mutate(Expr) */ virtual Expr VisitExpr(const Expr& e) { return e; } /*! \brief Mutate a primitive expression and verify that it remains primitive. */ PrimExpr VisitPrimExpr(const PrimExpr& e) { return VisitExpr(e).as_or_throw(); } /*! * \brief Visit buffer at definition site. Visits shape/strides/elem_offset via VisitExpr. * If any field changes, creates a new buffer and records it in buffer_remap_. * \param buffer The buffer being defined. * \param alloc_data If true, the buffer's data pointer is a new allocation (AllocBuffer); * if false, data references an existing variable (DeclBuffer). * \return The (possibly new) buffer. */ virtual Buffer VisitBufferDef(const Buffer& buffer, bool alloc_data); /*! * \brief Visit buffer at use site (BufferStore, BufferLoad, SBlock reads/writes). * By default, returns the remapped buffer from buffer_remap_ if exists, otherwise * returns the original buffer. Buffer fields are visited at their definition site. * \return The (possibly remapped) buffer. */ virtual Buffer VisitBufferUse(const Buffer& buffer); // statement visitor Stmt VisitStmt_(const BindNode* op) override; Stmt VisitStmt_(const AttrStmtNode* op) override; Stmt VisitStmt_(const IfThenElseNode* op) override; Stmt VisitStmt_(const ForNode* op) override; Stmt VisitStmt_(const WhileNode* op) override; Stmt VisitStmt_(const BreakNode* op) override; Stmt VisitStmt_(const ContinueNode* op) override; Stmt VisitStmt_(const AllocBufferNode* op) override; Stmt VisitStmt_(const DeclBufferNode* op) override; Stmt VisitStmt_(const BufferStoreNode* op) override; Stmt VisitStmt_(const AssertStmtNode* op) override; Stmt VisitStmt_(const SeqStmtNode* op) override; Stmt VisitStmt_(const EvaluateNode* op) override; Stmt VisitStmt_(const SBlockNode* op) override; Stmt VisitStmt_(const SBlockRealizeNode* op) override; Stmt VisitStmt_(const ScopeIdDefStmtNode* op) override; Stmt VisitStmt_(const tirx::TilePrimitiveCallNode* op) override; /*! * \brief Alternative advance method for SeqStmtNode. * * This function can be called when a child class override * VisitStmt_(const SeqStmtNode*) to introduce * the special behavior to visit * * \param op The sequence. * \param flatten_before_visit Whether to flatten the sequence before visit. * \param fmutate The mutate function, can be nullptr, which defaults to Visit. * \return The mutated result. */ Stmt VisitSeqStmt_(const SeqStmtNode* op, bool flatten_before_visit, std::function fmutate = nullptr); // internal helper. class Internal; }; /*! * \brief Visitor that recursively visit stmts and exprs on them. */ class TVM_DLL StmtExprVisitor : public ExprVisitor, public StmtVisitor { public: using StmtVisitor::operator(); using ExprVisitor::operator(); protected: using ExprVisitor::VisitExpr; using ExprVisitor::VisitExpr_; using StmtVisitor::VisitStmt; void VisitExpr(const Expr& e) override { return ExprVisitor::VisitExpr(e); } void VisitExpr_(const BufferLoadNode* op) override; }; /*! * \brief Mutator that recursively mutates stmts and exprs on them. */ class TVM_DLL StmtExprMutator : public ExprMutator, public StmtMutator { public: using StmtMutator::operator(); using ExprMutator::operator(); protected: using ExprMutator::VisitExpr; using ExprMutator::VisitExpr_; using ExprMutator::VisitPrimExpr; using StmtMutator::VisitStmt; Expr VisitExpr(const Expr& e) override { return ExprMutator::VisitExpr(e); } Expr VisitExpr_(const BufferLoadNode* op) override; }; /*! * \brief recursively visit the ir nodes in post DFS order, and transform it * * \param stmt The ir to be transformed. * \param preorder The function called in before recursive mutation * If preorder returns None, then the transform will proceed to recursive call. * If preorder returns a not None Stmt/Expr, the transformer will simply return it and * won't do further recursion. * \param postorder The function called after recursive mutation. * The recursive mutation result is passed to postorder for further mutation. * \param only_enable List of String. * If it is null, all IRNode will call preorder/postorder * If it is not null, preorder/postorder will only be called * when the IRNode's type key is in the list. */ TVM_DLL Stmt IRTransform(Stmt stmt, const ffi::Function& preorder, const ffi::Function& postorder, ffi::Optional> only_enable = std::nullopt); /*! * \brief Recursively visit a statement or expression in post DFS order, applying fvisit. * Each node is guaranteed to be visited only once. * \param node The statement or expression to be visited. * \param fvisit The visitor function to be applied. */ TVM_DLL void PostOrderVisit(const ffi::ObjectRef& node, std::function fvisit); /*! * \brief Substitute the var specified by vmap. * \param stmt The source statement to be substituted * \param vmap returns a new value if re-mapping is needed, otherwise returns nullptr. * \return The converted form. */ TVM_DLL Stmt Substitute(Stmt stmt, std::function(const Var& var)> vmap); /*! * \brief Substitute the var specified by vmap. * \param expr The source statement to be substituted * \param vmap returns a new value if re-mapping is needed, otherwise returns nullptr. * \return The result. */ TVM_DLL Expr Substitute(Expr expr, std::function(const Var& var)> vmap); inline PrimExpr Substitute(PrimExpr expr, std::function(const Var& var)> vmap) { return Substitute(Expr(expr), std::move(vmap)).as_or_throw(); } /*! * \brief Substitute the var specified by vmap. * \param arr The array of Stmt/PrimExpr to be substituted * \param vmap returns a new value if re-mapping is needed, otherwise returns nullptr. * \return The result. */ template ffi::Array Substitute(const ffi::Array& arr, std::function(const Var& var)> vmap) { return arr.Map([&vmap](const auto& elem) { return Substitute(elem, vmap); }); } /*! * \brief Substitute the vars specified by vmap. * \param range The array of Stmt/PrimExpr to be substituted * \param vmap returns a new value if re-mapping is needed, otherwise returns nullptr. * \return The modified Range. */ inline Range Substitute(const Range& range, std::function(const Var& var)> vmap) { return Range::FromMinExtent(Substitute(range->min, vmap), Substitute(range->extent, vmap)); } /*! * \brief Substitute the vars specified by vmap. * * Delegates to the Substitute methods that use std::function. This * overload allows braced-initialization of the Map, whereas the * template overload cannot. * * \param obj The object in which TIR variables should be substituted * \param vmap Map defining the TIR variables to be replaced * \return The modified object. */ template auto Substitute(Obj&& obj, const ffi::Map& vmap) { auto func = [&vmap](const Var& var) -> ffi::Optional { return vmap.Get(var); }; return Substitute(std::forward(obj), func); } /*! * \brief Substitute the vars specified by vmap. * * Delegates to the Substitute methods that use std::function. * * \param obj The object in which TIR variables should be substituted * \param vmap Map defining the TIR variables to be replaced * \return The modified object. */ template auto Substitute(Obj&& obj, const ffi::Map& vmap) { auto func = [&vmap](const Var& var) -> ffi::Optional { if (auto replacement = vmap.Get(var)) return Expr(replacement.value()); return std::nullopt; }; return Substitute(std::forward(obj), func); } /*! * \brief Substitute the vars specified by vmap. * * Delegates to the Substitute methods that use std::function. * * \param obj The object in which TIR variables should be substituted * \param vmap Map defining the TIR variables to be replaced * \return The modified object. */ template auto Substitute(Obj&& obj, const std::unordered_map& vmap) { auto func = [&vmap](const Var& var) -> ffi::Optional { if (auto it = vmap.find(var.get()); it != vmap.end()) { return Expr(it->second); } return std::nullopt; }; return Substitute(std::forward(obj), func); } /*! * \brief Substitute the vars specified by vmap. * * Delegates to the Substitute methods that use std::function. * * \param obj The object in which TIR variables should be substituted * \param vmap Map defining the TIR variables to be replaced * \return The modified object. */ template auto Substitute(Obj&& obj, const std::unordered_map& vmap) { auto func = [&vmap](const Var& var) -> ffi::Optional { if (auto it = vmap.find(var); it != vmap.end()) { return Expr(it->second); } return std::nullopt; }; return Substitute(std::forward(obj), func); } /*! * \brief Substitute the vars specified by vmap. * * Delegates to the Substitute methods that use std::function. * * \param obj The object in which TIR variables should be substituted * \param iter_vmap Map defining the TIR variables to be replaced * \return The modified object. */ template auto Substitute(Obj&& obj, const std::unordered_map& iter_vmap) { std::unordered_map vmap; for (const auto& [iter_var, expr] : iter_vmap) { vmap[iter_var->var.get()] = Expr(expr); } auto func = [&vmap](const Var& var) -> ffi::Optional { if (auto it = vmap.find(var.get()); it != vmap.end()) { return it->second; } else { return std::nullopt; } }; return Substitute(std::forward(obj), func); } /*! * \brief Substitute the var specified by vmap and legalize data types after substitution. * \param stmt The source statement to be substituted * \param vmap returns a new value if re-mapping is needed, otherwise returns nullptr. * * Unlike `Substitute`, this allows the substitution to change the data type of the expression. * * \sa Substitute * \return The result. */ TVM_DLL Stmt SubstituteWithDataTypeLegalization( Stmt stmt, std::function(const Var&)> vmap); /*! * \brief Substitute the var specified by vmap and legalize data types after substitution. * \param expr The source statement to be substituted * \param vmap returns a new value if re-mapping is needed, otherwise returns nullptr. * * Unlike `Substitute`, this allows the substitution to change the data type of the expression. * * \sa Substitute * \return The result. */ TVM_DLL PrimExpr SubstituteWithDataTypeLegalization( PrimExpr expr, std::function(const Var&)> vmap); /*! * \brief Recursively visit a statement or expression in pre DFS order, applying fvisit. * If fvisit returns false, it won't visit the children of the node. * \param stmt_or_expr The statement or expression to be visited. * \param fvisit The visitor function to be applied. If fvisit returns false, it won't visit the * children of the node */ TVM_DLL void PreOrderVisit(const ffi::ObjectRef& stmt_or_expr, const std::function& fvisit); /*! * \brief Check if the statement contains the specified node type. * * This utility potentially walks the entire statement, and should * therefore not be used if it could otherwise be merged with another * pass. * * \param stmt The statement to be searched * \return Whether stmt contains Node */ template >> bool ContainsNode(const Stmt& stmt) { struct Visitor : StmtVisitor { // Early bail-out, if we already found the node. void VisitStmt(const Stmt& stmt) final { if (contains_node) { return; } StmtVisitor::VisitStmt(stmt); } void VisitStmt_(const Node* block) override { contains_node = true; } bool contains_node{false}; }; Visitor visitor; visitor(stmt); return visitor.contains_node; } } // namespace tirx } // namespace tvm #endif // TVM_TIR_STMT_FUNCTOR_H_