/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ /*! * \file tvm/tirx/stmt.h * \brief TIR statements. */ // Acknowledgement: Many low-level stmts originate from Halide. #ifndef TVM_TIRX_STMT_H_ #define TVM_TIRX_STMT_H_ #include #include #include #include #include #include #include #include namespace tvm { namespace tirx { /*! \brief Base node of all statements. */ class StmtNode : public ffi::Object { public: /*! * \brief Span that points to the original source code. * Reserved debug information. */ mutable Span span; StmtNode() = default; explicit StmtNode(Span span) : span(span) {} static void RegisterReflection() { namespace refl = tvm::ffi::reflection; refl::ObjectDef().def_ro("span", &StmtNode::span); } static constexpr TVMFFISEqHashKind _type_s_eq_hash_kind = kTVMFFISEqHashKindTreeNode; static constexpr const uint32_t _type_child_slots = 15; TVM_FFI_DECLARE_OBJECT_INFO("tirx.Stmt", StmtNode, ffi::Object); }; /*! \brief Container of all statements */ class Stmt : public ffi::ObjectRef { public: TVM_FFI_DEFINE_OBJECT_REF_METHODS_NULLABLE(Stmt, ffi::ObjectRef, StmtNode); }; /*! * \brief Bind a variable to a value in the enclosing scope. * * BindNode has no body field. The bound variable is visible * in all subsequent statements within the same enclosing scope (SeqStmt, * ForNode.body, etc.). This enables flat (non-nested) IR sequences. */ class BindNode : public StmtNode { public: /*! \brief The variable being bound. */ Var var; /*! \brief The value to bind to the variable. */ Expr value; static void RegisterReflection() { namespace refl = tvm::ffi::reflection; refl::ObjectDef() // TODO(tqchen): use SEqHashDefNonRecursive after the next pypi tvm-ffi release .def_ro("var", &BindNode::var, refl::AttachFieldFlag::SEqHashDefRecursive()) .def_ro("value", &BindNode::value); } TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.Bind", BindNode, StmtNode); }; /*! * \brief Managed reference to BindNode. * \sa BindNode */ class Bind : public Stmt { public: TVM_DLL Bind(Var var, Expr value, Span span = Span()); TVM_FFI_DEFINE_OBJECT_REF_METHODS_NULLABLE(Bind, Stmt, BindNode); TVM_DEFINE_OBJECT_REF_COW_METHOD(BindNode); }; /*! * \brief Define certain auxiliary attribute for the body to be a symbolic value. * This provide auxiliary information for IR passes that transforms body. * * In terms of effect, this is equivalent to Block(Evaluate(value), body). * * Examples of possible usage: * - Bound of function, variables. * - Hint which block corresponds to a parallel region. */ class AttrStmtNode : public StmtNode { public: /*! \brief this is attribute about certain node */ ffi::Any node; /*! \brief the type key of the attribute */ ffi::String attr_key; /*! \brief The attribute value, value is well defined at current scope. */ PrimExpr value; /*! \brief The body statement to be executed */ Stmt body; static void RegisterReflection() { namespace refl = tvm::ffi::reflection; refl::ObjectDef() .def_ro("node", &AttrStmtNode::node) .def_ro("attr_key", &AttrStmtNode::attr_key) .def_ro("value", &AttrStmtNode::value) .def_ro("body", &AttrStmtNode::body); } TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.AttrStmt", AttrStmtNode, StmtNode); }; /*! * \brief Managed reference to AttrStmtNode. * \sa AttrStmtNode */ class AttrStmt : public Stmt { public: TVM_DLL AttrStmt(ffi::Any node, ffi::String attr_key, PrimExpr value, Stmt body, Span span = Span()); TVM_FFI_DEFINE_OBJECT_REF_METHODS_NULLABLE(AttrStmt, Stmt, AttrStmtNode); TVM_DEFINE_OBJECT_REF_COW_METHOD(AttrStmtNode); }; /*! * \brief Assert condition, if an error occurs, return the error message. * * The error is described by: * - \p error_kind: the error kind (e.g. "RuntimeError", "TypeError", "ValueError") * - \p message_parts: an array of string fragments that are concatenated at runtime * via TVMFFIErrorSetRaisedFromCStrParts. This enables string fragment reuse * across multiple assertions to reduce binary size. */ class AssertStmtNode : public StmtNode { public: /*! \brief Condition to be checked. */ PrimExpr condition; /*! \brief The error kind, e.g. "RuntimeError", "TypeError", "ValueError". */ StringImm error_kind; /*! \brief Error message fragments, concatenated at runtime when assertion fails. */ ffi::Array message_parts; static void RegisterReflection() { namespace refl = tvm::ffi::reflection; refl::ObjectDef() .def_ro("condition", &AssertStmtNode::condition) .def_ro("error_kind", &AssertStmtNode::error_kind) .def_ro("message_parts", &AssertStmtNode::message_parts); } TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.AssertStmt", AssertStmtNode, StmtNode); }; /*! * \brief Managed reference to AssertStmtNode. * \sa AssertStmtNode */ class AssertStmt : public Stmt { public: TVM_DLL AssertStmt(PrimExpr condition, StringImm error_kind, ffi::Array message_parts, Span span = Span()); TVM_FFI_DEFINE_OBJECT_REF_METHODS_NULLABLE(AssertStmt, Stmt, AssertStmtNode); TVM_DEFINE_OBJECT_REF_COW_METHOD(AssertStmtNode); }; /*! * \brief Store value to the high dimension buffer. * * \code * * buffer[i, j] = value; * * \endcode * \sa BufferLoad */ class BufferStoreNode : public StmtNode { public: /*! \brief The buffer variable. */ Buffer buffer; /*! \brief The value to be stored. */ PrimExpr value; /*! \brief The indices location to be stored. */ ffi::Array indices; /*! \brief The predicate mask for storing values. */ ffi::Optional predicate; static void RegisterReflection() { namespace refl = tvm::ffi::reflection; refl::ObjectDef() .def_ro("buffer", &BufferStoreNode::buffer) .def_ro("value", &BufferStoreNode::value) .def_ro("indices", &BufferStoreNode::indices) .def_ro("predicate", &BufferStoreNode::predicate); } TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.BufferStore", BufferStoreNode, StmtNode); }; /*! * \brief Managed reference to BufferStoreNode. * \sa BufferStoreNode */ class BufferStore : public Stmt { public: TVM_DLL explicit BufferStore(Buffer buffer, PrimExpr value, ffi::Array indices, ffi::Optional predicate = std::nullopt, Span span = Span()); TVM_FFI_DEFINE_OBJECT_REF_METHODS_NULLABLE(BufferStore, Stmt, BufferStoreNode); TVM_DEFINE_OBJECT_REF_COW_METHOD(BufferStoreNode); }; /*! \brief Declare a buffer that can be used in the body */ class DeclBufferNode : public StmtNode { public: /*! \brief The buffer being declared */ Buffer buffer; static void RegisterReflection() { namespace refl = tvm::ffi::reflection; refl::ObjectDef().def_ro("buffer", &DeclBufferNode::buffer); } TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.DeclBuffer", DeclBufferNode, StmtNode); }; /*! \brief Managed reference to DeclBufferNode */ class DeclBuffer : public Stmt { public: TVM_DLL DeclBuffer(Buffer buffer, Span span = Span()); TVM_FFI_DEFINE_OBJECT_REF_METHODS_NULLABLE(DeclBuffer, Stmt, DeclBufferNode); TVM_DEFINE_OBJECT_REF_COW_METHOD(DeclBufferNode); }; /*! \brief Allocate a buffer and declare it in scope */ class AllocBufferNode : public StmtNode { public: /*! \brief The buffer being allocated and declared */ Buffer buffer; /*! * \brief Additional annotations about the allocation. * * These annotations can be used as auxiliary hint * to future transformations. */ ffi::Map annotations; static void RegisterReflection() { namespace refl = tvm::ffi::reflection; refl::ObjectDef() // TODO(tqchen): use SEqHashDefNonRecursive after the next pypi tvm-ffi release .def_ro("buffer", &AllocBufferNode::buffer, refl::AttachFieldFlag::SEqHashDefRecursive()) .def_ro("annotations", &AllocBufferNode::annotations); } TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.AllocBuffer", AllocBufferNode, StmtNode); }; /*! \brief Managed reference to AllocBufferNode */ class AllocBuffer : public Stmt { public: TVM_DLL AllocBuffer( Buffer buffer, ffi::Map annotations = ffi::Map(), Span span = Span()); /*! * \brief If the buffer's shape is constant, return the total number of elements. * \return The product of all shape extents if all are constant, std::nullopt otherwise. */ std::optional ConstantAllocationSize() const { int64_t result = 1; for (const PrimExpr& extent : (*this)->buffer->shape) { if (const auto* int_size = extent.as()) { result *= int_size->value; } else { return std::nullopt; } } return result; } TVM_FFI_DEFINE_OBJECT_REF_METHODS_NULLABLE(AllocBuffer, Stmt, AllocBufferNode); TVM_DEFINE_OBJECT_REF_COW_METHOD(AllocBufferNode); }; /*! * \brief The container of seq statement. * Represent a sequence of statements. */ class SeqStmtNode : public StmtNode { public: /*! \brief internal sequence content. */ ffi::Array seq; /*! \return get the size of the sequence */ size_t size() const { return seq.size(); } /*! * \brief Get the index-th element in the sequence. */ Stmt operator[](size_t index) const { return seq[index]; } static void RegisterReflection() { namespace refl = tvm::ffi::reflection; refl::ObjectDef().def_ro("seq", &SeqStmtNode::seq); } TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.SeqStmt", SeqStmtNode, StmtNode); }; /*! * \brief Evaluates an expression. * This is mostly used for putting a Call node into Stmt. * * If value do not have side-effect, this node can be safely removed. */ class EvaluateNode : public StmtNode { public: /*! \brief The expression to be evaluated. */ Expr value; static void RegisterReflection() { namespace refl = tvm::ffi::reflection; refl::ObjectDef().def_ro("value", &EvaluateNode::value); } TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.Evaluate", EvaluateNode, StmtNode); }; /*! * \brief Managed reference to EvaluateNode. * \sa EvaluateNode */ class Evaluate : public Stmt { public: TVM_DLL explicit Evaluate(Expr value, Span span = Span()); explicit Evaluate(int value, Span span = Span()) : Evaluate(PrimExpr(value), span) {} TVM_FFI_DEFINE_OBJECT_REF_METHODS_NULLABLE(Evaluate, Stmt, EvaluateNode); TVM_DEFINE_OBJECT_REF_COW_METHOD(EvaluateNode); }; /*! \brief Sequence statement. */ class SeqStmt : public Stmt { public: /*! * \brief Construct SeqStmt. * \param seq The sequence. * \param span The location of this object in the source code. */ TVM_DLL explicit SeqStmt(ffi::Array seq, Span span = Span()); /*! \return get the size of the sequence */ size_t size() const { return operator->()->size(); } /*! * \brief Get the index-th element in the sequence. */ Stmt operator[](size_t index) const { return (*(operator->()))[index]; } /*! * \brief Construct a sequence statement by flattening * all the arrays and sequences in the arguments * recursively. * * - When an argument is nullptr, it will be ignored. * - When an argument is an array or a SeqStmt, it will be flattened recursively. * - A normal Stmt will be appended to the end of the sequence. * * \note This function can directly return an element * if it is the only element in the sequence. * * \note If the only argument to this function is a SeqStmt, and if * no flattening of the SeqStmt is required, then the SeqStmt * will be returned as-is. * * \param seq_args The list of arguments to be flattened. * \tparam Args arguments * \return The constructed statement */ template static Stmt Flatten(Args&&... seq_args) { ffi::Array seq; ffi::details::for_each(Flattener(&seq), std::forward(seq_args)...); if (seq.empty()) { return Evaluate(0); } else if (seq.size() == 1) { return seq[0]; } // If the argument is a single SeqStmt argument with no // flattening or unwrapping required, then we may // return the SeqStmt as-is. if constexpr (sizeof...(seq_args) == 1) { if (auto opt = Flattener::AsSeqStmt(std::forward(seq_args)...)) { SeqStmt original = opt.value(); bool all_same = [&]() { if (original->seq.size() != seq.size()) { return false; } for (size_t i = 0; i < seq.size(); i++) { if (!original->seq[i].same_as(seq[i])) { return false; } } return true; }(); if (all_same) { return original; } } } return SeqStmt(seq); } /*! \brief Helper class to flatten sequence of arguments into Array. */ class Flattener { public: explicit Flattener(ffi::Array* seq) : seq_(seq) {} template static ffi::Optional AsSeqStmt(const T& t) { if constexpr (std::is_same_v) { return t; } if constexpr (!std::is_base_of_v) { return std::nullopt; } if constexpr (std::is_base_of_v) { if (const SeqStmtNode* ptr = t.template as()) { return ffi::GetRef(ptr); } else { return std::nullopt; } } return std::nullopt; } template void operator()(size_t i, const T& stmt_or_seq) const { if constexpr (std::is_base_of_v) { // Early bail-out, applicable to any ObjectRef if (!stmt_or_seq.defined()) { return; } } if constexpr (std::is_same_v) { // Static type-checking for a SeqStmt that could be flattened. (*this)(0, stmt_or_seq->seq); return; } if constexpr (std::is_base_of_v) { // Dynamic type-checking for a SeqStmt that could be // flattened. if (auto* op = stmt_or_seq.template as()) { operator()(0, op->seq); return; } } if constexpr (std::is_base_of_v) { // Evaluate(0) is used to represent a no-op, and may be // generated by previous calls to SeqStmt::Flatten(). These // should be removed to ensure that Flatten(a+b) is equivalent // to Flatten(Flatten(a), Flatten(b)). if (auto* op = stmt_or_seq.template as()) { if (auto* as_int = op->value.template as(); as_int && as_int->value == 0) { return; } } } if constexpr (std::is_base_of_v) { // Any other Stmt type just gets appended. seq_->push_back(stmt_or_seq); } else { // Anything else is treated as an iterable of Stmt. for (auto v : stmt_or_seq) { this->operator()(0, v); } } } private: ffi::Array* seq_; }; TVM_FFI_DEFINE_OBJECT_REF_METHODS_NULLABLE(SeqStmt, Stmt, SeqStmtNode); TVM_DEFINE_OBJECT_REF_COW_METHOD(SeqStmtNode); }; /*! * \brief IfThenElse statement. */ class IfThenElseNode : public StmtNode { public: /*! \brief The condition. */ PrimExpr condition; /*! \brief The branch to be executed when condition is true. */ Stmt then_case; /*! \brief The branch to be executed when condition is false, can be null. */ ffi::Optional else_case; static void RegisterReflection() { namespace refl = tvm::ffi::reflection; refl::ObjectDef() .def_ro("condition", &IfThenElseNode::condition) .def_ro("then_case", &IfThenElseNode::then_case) .def_ro("else_case", &IfThenElseNode::else_case); } TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.IfThenElse", IfThenElseNode, StmtNode); }; /*! * \brief Managed reference to IfThenElseNode. * \sa IfThenElseNode */ class IfThenElse : public Stmt { public: TVM_DLL IfThenElse(PrimExpr condition, Stmt then_case, ffi::Optional else_case = std::nullopt, Span span = Span()); TVM_FFI_DEFINE_OBJECT_REF_METHODS_NULLABLE(IfThenElse, Stmt, IfThenElseNode); TVM_DEFINE_OBJECT_REF_COW_METHOD(IfThenElseNode); }; /*! * \brief The kind of the loop. * * ForKind can change the control flow semantics * of the loop. So the kind field needs to be considered * in all TIR passes. */ enum class ForKind : int { /*! \brief default semantics -- serial execution. */ kSerial = 0, /*! \brief Parallel execution on CPU. */ kParallel = 1, /*! * \brief Vector SIMD loop. * The loop body will be vectorized. */ kVectorized = 2, /*! \brief The loop body must be unrolled. */ kUnrolled = 3, /*! * \brief The loop variable is bound to a thread in * an environment. In the final stage of lowering, * the loop is simply removed and the loop variable is * mapped to the corresponding context thread. */ kThreadBinding = 4 }; /*! * \brief A for loop, with possible type annotations. * * \code * * for (loop_var = min; loop_var < min + extent; loop_var += step) { * // body * } * \endcode */ class ForNode : public StmtNode { public: /*! \brief The loop variable. */ PrimVar loop_var; /*! \brief The minimum value of iteration. */ PrimExpr min; /*! \brief The extent of the iteration. */ PrimExpr extent; /*! \brief The kind of the for loop. */ ForKind kind; /*! \brief The body of the for loop. */ Stmt body; /*! * \brief Only valid when kind == ForKind::kThreadBinding * The context thread that this loop variable bounds to. */ ffi::Optional thread_binding; /*! * \brief Additional annotations about the loop. * * These annotations can be used as auxiliary hint * to future transformations. An annotation should * not change the control flow semantics of the loop * and can be ignored in most passes. */ ffi::Map annotations; /*! * \brief The loop step. It is one if not specified. */ ffi::Optional step; static void RegisterReflection() { namespace refl = tvm::ffi::reflection; refl::ObjectDef() .def_ro("loop_var", &ForNode::loop_var, refl::AttachFieldFlag::SEqHashDefRecursive()) .def_ro("min", &ForNode::min) .def_ro("extent", &ForNode::extent) .def_ro("kind", &ForNode::kind) .def_ro("body", &ForNode::body) .def_ro("thread_binding", &ForNode::thread_binding) .def_ro("annotations", &ForNode::annotations) .def_ro("step", &ForNode::step); } /*! \brief Check it is a loop without nontrivial loop step. */ bool HasTrivialStep() const; TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.For", ForNode, StmtNode); }; /*! * \brief Managed reference to ForNode. * \sa ForNode */ class For : public Stmt { public: TVM_DLL For(PrimVar loop_var, PrimExpr min, PrimExpr extent, ForKind kind, Stmt body, ffi::Optional thread_binding = std::nullopt, ffi::Map annotations = {}, ffi::Optional step = std::nullopt, Span span = Span()); TVM_FFI_DEFINE_OBJECT_REF_METHODS_NULLABLE(For, Stmt, ForNode); TVM_DEFINE_OBJECT_REF_COW_METHOD(ForNode); }; /*! * \brief A While loop * * \code * * while (condition) * body * * \endcode */ class WhileNode : public StmtNode { public: /*! \brief The termination condition. */ PrimExpr condition; /*! \brief The body of the while loop. */ Stmt body; static void RegisterReflection() { namespace refl = tvm::ffi::reflection; refl::ObjectDef() .def_ro("condition", &WhileNode::condition) .def_ro("body", &WhileNode::body); } TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.While", WhileNode, StmtNode); }; /*! * \brief Managed reference to WhileNode. * \sa WhileNode */ class While : public Stmt { public: TVM_DLL While(PrimExpr condition, Stmt body, Span span = Span()); TVM_FFI_DEFINE_OBJECT_REF_METHODS_NULLABLE(While, Stmt, WhileNode); TVM_DEFINE_OBJECT_REF_COW_METHOD(WhileNode); }; /*! * \brief A Break in control flow. */ class BreakNode : public StmtNode { public: static void RegisterReflection() { namespace refl = tvm::ffi::reflection; refl::ObjectDef(); } TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.Break", BreakNode, StmtNode); }; /*! * \brief Managed reference to BreakNode. * \sa BreakNode */ class Break : public Stmt { public: TVM_DLL explicit Break(Span span); TVM_FFI_DEFINE_OBJECT_REF_METHODS_NULLABLE(Break, Stmt, BreakNode); TVM_DEFINE_OBJECT_REF_COW_METHOD(BreakNode); }; /*! * \brief A Continue in control flow. */ class ContinueNode : public StmtNode { public: static void RegisterReflection() { namespace refl = tvm::ffi::reflection; refl::ObjectDef(); } TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.Continue", ContinueNode, StmtNode); }; /*! * \brief Managed reference to ContinueNode. * \sa ContinueNode */ class Continue : public Stmt { public: TVM_DLL explicit Continue(Span span); TVM_FFI_DEFINE_OBJECT_REF_METHODS_NULLABLE(Continue, Stmt, ContinueNode); TVM_DEFINE_OBJECT_REF_COW_METHOD(ContinueNode); }; /*! * \brief Representing the region of multi-dimensional buffer access. */ class BufferRegionNode : public PrimExprConvertibleNode { public: /*! \brief The buffer of the buffer region. */ Buffer buffer; /*! \brief The region array of the buffer region. */ ffi::Array region; static void RegisterReflection() { namespace refl = tvm::ffi::reflection; refl::ObjectDef() .def_ro("buffer", &BufferRegionNode::buffer) .def_ro("region", &BufferRegionNode::region); } TVM_DLL PrimExpr ToPrimExpr() const final; static constexpr TVMFFISEqHashKind _type_s_eq_hash_kind = kTVMFFISEqHashKindTreeNode; TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.BufferRegion", BufferRegionNode, PrimExprConvertibleNode); }; /*! * \brief Managed reference to BufferRegionNode. * \sa BufferRegionNode */ class BufferRegion : public PrimExprConvertible { public: TVM_DLL explicit BufferRegion(Buffer buffer, ffi::Array region); /*! * \brief Create a BufferRegion which is full region of the given buffer. * \param buffer The buffer to generate full BufferRegion. * \return The BufferRegion which covers all region of the given buffer */ TVM_DLL static BufferRegion FullRegion(Buffer buffer); /*! * \brief Create a BufferRegion which is a single point of the given buffer. * \param buffer The buffer to generate single point BufferRegion. * \param indices The access point indices of the buffer * \return The BufferRegion which is the single point of the given buffer. */ TVM_DLL static BufferRegion FromPoint(Buffer buffer, ffi::Array indices); TVM_FFI_DEFINE_OBJECT_REF_METHODS_NULLABLE(BufferRegion, PrimExprConvertible, BufferRegionNode); TVM_DEFINE_OBJECT_REF_COW_METHOD(BufferRegionNode); }; /*! * \brief Match introduces a constraint that the source buffer region can be remapped to the data * layout specified by the buffer field. The constraint can be checked in later part of lowering (or * optionally during runtime). * * MatchBufferRegion provides a mechanism to represent data layout and compactness constraints in * low-level hardware primitives in the IR and defer the check after the sequence of * transformations. */ class MatchBufferRegionNode : public ffi::Object { public: /*! \brief The target buffer. */ Buffer buffer; /*! \brief The source buffer region. */ BufferRegion source; static void RegisterReflection() { namespace refl = tvm::ffi::reflection; refl::ObjectDef() .def_ro("buffer", &MatchBufferRegionNode::buffer) .def_ro("source", &MatchBufferRegionNode::source); } static constexpr TVMFFISEqHashKind _type_s_eq_hash_kind = kTVMFFISEqHashKindTreeNode; TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.MatchBufferRegion", MatchBufferRegionNode, ffi::Object); }; /*! * \brief Managed reference to MatchBufferRegionNode. * \sa MatchBufferRegionNode */ class MatchBufferRegion : public ffi::ObjectRef { public: TVM_DLL explicit MatchBufferRegion(Buffer buffer, BufferRegion source); TVM_FFI_DEFINE_OBJECT_REF_METHODS_NULLABLE(MatchBufferRegion, ffi::ObjectRef, MatchBufferRegionNode); TVM_DEFINE_OBJECT_REF_COW_METHOD(MatchBufferRegionNode); }; /*! * \brief A block is a basic schedule unit in TIR. * \note SBlock's body is parameterized by iter vars. * \code * * with T.sblock(name): * v0 = T.axis.S(domain, value0) * v1 = T.axis.R(domain, value1) * ... * T.reads([buffer0[start:end, ...], ...]) * T.writes([buffer1[start:end, ...], ...]) * T.where(predicate) * buffer2 = T.alloc_buffer(shape, dtype) * buffer3 = T.match_buffer(source_buffer[start:end, ...]) * T.attr({attr_key: attr_value, ...}) * with T.init(): * // init body * // body * * \endcode */ class SBlockNode : public StmtNode { public: /*! \brief The variables of the block. */ ffi::Array iter_vars; /*! \brief The read buffer regions of the block. */ ffi::Array reads; /*! \brief The write buffer regions of the block. */ ffi::Array writes; /*! \brief The name_hint of the block. */ ffi::String name_hint; /*! \brief The buffer allocated in the block. */ ffi::Array alloc_buffers; /*! \brief The match buffer regions. */ ffi::Array match_buffers; /*! \brief The annotation of the block. */ ffi::Map annotations; /*! * \brief The init statement is executed during the first iteration of reduction loops in a * reduction block. The optional init field allows us to represent initialization and * reduction update in a single block and transform them collectively. * We also provide primitives to decompose the init into a separate block during scheduling. * Init field is `std::nullopt` if there is no reduction iter_vars */ ffi::Optional init; /*! \brief The body of the block. */ Stmt body; static void RegisterReflection() { namespace refl = tvm::ffi::reflection; refl::ObjectDef() .def_ro("iter_vars", &SBlockNode::iter_vars, refl::AttachFieldFlag::SEqHashDefRecursive()) .def_ro("reads", &SBlockNode::reads) .def_ro("writes", &SBlockNode::writes) .def_ro("name_hint", &SBlockNode::name_hint, refl::AttachFieldFlag::SEqHashIgnore()) .def_ro("alloc_buffers", &SBlockNode::alloc_buffers) .def_ro("match_buffers", &SBlockNode::match_buffers) .def_ro("annotations", &SBlockNode::annotations) .def_ro("init", &SBlockNode::init) .def_ro("body", &SBlockNode::body); } TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.SBlock", SBlockNode, StmtNode); }; /*! * \brief Managed reference to SBlockNode. * \sa SBlockNode */ class SBlock : public Stmt { public: TVM_DLL explicit SBlock( ffi::Array iter_vars, ffi::Array reads, ffi::Array writes, ffi::String name_hint, Stmt body, ffi::Optional init = std::nullopt, ffi::Array alloc_buffers = ffi::Array(), ffi::Array match_buffers = ffi::Array(), ffi::Map annotations = ffi::Map(), Span span = Span()); TVM_DLL explicit SBlock(ffi::String name_hint, Stmt body, ffi::Array alloc_buffers = ffi::Array(), Span span = Span()); TVM_FFI_DEFINE_OBJECT_REF_METHODS_NULLABLE(SBlock, Stmt, SBlockNode); TVM_DEFINE_OBJECT_REF_COW_METHOD(SBlockNode); }; /*! * \brief A block realization node represents execution of the block at the binding values. */ class SBlockRealizeNode : public StmtNode { public: /*! \brief The corresponding values of the iter vars. */ ffi::Array iter_values; /*! * \brief The predicate of the block realization, the block will only be executed when the * predicate is true. */ PrimExpr predicate; /*! \brief The block to be realized. */ SBlock block; static void RegisterReflection() { namespace refl = tvm::ffi::reflection; refl::ObjectDef() .def_ro("iter_values", &SBlockRealizeNode::iter_values) .def_ro("predicate", &SBlockRealizeNode::predicate) .def_ro("block", &SBlockRealizeNode::block); } TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.SBlockRealize", SBlockRealizeNode, StmtNode); }; /*! * \brief Managed reference to BlockRealizeNode * \sa BlockRealizeNode */ class SBlockRealize : public Stmt { public: TVM_DLL explicit SBlockRealize(ffi::Array iter_values, PrimExpr predicate, SBlock block, Span span = Span()); TVM_FFI_DEFINE_OBJECT_REF_METHODS_NULLABLE(SBlockRealize, Stmt, SBlockRealizeNode); TVM_DEFINE_OBJECT_REF_COW_METHOD(SBlockRealizeNode); }; /*! * \brief Standalone statement that declares a scope-id binding (e.g. cta_id, * warp_id, lane_id). Carries a ``ScopeIdDef`` value. * * Each declaration is a flat stmt within the device-region body. The declared * ``Var``\ s are visible in subsequent stmts in the same enclosing scope * (the AttrStmt ``kDeviceEntry`` body), analogous to ``BindNode``. */ class ScopeIdDefStmtNode : public StmtNode { public: /*! \brief The scope-id definition (Vars + extents + binding). */ ScopeIdDef def; static void RegisterReflection() { namespace refl = tvm::ffi::reflection; refl::ObjectDef().def_ro("def", &ScopeIdDefStmtNode::def); } TVM_FFI_DECLARE_OBJECT_INFO_FINAL("tirx.ScopeIdDefStmt", ScopeIdDefStmtNode, StmtNode); }; /*! \brief Managed reference to ScopeIdDefStmtNode. */ class ScopeIdDefStmt : public Stmt { public: TVM_DLL ScopeIdDefStmt(ScopeIdDef def, Span span = Span()); TVM_FFI_DEFINE_OBJECT_REF_METHODS_NULLABLE(ScopeIdDefStmt, Stmt, ScopeIdDefStmtNode); TVM_DEFINE_OBJECT_REF_COW_METHOD(ScopeIdDefStmtNode); }; /*! \brief namespace of possible attributes in AttrStmt.attr_key */ namespace attr { /*! \brief Mark stores/loads with their bounds. */ constexpr const char* buffer_bound = "buffer_bound"; /*! * \brief Mark the scope as when computation start to happen. * This can hint some code generator to create a new function for compute. */ constexpr const char* compute_scope = "compute_scope"; /*! \brief The allocation device for global malloc in host. */ constexpr const char* device_id = "device_id"; /*! \brief Mark that it is in the device scope. */ constexpr const char* device_scope = "device_scope"; /*! \brief The device type. */ constexpr const char* device_type = "device_type"; /*! * \brief Mark the scope as generated by extern primitive. * Such scope can contain arbitrary ir program and we need to be careful * when making certain assumptions about the structure of the program. */ constexpr const char* extern_scope = "extern_scope"; /*! \brief Pragma: auto-unroll, max_step */ constexpr const char* pragma_auto_unroll_max_step = "pragma_auto_unroll_max_step"; /*! \brief Import C source or file into the final code gen module */ constexpr const char* pragma_import_c = "pragma_import_c"; /*! \brief Import llvm source or file into the final code gen module */ constexpr const char* pragma_import_llvm = "pragma_import_llvm"; /*! \brief Mark region is guarded by the pragma extension */ constexpr const char* pragma_scope_prefix = "pragma_"; /*! \brief Try to modify the AST to support Tensor Core */ constexpr const char* pragma_tensor_core = "pragma_tensor_core"; /*! \brief Pragma: unroll explicit */ constexpr const char* pragma_unroll_explicit = "pragma_unroll_explicit"; /*! \brief Mark storage alignment requirement of buffers */ constexpr const char* storage_alignment = "storage_alignment"; /*! \brief Mark launching extent of thread, used by device API. */ constexpr const char* thread_extent = "thread_extent"; /*! \brief Annotation key on AllocBuffer marking the allocation as volatile. */ constexpr const char* kVolatile = "tirx.volatile"; /*! * \brief Marks the layout transforms to be used for a tensor. * * Only applies to a DataProducer, as it should be made part of the * PrimFunc attributes for TIR. */ constexpr const char* layout_transforms = "layout_transforms"; /*! * \brief Marks the physical axis separators * * Only applies to a DataProducer, as it should be made part of the * Buffer definition in a PrimFunc. See `BufferNode::axis_separators` * for more details. */ constexpr const char* axis_separators = "axis_separators"; /*! * \brief Marks production of double buffer data */ constexpr const char* double_buffer_scope = "double_buffer_scope"; /*! * \brief Marks region used by double buffer write */ constexpr const char* double_buffer_write = "double_buffer_write"; /*! \brief Mark of scan update scope */ constexpr const char* scan_update_scope = "scan_update_scope"; /*! \brief Mark of scan init scope */ constexpr const char* scan_init_scope = "scan_init_scope"; /*! * \brief Mark alignment of buffer dimension * stmt.node is Tensor * stmt.value is tvm_tuple(dim, align, offset) * This gives hint to require stride of dim to be k * align + offset. */ constexpr const char* buffer_dim_align = "buffer_dim_align"; /*! \brief Mark buffer initial addr alignment in bytes */ constexpr const char* buffer_data_alignment = "buffer_data_alignment"; /*! \brief Mark buffer allocated addr in bytes */ constexpr const char* buffer_allocated_addr = "buffer_allocated_addr"; /*! * \brief Bind the buffer specification to the region of the op * When this scope occurs, the stmt.node is a ffi::Array = [buffer, tensor] * stmt.value is a tvm_tuple(min0, extent0, min1, extent1, ...). * The scope represents that we need to bind the storage region of tensor to buffer. * This will affect replacement of some variables inside the scope that * corresponds to field of buffer to be the actual expressions of tensor during * storage flattening phase. */ constexpr const char* buffer_bind_scope = "buffer_bind_scope"; // Pipeline related attributes /*! \brief channel read scope */ constexpr const char* channel_read_scope = "channel_read_scope"; /*! \brief Advance step of channel after end of scope */ constexpr const char* channel_read_advance = "channel_read_advance"; /*! \brief channel write scope */ constexpr const char* channel_write_scope = "channel_write_scope"; /*! \brief Advance step of channel after end of scope */ constexpr const char* channel_write_advance = "channel_write_advance"; /*! \brief pipeline stage scope, implies always execution */ constexpr const char* pipeline_stage_scope = "pipeline_stage_scope"; /*! \brief pipeline execution scope, implies the scope can be pipelined. */ constexpr const char* pipeline_exec_scope = "pipeline_exec_scope"; /*! * \brief Mark that the attached statement runs asynchronously. */ constexpr const char* async_scope = "async_scope"; /*! * \brief Annotations for invoking and synchronizing asynchronous operations. * Synchronization is done in terms of "queue": It is an abstract entity associated * with each asynchronous unit, and it tracks invocations and completions of asynchronous * operations in the FIFO order. * * Similarly to PTX instructions commit_group and wait_group, these annotations express * synchronization by "counting": * * async_commit_queue(i): Group one or more invocations of async operations in the given scope, * and "commit" (or push) them to the queue i. A group of operations committed together is * awaited as one chunk. Groups committed to the same queue complete in the FIFO order. * * async_wait_queue(i, N): Block until only N most recent committed groups are still in-flight at * the queue i. N does not have to be a constant, but some backends may require a constant count. */ constexpr const char* async_commit_queue_scope = "async_commit_queue_scope"; constexpr const char* async_wait_queue_scope = "async_wait_queue_scope"; constexpr const char* async_wait_inflight_count = "async_wait_inflight_count"; /*! * \brief Mark that the shape of TensorCore fragment */ constexpr const char* fragment_shape = "fragment_shape"; /*! * \brief Mark that the layout of TensorCore fragment */ constexpr const char* fragment_layout = "fragment_layout"; /*! * \brief Mark that the kernel is hand threaded and doesn't need syncs inserted */ constexpr const char* hand_threaded = "hand_threaded"; /*! * \brief Mark whether the script-completer need to fill in missing access region * during script parsing. * \note The result should be a integer mask with range [0, 4). * if (mask & 1) the read region should be detected, * if (mask & 2) the write region should be detected. */ constexpr const char* script_parsing_detect_access = "tirx.script_parsing_detect_access"; /*! * \brief Mark that the loop should be partitioned. */ constexpr const char* pragma_loop_partition_hint = "pragma_loop_partition_hint"; /*! \brief Mark the stage of a statement in the software pipeline */ constexpr const char* software_pipeline_stage = "software_pipeline_stage"; /*! \brief Mark the order of a statement in the software pipeline */ constexpr const char* software_pipeline_order = "software_pipeline_order"; /*! \brief List stages in the software pipeline that should run asynchronously * \note All statements in the provided stages are assumed to have asynchronous * semantics (e.g. CUDA async global to shared memory copy). */ constexpr const char* software_pipeline_async_stages = "software_pipeline_async_stages"; /*! \brief Mark the buffers which is const access and can be transformed layout. */ constexpr const char* layout_free_buffers = "layout_free_buffers"; /*! \brief Mark the local stage for the shared memory access should be added. */ constexpr const char* manifest_shared_memory_local_stage = "tirx.manifest_shared_memory_local_stage"; /*! \brief Mark the tiling structure of blocks that are applied by rule Multi-Level-Tiling */ constexpr const char* meta_schedule_tiling_structure = "meta_schedule.tiling_structure"; /*! * \brief Mark that the loop should be further skip and bound to environment threads to enable * cooperative fetching. */ constexpr const char* meta_schedule_cooperative_fetch = "meta_schedule.cooperative_fetch"; /*! \brief The allowed range of thread extent in thread bindings */ constexpr const char* meta_schedule_thread_extent_low_inclusive = "meta_schedule.thread_extent_low_inclusive"; /*! \brief The allowed range of thread extent in thread bindings */ constexpr const char* meta_schedule_thread_extent_high_inclusive = "meta_schedule.thread_extent_high_inclusive"; /*! \brief Mark the block whose producer needs to be applied by rule Random-Compute-Location */ constexpr const char* meta_schedule_random_compute_producer = "meta_schedule.random_compute_producer"; /*! \brief Mark auto-parallel setting on the block. */ constexpr const char* meta_schedule_parallel = "meta_schedule.parallel"; /*! \brief Mark auto-vectorize setting on the block. */ constexpr const char* meta_schedule_vectorize = "meta_schedule.vectorize"; /*! \brief Mark auto-unroll setting on the block. */ constexpr const char* meta_schedule_unroll_explicit = "meta_schedule.unroll_explicit"; /*! \brief Mark auto-unroll setting on the block. */ constexpr const char* meta_schedule_unroll_implicit = "meta_schedule.unroll_implicit"; /*! \brief Mark that a block should be further rewritten using tensorization. */ constexpr const char* meta_schedule_auto_tensorize = "meta_schedule.auto_tensorize"; /*! \brief Mark that a block is a preprocessor block for layout rewrite. */ constexpr const char* meta_schedule_layout_rewrite_preproc = "meta_schedule.layout_rewrite_preproc"; /*! * \brief Mark that the init statement of a block should be further rewritten using tensorization. */ constexpr const char* meta_schedule_auto_tensorize_init = "meta_schedule.auto_tensorize_init"; /*! * \brief Mark that the block need to add predicate for block var bounds during lowering */ constexpr const char* require_block_var_bound_predicate = "require_bound_predicate"; /*! \brief Mark that tensor core is enabled in the PrimExpr */ constexpr const char* meta_schedule_tensor_core_enabled = "meta_schedule.tensor_core_enabled"; /*! * \brief Mark a block as generated by cache_read or cache_write block. * 0 means cache_read; 1 means cache_write. * \sa meta_schedule_cache_type_read * \sa meta_schedule_cache_type_write */ constexpr const char* meta_schedule_cache_type = "meta_schedule.cache_type"; /*! \sa meta_schedule_cache_type */ constexpr const int meta_schedule_cache_type_read = 0; /*! \sa meta_schedule_cache_type */ constexpr const int meta_schedule_cache_type_write = 1; /*! \brief Mark auto copy for memhammer */ constexpr const char* auto_copy = "auto_copy"; /*! \brief Mark local stage constraint on data copy */ constexpr const char* local_stage = "local_stage"; /*! \brief Mark vectorization length constraint on block */ constexpr const char* vector_bytes = "vector_bytes"; /*! * \brief Mark that a block is executed by a warp. This implies the extend of threadIdx.x is * warp size. */ constexpr const char* warp_execution = "warp_execution"; /*! \brief Mark that a block is disallowed in auto inline. */ constexpr const char* meta_schedule_inline_rule = "meta_schedule.inline_rule"; /*! \brief Mark that a block has an explicitly specified read region. * This is used to override the default read region inference in TIR. */ constexpr const char* explicit_read_region = "explicit_read_region"; /*! \brief Mark that a block has an explicitly specified write region. * This is used to override the default write region inference in TIR. */ constexpr const char* explicit_write_region = "explicit_write_region"; constexpr const char* tensorized_nki_instruction = "tensorized_nki_instruction"; /*! \brief ,ark a ForNode represent an irregular loop of non-structural control flow edges. */ constexpr const char* irregular_loop_mark = "irregular_loop_mark"; /*! * \brief Mark the kernel as persistent. */ constexpr const char* kPersistentKernel = "tirx.persistent_kernel"; /*! * \brief Mark the device-region entry within a PrimFunc body. The * ``AttrStmt`` so-keyed has a body that is the device-side region; anything * before the marker (within the PrimFunc body) is host code. Value is * ``IntImm("bool", 1)`` -- a boolean marker, similar to ``kPersistentKernel``. */ constexpr const char* kDeviceEntry = "tirx.device_entry"; /*! * \brief Check if attr_key is a pragma key extension * \param attr_key The attr key to be compared * \return true if it is a pragma key */ inline bool IsPragmaKey(const std::string& attr_key) { return attr_key.compare(0, 7, "pragma_") == 0; } } // namespace attr /*! * \brief Create a type annotation expression * \param dtype The data type * \param span The location of this object in the source code. * \return Expr a expression with dtype. */ TVM_DLL PrimExpr TypeAnnotation(PrimType dtype, Span span = Span()); // overload printing of for type. TVM_DLL std::ostream& operator<<(std::ostream& os, ForKind kind); // inline implementations inline const char* ForKind2String(ForKind t) { switch (t) { case ForKind::kSerial: return "serial"; case ForKind::kParallel: return "parallel"; case ForKind::kVectorized: return "vectorized"; case ForKind::kUnrolled: return "unroll"; case ForKind::kThreadBinding: return "thread_binding"; } TVM_FFI_THROW(InternalError) << "Unknown ForKind" << t; TVM_FFI_UNREACHABLE(); } } // namespace tirx } // namespace tvm #endif // TVM_TIR_STMT_H_