/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ /*! * \file tvm/s_tir/data_layout.h * \brief SLayout expression to describe the data organization of a tensor. * And SBijectiveLayout to mapping two data layouts between each other. */ #ifndef TVM_S_TIR_DATA_LAYOUT_H_ #define TVM_S_TIR_DATA_LAYOUT_H_ #include #include #include #include #include #include #include #include #include "tvm/tirx/var.h" namespace tvm { namespace tirx { class SLayout; class SLayoutAxis { public: static const SLayoutAxis& Get(const char name); // Get the singleton SLayoutAxis using itvar->var->name_hint static const SLayoutAxis& Get(const tirx::IterVar& itvar); // Get the singleton SLayoutAxis using name[0] (size of name must be 1). static const SLayoutAxis& Get(const std::string& name); inline bool IsPrimal() const { return name_ >= 'A' && name_ <= 'Z'; } inline std::string name() const { return std::string(1, name_); } // if current axis is primal, switch the axis to its subordinate one, // else switch to the primal. inline const SLayoutAxis& ToDual() const { if (name_ >= 'A' && name_ <= 'Z') { return SLayoutAxis::Get(name_ - 'A' + 'a'); } else { return SLayoutAxis::Get(name_ - 'a' + 'A'); } } // return the primal axis. If it is already primal, return itself. const SLayoutAxis& ToPrimal() const { return IsPrimal() ? *this : ToDual(); } // return the subordinate axis. If it is already subordinate, return itself. const SLayoutAxis& ToSubordinate() const { return IsPrimal() ? ToDual() : *this; } inline bool operator==(const SLayoutAxis& rhs) const { return name_ == rhs.name_; } friend std::ostream& operator<<(std::ostream& os, const SLayoutAxis& l) { os << l.name(); return os; } private: static const SLayoutAxis UPPER_CASE[]; static const SLayoutAxis LOWER_CASE[]; SLayoutAxis(const SLayoutAxis&); SLayoutAxis& operator=(const SLayoutAxis&); explicit SLayoutAxis(const char name) : name_(name) {} const char name_; }; /*! * \brief SLayout is to describe how data is organized within an N-dimention tensor. * It is composed of upper cases, lower cases and numbers, * where upper case indicates a primal axis and * the corresponding lower case with factor size indicates the subordinate axis. * For example, NCHW16c can describe a 5-D tensor of * [batch_size, channel, height, width, channel_block]. * Here subordinate axis channel_block=16 is the factor size of the primal axis C (channel). * SLayout for scalar is defined, while both its name and axes have size 0. */ class SLayoutNode : public ffi::Object { public: /*! \brief string representation of layout, "" for scalar. */ ffi::String name; /*! \brief specify each axis of the layout, * in which the variable name is the name of the axis. * The IterVar's extent indicates the size of the axis, * it is a variable for a primal axis, but a constant for a subordinate axis. * Empty for scalar's layout. */ ffi::Array axes; static void RegisterReflection() { namespace refl = tvm::ffi::reflection; refl::ObjectDef() .def_ro("name", &SLayoutNode::name) .def_ro("axes", &SLayoutNode::axes); } TVM_FFI_DECLARE_OBJECT_INFO_FINAL("s_tir.SLayout", SLayoutNode, ffi::Object); }; /*! * \brief Managed reference to SLayoutNode * \sa SLayoutNode */ class SLayout : public ffi::ObjectRef { public: explicit SLayout(const ffi::Array& axes); /*! \brief construct from a string */ SLayout(const tvm::ffi::String& name) : SLayout(name.operator std::string()) {} // NOLINT(*) /*! \brief construct from a string */ SLayout(const char* name) : SLayout(std::string(name)) {} // NOLINT(*) /*! * \brief construct from a string. * \param name input in layout convention: * upper case indicates a dimension and * the corresponding lower case with factor size * indicates the split dimension. * return undefined layout if "__undef__" is passed. * \param index_ty The type of generated axes vars in the returned layout. * It is required to be integer type. */ TVM_DLL SLayout(const std::string& name, PrimType index_ty = PrimType::Int(32)); // NOLINT(*) /*! * \brief access the internal node container * \return the pointer to the internal node container */ SLayoutNode* operator->() { return static_cast(get_mutable()); } /*! * \brief Return an undefined layout. * \return a (global) undefined layout. */ static const SLayout& Undef() { static SLayout undef; return undef; } /*! * \brief Packs the Given Array of IterVars into a Single IterVar. Each IterVar in the Array * should represent either a single primal axis or one or more subordinate axis * \param iters Array of iter vars to be packed * \return A packed iter var */ static IterVar PackIterVar(ffi::Array iters); /*! * \brief Unpacks a Packed IterVar into its constituents * \param packed_iter A Packed IterVar containing a single primal axis or one or more subordinate * axis * \return Constituent IterVars */ static ffi::Array UnpackIterVar(IterVar packed_iter); /*! * \brief Returns a sub-layout which is the portion of the object * that starts at dimension \p pos and spans \p len dimensions * (or until the end of the layout, whichever comes first). * \param pos The start position. * \param len The length of the sub-layout. if 0, return layout of scalar * \return A newly constructed SLayout object. */ SLayout SubLayout(size_t pos, size_t len) const; /*! * \brief Split \p axis by \p size and put the sub-axis to position \p target_pos. * \param axis The source axis to be split. It must be a primal-axis; * \param target_pos The target position of the newly split subordinate-axis. * \param factor size of the sub-dimension. * \return A newly constructed SLayout object. */ SLayout Split(const SLayoutAxis& axis, size_t target_pos, int32_t factor) const; /*! \return number of dimensions */ inline size_t ndim() const { if (!defined()) return 0; return operator->()->axes.size(); } /*! \return number of super dimensions */ inline size_t ndim_primal() const { if (!defined()) return 0; size_t ct = 0; for (auto px : operator->()->axes) { auto iter_vars = UnpackIterVar(px); for (auto x : iter_vars) { if (SLayoutAxis::Get(x).IsPrimal()) { ct++; } } } return ct; } /*! * \brief Returns a new layout where the dims have been expanded to match the primal dimensions. * \param dst_layout The dst layout to which current layout has to be expanded. * \return The expanded SLayout. */ inline SLayout ExpandPrimal(const SLayout& dst_layout) { SLayout new_src_layout; // 1) Find the axis which are missing in the current layout. Make them the prefix. std::string new_src_layout_str = ""; for (auto packed_axis : dst_layout->axes) { auto iter_vars = UnpackIterVar(packed_axis); for (auto dst_axis : iter_vars) { if (SLayoutAxis::Get(dst_axis).IsPrimal()) { if (!this->Contains(SLayoutAxis::Get(dst_axis))) { new_src_layout_str += dst_axis->var->name_hint; } } } } // 2) Now, add the primal axis of the current layout. new_src_layout_str += this->name(); new_src_layout = SLayout(new_src_layout_str); return new_src_layout; } /*! * \brief return the index of the input axis. * If it is not found in the layout or the layout is undefined, * return -1. * \param axis The input axis either a layout axis, or a packed axis * \return the index or -1 if not found. */ inline int32_t IndexOf(const std::string& axis) const { if (!this->defined()) return -1; const auto axes = operator->()->axes; for (size_t i = 0; i < axes.size(); ++i) { if (axes[i]->var->name_hint == axis) return static_cast(i); } return -1; } /*! * \brief return the index of the input axis. * If it is not found in the layout or the layout is undefined, * return -1. * \param axis the input layout axis. * \return the index or -1 if not found. */ inline int32_t IndexOf(const SLayoutAxis& axis) const { return IndexOf(axis.name()); } /*! * \brief return the index of the input axis. * If it is not found in the layout or the layout is undefined, * return -1. * \param iter the input iter var. * \return the index or -1 if not found. */ inline int32_t IndexOf(const tirx::IterVar& iter) const { return IndexOf(iter->var->name_hint); } /*! * \brief Get the factor size of the subordinate axis. * \param axis the input primal-axis or subordinate-axis. * \return the size of the subordinate-axis of \p axis (if \p axis is a primal-axis), * or the size of \p axis itself (if \p axis is a subordinate-axis). * Return -1 if \p axis is not in the layout the layout is undefined. */ int32_t FactorOf(const SLayoutAxis& axis) const; /*! * \brief Whether the layout contains an axis. * \param axis axis to be checked. * \return Whether the layout contains the axis. */ bool Contains(const SLayoutAxis& axis) const { if (!defined()) return false; for (const tirx::IterVar packed_var : operator->()->axes) { auto iter_vars = UnpackIterVar(packed_var); for (auto var : iter_vars) { if (var->var->name_hint == axis.name()) { return true; } } } return false; } const SLayoutAxis& operator[](int32_t i) const { TVM_FFI_ICHECK(defined()) << "Try to access axis from an undefined layout."; int32_t index = i < 0 ? static_cast(ndim() + i) : i; TVM_FFI_ICHECK(index >= 0 && static_cast(index) < ndim()) << "Invalid index " << i; const tirx::IterVar axis = operator->()->axes[index]; return SLayoutAxis::Get(axis); } IterVar PackedAxisAt(int32_t i) const { TVM_FFI_ICHECK(defined()) << "Try to access axis from an undefined layout."; int32_t index = i < 0 ? static_cast(ndim() + i) : i; TVM_FFI_ICHECK(index >= 0 && static_cast(index) < ndim()) << "Invalid index " << i; const tirx::IterVar axis = operator->()->axes[index]; return axis; } /*! \return the string description of the layout */ inline std::string name() const { if (!defined()) return "__undef__"; return operator->()->name; } /*! * \brief Whether the two layouts are equal. * \param rhs Another layout. * \return whether the two layouts are equal. */ inline bool Equals(const SLayout& rhs) const { return name() == rhs.name(); } /*! * \brief allow output string of layout to ostream * \param os the output stream * \param l the layout * \return the ostream */ friend std::ostream& operator<<(std::ostream& os, const SLayout& l) { os << l.name(); return os; } TVM_FFI_DEFINE_OBJECT_REF_METHODS_NULLABLE(SLayout, ffi::ObjectRef, SLayoutNode); }; // Internal node container SBijectiveLayout class SBijectiveLayoutNode : public ffi::Object { public: /*! \brief Describes how source axes can be mapped to the destination axes, * e.g., [i0 / 16, i1, i0 % 16] can describe NC -> NC16n */ ffi::Array index_forward_rule; /*! \brief Describes how destination axes can be mapped to the source axes */ ffi::Array index_backward_rule; /*! \brief Describes how source shapes can be mapped to the destination shapes */ ffi::Array shape_forward_rule; /*! \brief Describes how destination shapes can be mapped to the source shapes */ ffi::Array shape_backward_rule; /*! \brief The source layout */ SLayout src_layout; /*! \brief The destination layout */ SLayout dst_layout; static void RegisterReflection() { namespace refl = tvm::ffi::reflection; refl::ObjectDef() .def_ro("src_layout", &SBijectiveLayoutNode::src_layout) .def_ro("dst_layout", &SBijectiveLayoutNode::dst_layout) .def_ro("index_forward_rule", &SBijectiveLayoutNode::index_forward_rule) .def_ro("index_backward_rule", &SBijectiveLayoutNode::index_backward_rule) .def_ro("shape_forward_rule", &SBijectiveLayoutNode::shape_forward_rule) .def_ro("shape_backward_rule", &SBijectiveLayoutNode::shape_backward_rule); } TVM_FFI_DECLARE_OBJECT_INFO_FINAL("s_tir.SBijectiveLayout", SBijectiveLayoutNode, ffi::Object); }; /*! * \brief Bijective function mapping for data layout transformation. * Given two SLayout, SBijectiveLayout build and store the mapping rules, * provides API to transform N-dimention tensor from the source indices (i0, i1, .., im) * to the destination indices (j0, j1, .., jm). */ class SBijectiveLayout : public ffi::ObjectRef { public: /*! * \brief The constructor * \param src_layout The source layout * \param dst_layout The destination layout */ TVM_DLL SBijectiveLayout(SLayout src_layout, SLayout dst_layout); // Given the source shape, infer the destination shape. TVM_DLL ffi::Array ForwardShape(const ffi::Array& shape) const; // Given the destination shape, recover the source shape. TVM_DLL ffi::Array BackwardShape(const ffi::Array& dst_shape) const; // Given the destination indices, infer the destination indices. TVM_DLL ffi::Array ForwardIndex(const ffi::Array& index) const; // Given the destination indices, recover the source indices. TVM_DLL ffi::Array BackwardIndex(const ffi::Array& dst_index) const; TVM_FFI_DEFINE_OBJECT_REF_METHODS_NULLABLE(SBijectiveLayout, ffi::ObjectRef, SBijectiveLayoutNode); }; } // namespace tirx } // namespace tvm #endif // TVM_S_TIR_DATA_LAYOUT_H_