chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,21 @@
|
||||
The MIT License
|
||||
|
||||
Copyright (c) Carl Rogers, 2011
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
@@ -0,0 +1,36 @@
|
||||
Purpose:
|
||||
|
||||
Numpy offers the save method for easy saving of arrays into .npy and savez for zipping multiple .npy arrays together into a .npz file. cnpy lets you read and write to these formats in C++. The motivation comes from scientific programming where large amounts of data are generated in C++ and analyzed in Python. Writing to .npy has the advantage of using low-level C++ I/O (fread and fwrite) for speed and binary format for size. The .npy file header takes care of specifying the size, shape, and data type of the array, so specifying the format of the data is unnecessary. Loading data written in numpy formats into C++ is equally simple, but requires you to type-cast the loaded data to the type of your choice.
|
||||
|
||||
Installation:
|
||||
|
||||
Default installation directory is /usr/local. To specify a different directory, add -DCMAKE_INSTALL_PREFIX=/path/to/install/dir to the cmake invocation in step 4.
|
||||
|
||||
1. get cmake at www.cmake.org
|
||||
2. create a build directory, say $HOME/build
|
||||
3. cd $HOME/build
|
||||
4. cmake /path/to/cnpy
|
||||
5. make
|
||||
6. make install
|
||||
|
||||
Using:
|
||||
|
||||
To use, #include"cnpy.h" in your source code. Compile the source code mycode.cpp as
|
||||
/path/to/install/dir -lcnpy
|
||||
|
||||
Description:
|
||||
|
||||
There are two functions for writing data: npy_save, npz_save.
|
||||
|
||||
There are 3 functions for reading. npy_load will load a .npy file. npz_load(fname) will load a .npz and return a dictionary of NpyArray structures. npz_load(fname,varname) will load and return the NpyArray for data varname from the specified .npz file.
|
||||
Note that NpyArray allocates char* data using new[] and *will not* delete the data upon the NpyArray destruction. You are responsible for delete the data yourself.
|
||||
|
||||
The data structure for loaded data is below. Data is loaded into a raw byte array. The array shape and word size are read from the npy header. You are responsible for casting/copying the data to its intended data type.
|
||||
|
||||
struct NpyArray {
|
||||
char* data;
|
||||
std::vector<unsigned int> shape;
|
||||
unsigned int word_size;
|
||||
};
|
||||
|
||||
See example1.cpp for examples of how to use the library. example1 will also be build during cmake installation.
|
||||
@@ -0,0 +1,264 @@
|
||||
/*******************************************************************************
|
||||
* The MIT License
|
||||
*
|
||||
* Copyright (c) Carl Rogers, 2011
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
* of this software and associated documentation files (the "Software"), to deal
|
||||
* in the Software without restriction, including without limitation the rights
|
||||
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
|
||||
* The above copyright notice and this permission notice shall be included in
|
||||
* all copies or substantial portions of the Software.
|
||||
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
* THE SOFTWARE.
|
||||
******************************************************************************/
|
||||
|
||||
// Copyright (C) 2011 Carl Rogers
|
||||
// Released under MIT License
|
||||
// license available in LICENSE file, or at http://www.opensource.org/licenses/mit-license.php
|
||||
|
||||
#ifndef LIBCNPY_H_
|
||||
#define LIBCNPY_H_
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
#include <array/DataType.h>
|
||||
#include <assert.h>
|
||||
#include <system/op_boilerplate.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <complex>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
#include <streambuf>
|
||||
#include <string>
|
||||
#include <typeinfo>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
namespace cnpy {
|
||||
|
||||
/**
|
||||
* The numpy array
|
||||
*/
|
||||
struct NpyArray {
|
||||
char *data;
|
||||
std::vector<unsigned int> shape;
|
||||
unsigned int wordSize;
|
||||
bool fortranOrder;
|
||||
void destruct() { delete[] data; }
|
||||
};
|
||||
|
||||
struct SD_LIB_EXPORT npz_t : public std::map<std::string, NpyArray> {
|
||||
void destruct() {
|
||||
npz_t::iterator it = this->begin();
|
||||
for (; it != this->end(); ++it) (*it).second.destruct();
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
*
|
||||
* @param path
|
||||
* @return
|
||||
*/
|
||||
SD_LIB_EXPORT char *loadFile(const char *path);
|
||||
|
||||
/**
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
char BigEndianTest();
|
||||
|
||||
/**
|
||||
*
|
||||
* @param t
|
||||
* @return
|
||||
*/
|
||||
#ifdef __cpp_rtti
|
||||
SD_LIB_EXPORT char mapType(const std::type_info &t);
|
||||
#endif
|
||||
|
||||
template <typename T>
|
||||
SD_LIB_EXPORT char mapType();
|
||||
|
||||
/**
|
||||
*
|
||||
* @param T the type of the ndarray
|
||||
* @param data the data for the ndarray
|
||||
* @param shape the shape of the ndarray
|
||||
* @param ndims the rank of the ndarray
|
||||
* @return
|
||||
*/
|
||||
template <typename T>
|
||||
SD_LIB_EXPORT std::vector<char> createNpyHeader(const unsigned int *shape, const unsigned int ndims,
|
||||
unsigned int wordSize);
|
||||
/**
|
||||
* Parse the numpy header from
|
||||
* the given file
|
||||
* based on the pointers passed in
|
||||
* @param fp the file to parse from
|
||||
* @param wordSize the size of
|
||||
* the individual elements
|
||||
* @param shape the shape of the array
|
||||
* @param ndims the number of dimensions in the array
|
||||
* @param fortranOrder whether the array is fortran ordered or not
|
||||
* @param dtype the datatype of the array
|
||||
*/
|
||||
SD_LIB_EXPORT void parseNpyHeader(FILE *fp, unsigned int &wordSize, unsigned int *&shape, unsigned int &ndims,
|
||||
bool &fortranOrder);
|
||||
|
||||
/**
|
||||
* Parse the numpy header from
|
||||
* the given file
|
||||
* based on the pointers passed in
|
||||
* @param header the file to parse from
|
||||
* @param word_size the size of
|
||||
* the individual elements
|
||||
* @param shape the shape of the array
|
||||
* @param ndims the number of dimensions of the ndarray
|
||||
* @param fortran_order whether the array is fortran ordered or not
|
||||
*/
|
||||
SD_LIB_EXPORT void parseNpyHeaderPointer(const char *header, unsigned int &word_size, unsigned int *&shape,
|
||||
unsigned int &ndims, bool &fortran_order);
|
||||
/**
|
||||
*
|
||||
* @param fp
|
||||
* @param nrecs
|
||||
* @param global_header_size
|
||||
* @param global_header_offset
|
||||
*/
|
||||
SD_LIB_EXPORT void parseZipFooter(FILE *fp, unsigned short &nrecs, unsigned int &global_header_size,
|
||||
unsigned int &global_header_offset);
|
||||
|
||||
/**
|
||||
*
|
||||
* @param fname
|
||||
* @param varname
|
||||
* @return
|
||||
*/
|
||||
SD_LIB_EXPORT NpyArray npzLoad(std::string fname, std::string varname);
|
||||
|
||||
/**
|
||||
*
|
||||
* @param fname
|
||||
* @return
|
||||
*/
|
||||
SD_LIB_EXPORT NpyArray npyLoad(std::string fname);
|
||||
|
||||
/**
|
||||
* Parse the numpy header from
|
||||
* the given file
|
||||
* based on the pointers passed in
|
||||
* @param fp the file to parse from
|
||||
* @param wordSize the size of
|
||||
* the individual elements
|
||||
* @param shape the shape of the array
|
||||
* @param ndims the number of dimensions of the shape
|
||||
* @param fortranOrder whether the array is fortran ordered or c ordered
|
||||
*/
|
||||
SD_LIB_EXPORT void parseNpyHeaderStr(std::string header, unsigned int &wordSize, unsigned int *&shape,
|
||||
unsigned int &ndims, bool &fortranOrder);
|
||||
|
||||
/**
|
||||
*
|
||||
* @param fp
|
||||
* @return
|
||||
*/
|
||||
SD_LIB_EXPORT int *shapeFromFile(FILE *fp);
|
||||
|
||||
/**
|
||||
*
|
||||
* @param data
|
||||
* @return
|
||||
*/
|
||||
SD_LIB_EXPORT int *shapeFromPointer(char *data);
|
||||
|
||||
/**
|
||||
* Load the numpy array from the given file.
|
||||
* @param fp the file to load
|
||||
* @return the loaded array
|
||||
*/
|
||||
SD_LIB_EXPORT NpyArray loadNpyFromFile(FILE *fp);
|
||||
|
||||
/**
|
||||
* Load the numpy array archive from the given file.
|
||||
* @param fp the file to load
|
||||
* @return the loaded archive
|
||||
*/
|
||||
SD_LIB_EXPORT npz_t npzLoad(FILE *fp);
|
||||
/**
|
||||
*
|
||||
* @param data
|
||||
* @return
|
||||
*/
|
||||
SD_LIB_EXPORT NpyArray loadNpyFromPointer(char *data);
|
||||
|
||||
/**
|
||||
*
|
||||
* @param data
|
||||
* @return
|
||||
*/
|
||||
SD_LIB_EXPORT NpyArray loadNpyFromHeader(char *data);
|
||||
|
||||
SD_LIB_EXPORT npz_t npzLoad(std::string fname);
|
||||
|
||||
SD_LIB_EXPORT sd::DataType dataTypeFromHeader(char *data);
|
||||
/**
|
||||
* Parse the numpy header from
|
||||
* the given file
|
||||
* based on the pointers passed in
|
||||
* @param fp the file to parse from
|
||||
* @param word_size the size of
|
||||
* the individual elements
|
||||
* @param shape
|
||||
* @param ndims
|
||||
* @param fortran_order
|
||||
*/
|
||||
SD_LIB_EXPORT void parseNpyHeader(std::string header, unsigned int &word_size, unsigned int *&shape,
|
||||
unsigned int &ndims, bool &fortran_order);
|
||||
|
||||
/**
|
||||
*
|
||||
* @tparam T
|
||||
* @param i
|
||||
* @param pad
|
||||
* @param padval
|
||||
* @return
|
||||
*/
|
||||
template <typename T>
|
||||
SD_INLINE std::string tostring(T i, int pad = 0, char padval = ' ') {
|
||||
std::stringstream s;
|
||||
s << i;
|
||||
return s.str();
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
SD_LIB_EXPORT void npy_save(std::string fname, const void *data, const unsigned int *shape, const unsigned int ndims,
|
||||
std::string mode = "w");
|
||||
|
||||
} // namespace cnpy
|
||||
|
||||
/**
|
||||
*
|
||||
* @tparam T
|
||||
* @param lhs
|
||||
* @param rhs
|
||||
* @return
|
||||
*/
|
||||
template <typename T>
|
||||
SD_LIB_EXPORT std::vector<char> &operator+=(std::vector<char> &lhs, const T rhs);
|
||||
|
||||
#endif
|
||||
Reference in New Issue
Block a user