22 #include <boost/filesystem/path.hpp>
23 #include <boost/iostreams/stream.hpp>
29 typedef boost::iostreams::stream<boost::iostreams::mapped_file> MappedStream;
32 NdArray<T>
mmapNpy(
const boost::filesystem::path&
path, boost::iostreams::mapped_file_base::mapmode mode,
size_t max_size) {
34 size_t n_elements = 0;
38 boost::iostreams::mapped_file_params map_params;
39 map_params.path = path.native();
40 map_params.flags = mode;
42 map_params.length = max_size;
44 max_size = boost::filesystem::file_size(path);
46 boost::iostreams::mapped_file input(map_params);
47 MappedStream stream(input);
48 stream.set_auto_close(
false);
51 if (dtype != NpyDtype<T>::str)
55 n_elements *= attrs.
size();
58 return {shape, attrs,
std::move(MappedContainer<T>(path, stream.tellg(), n_elements, attrs,
std::move(input), max_size))};
66 writeNpyHeader<T>(header, appendAttrShape(shape, attrs.
size()), attrs);
67 auto header_str = header.str();
68 auto header_size = header_str.size();
70 assert(header_size % 64 == 0);
75 n_elements *= attrs.
size();
76 size_t data_size = n_elements *
sizeof(T);
77 size_t total_size = header_size + data_size;
79 boost::iostreams::mapped_file_params map_params;
80 map_params.path = path.native();
81 map_params.flags = boost::iostreams::mapped_file_base::readwrite;
82 map_params.new_file_size = total_size;
83 if (max_size >= total_size)
84 map_params.
length = max_size;
86 max_size = total_size;
88 boost::iostreams::mapped_file output(map_params);
89 std::copy(header_str.begin(), header_str.end(), output.begin());
90 return {shape, attrs,
std::move(MappedContainer<T>(path, header_size, n_elements, attrs,
std::move(output), max_size))};
96 #endif // NPYMMAP_IMPL
void readNpyHeader(std::istream &input, std::string &dtype, std::vector< size_t > &shape, std::vector< std::string > &attrs, size_t &n_elements)
NdArray< T > mmapNpy(const boost::filesystem::path &path, boost::iostreams::mapped_file_base::mapmode mode=boost::iostreams::mapped_file_base::readwrite, size_t max_size=0)
NdArray< T > createMmapNpy(const boost::filesystem::path &path, const std::vector< size_t > &shape, const std::vector< std::string > &attr_names, size_t max_size=0)
Elements::Path::Item path