Alexandria  2.25.0
SDC-CH common library for the Euclid project
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
NpyMmap.icpp
Go to the documentation of this file.
1 /*
2  * Copyright (C) 2012-2021 Euclid Science Ground Segment
3  *
4  * This library is free software; you can redistribute it and/or modify it under
5  * the terms of the GNU Lesser General Public License as published by the Free
6  * Software Foundation; either version 3.0 of the License, or (at your option)
7  * any later version.
8  *
9  * This library is distributed in the hope that it will be useful, but WITHOUT
10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
11  * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
12  * details.
13  *
14  * You should have received a copy of the GNU Lesser General Public License
15  * along with this library; if not, write to the Free Software Foundation, Inc.,
16  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17  */
18 
19 #ifdef NPYMMAP_IMPL
20 
21 #include "NpyCommon.h"
22 #include <boost/filesystem/path.hpp>
23 #include <boost/iostreams/stream.hpp>
24 #include <numeric>
25 
26 namespace Euclid {
27 namespace NdArray {
28 
29 typedef boost::iostreams::stream<boost::iostreams::mapped_file_source> MappedStream;
30 
31 template <typename T>
32 NdArray<T> mmapNpy(const boost::filesystem::path& path, boost::iostreams::mapped_file_base::mapmode mode,
33  size_t max_size) {
34  std::string dtype;
35  size_t n_elements = 0;
36  std::vector<size_t> shape;
38 
39  boost::iostreams::mapped_file_params map_params;
40  map_params.path = path.native();
41  map_params.flags = mode;
42  if (max_size)
43  map_params.length = max_size;
44  else
45  max_size = boost::filesystem::file_size(path);
46 
47  boost::iostreams::mapped_file input(map_params);
48  MappedStream stream(input.operator boost::iostreams::mapped_file_source&());
49  stream.set_auto_close(false);
50  readNpyHeader(stream, dtype, shape, attrs, n_elements);
51 
52  if (dtype != NpyDtype<T>::str)
53  throw Elements::Exception() << "Can not cast " << dtype << " into " << typeid(T).name();
54 
55  if (!attrs.empty()) {
56  n_elements *= attrs.size();
57  }
58 
59  return {shape, attrs,
60  std::move(MappedContainer<T>(path, stream.tellg(), n_elements, attrs, std::move(input), max_size))};
61 }
62 
63 template <typename T>
64 NdArray<T> createMmapNpy(const boost::filesystem::path& path, const std::vector<size_t>& shape,
65  const std::vector<std::string>& attrs, size_t max_size) {
66  // Pre-generate header
67  std::stringstream header;
68  writeNpyHeader<T>(header, appendAttrShape(shape, attrs.size()), attrs);
69  auto header_str = header.str();
70  auto header_size = header_str.size();
71 
72  assert(header_size % 64 == 0);
73 
74  // Compute file expected size
75  size_t n_elements = std::accumulate(shape.begin(), shape.end(), 1, std::multiplies<size_t>());
76  if (!attrs.empty())
77  n_elements *= attrs.size();
78  size_t data_size = n_elements * sizeof(T);
79  size_t total_size = header_size + data_size;
80 
81  boost::iostreams::mapped_file_params map_params;
82  map_params.path = path.native();
83  map_params.flags = boost::iostreams::mapped_file_base::readwrite;
84  map_params.new_file_size = total_size;
85  if (max_size >= total_size)
86  map_params.length = max_size;
87  else
88  max_size = total_size;
89 
90  boost::iostreams::mapped_file output(map_params);
91  std::copy(header_str.begin(), header_str.end(), output.begin());
92  return {shape, attrs,
93  std::move(MappedContainer<T>(path, header_size, n_elements, attrs, std::move(output), max_size))};
94 }
95 
96 } // end of namespace NdArray
97 } // end of namespace Euclid
98 
99 #endif // NPYMMAP_IMPL
void readNpyHeader(std::istream &input, std::string &dtype, std::vector< size_t > &shape, std::vector< std::string > &attrs, size_t &n_elements)
Definition: NpyCommon.cpp:81
T empty(T...args)
T copy(T...args)
T end(T...args)
STL class.
NdArray< T > mmapNpy(const boost::filesystem::path &path, boost::iostreams::mapped_file_base::mapmode mode=boost::iostreams::mapped_file_base::readwrite, size_t max_size=0)
NdArray< T > createMmapNpy(const boost::filesystem::path &path, const std::vector< size_t > &shape, const std::vector< std::string > &attr_names, size_t max_size=0)
T move(T...args)
T size(T...args)
T begin(T...args)
T accumulate(T...args)
Path::Item path