bes Updated for version 3.20.10
Chunk.h
1// -*- mode: c++; c-basic-offset:4 -*-
2
3// This file is part of the BES
4
5// Copyright (c) 2016 OPeNDAP, Inc.
6// Author: Nathan Potter <ndp@opendap.org>
7//
8// This library is free software; you can redistribute it and/or
9// modify it under the terms of the GNU Lesser General Public
10// License as published by the Free Software Foundation; either
11// version 2.1 of the License, or (at your option) any later version.
12//
13// This library is distributed in the hope that it will be useful,
14// but WITHOUT ANY WARRANTY; without even the implied warranty of
15// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16// Lesser General Public License for more details.
17//
18// You should have received a copy of the GNU Lesser General Public
19// License along with this library; if not, write to the Free Software
20// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21//
22// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
23
24#ifndef _Chunk_h
25#define _Chunk_h 1
26
27#include <string>
28#include <utility>
29#include <vector>
30#include <memory>
31
32// BES
33#include "url_impl.h"
34
35// libdap4
36#include <libdap/util.h>
37
38
39// This is used to track access to 'cloudydap' accesses in the S3 logs
40// by adding a query string that will show up in those logs. This is
41// activated by using a special BES context with the name 'cloudydap.'
42#define S3_TRACKING_CONTEXT "cloudydap"
43#define ENABLE_TRACKING_QUERY_PARAMETER 0
44
45namespace dmrpp {
46
47// Callback functions used by chunk readers
48size_t chunk_header_callback(char *buffer, size_t size, size_t nitems, void *data);
49size_t chunk_write_data(void *buffer, size_t size, size_t nmemb, void *data);
50
51void process_s3_error_response(const std::shared_ptr<http::url> &data_url, const std::string &xml_message);
52
59class Chunk {
60private:
61 std::shared_ptr<http::url> d_data_url;
62 std::string d_query_marker;
63 std::string d_byte_order;
64 unsigned long long d_size;
65 unsigned long long d_offset;
66
67 std::vector<unsigned long long> d_chunk_position_in_array;
68
69 // These are used only during the libcurl callback;
70 // they are not duplicated by the copy ctor or assignment
71 // operator.
72
89 bool d_read_buffer_is_mine;
90 unsigned long long d_bytes_read;
91 char *d_read_buffer;
92 unsigned long long d_read_buffer_size;
93 bool d_is_read;
94 bool d_is_inflated;
95 std::string d_response_content_type;
96
97 // static const std::string tracking_context;
98
99
100 friend class ChunkTest;
101 friend class DmrppCommonTest;
102 friend class MockChunk;
103
104protected:
105
106 void _duplicate(const Chunk &bs)
107 {
108 // See above
109 d_read_buffer_is_mine = true;
110 d_bytes_read = 0;
111 d_read_buffer = nullptr;
112 d_read_buffer_size = 0;
113 d_is_read = false;
114 d_is_inflated = false;
115
116 d_size = bs.d_size;
117 d_offset = bs.d_offset;
118 d_data_url = bs.d_data_url;
119 d_byte_order = bs.d_byte_order;
120 d_query_marker = bs.d_query_marker;
121 d_chunk_position_in_array = bs.d_chunk_position_in_array;
122 }
123
124public:
125
136 d_data_url(nullptr), d_size(0), d_offset(0),
137 d_read_buffer_is_mine(true), d_bytes_read(0), d_read_buffer(nullptr),
138 d_read_buffer_size(0), d_is_read(false), d_is_inflated(false)
139 {
140 }
141
153 std::shared_ptr<http::url> data_url,
154 std::string order,
155 unsigned long long size,
156 unsigned long long offset,
157 const std::string &pia_str = "") :
158 d_data_url(std::move(data_url)),
159 d_byte_order(std::move(order)),
160 d_size(size),
161 d_offset(offset),
162 d_read_buffer_is_mine(true),
163 d_bytes_read(0),
164 d_read_buffer(nullptr),
165 d_read_buffer_size(0),
166 d_is_read(false),
167 d_is_inflated(false)
168 {
169#if ENABLE_TRACKING_QUERY_PARAMETER
171#endif
172 set_position_in_array(pia_str);
173 }
174
186 std::string order,
187 unsigned long long size,
188 unsigned long long offset,
189 const std::string &pia_str = "") :
190 d_byte_order(std::move(order)),
191 d_size(size),
192 d_offset(offset),
193 d_read_buffer_is_mine(true),
194 d_bytes_read(0),
195 d_read_buffer(nullptr),
196 d_read_buffer_size(0),
197 d_is_read(false),
198 d_is_inflated(false)
199 {
200#if ENABLE_TRACKING_QUERY_PARAMETER
202#endif
203 set_position_in_array(pia_str);
204 }
205
217 std::shared_ptr<http::url> data_url,
218 std::string order,
219 unsigned long long size,
220 unsigned long long offset,
221 const std::vector<unsigned long long> &pia_vec) :
222 d_data_url(std::move(data_url)),
223 d_byte_order(std::move(order)),
224 d_size(size),
225 d_offset(offset),
226 d_read_buffer_is_mine(true),
227 d_bytes_read(0),
228 d_read_buffer(nullptr),
229 d_read_buffer_size(0),
230 d_is_read(false),
231 d_is_inflated(false)
232 {
233#if ENABLE_TRACKING_QUERY_PARAMETER
235#endif
236 set_position_in_array(pia_vec);
237 }
238
239
251 std::string order,
252 unsigned long long size,
253 unsigned long long offset,
254 const std::vector<unsigned long long> &pia_vec) :
255 d_query_marker(""),
256 d_byte_order(std::move(order)),
257 d_size(size), d_offset(offset),
258 d_read_buffer_is_mine(true),
259 d_bytes_read(0),
260 d_read_buffer(nullptr),
261 d_read_buffer_size(0),
262 d_is_read(false),
263 d_is_inflated(false)
264 {
265#if ENABLE_TRACKING_QUERY_PARAMETER
267#endif
268 set_position_in_array(pia_vec);
269 }
270
271
272 Chunk(const Chunk &h4bs)
273 {
274 _duplicate(h4bs);
275 }
276
277 virtual ~Chunk()
278 {
279 if(d_read_buffer_is_mine)
280 delete[] d_read_buffer;
281 d_read_buffer = nullptr;
282 }
283
288 Chunk &operator=(const Chunk &rhs)
289 {
290 if (this == &rhs) return *this;
291
292 _duplicate(rhs);
293
294 return *this;
295 }
296
298 virtual std::string get_response_content_type() { return d_response_content_type; }
299
301 void set_response_content_type(const std::string &ct) { d_response_content_type = ct; }
302
304 virtual std::string get_byte_order() { return d_byte_order; }
305
309 virtual unsigned long long get_size() const
310 {
311 return d_size;
312 }
313
317 virtual unsigned long long get_offset() const
318 {
319 return d_offset;
320 }
321
325 virtual std::shared_ptr<http::url> get_data_url() const;
326
330 virtual void set_data_url(std::shared_ptr<http::url> data_url)
331 {
332 d_data_url = std::move(data_url);
333 }
334
338 virtual unsigned long long get_bytes_read() const
339 {
340 return d_bytes_read;
341 }
342
347 virtual void set_bytes_read(unsigned long long bytes_read)
348 {
349 d_bytes_read = bytes_read;
350 }
351
366 virtual void set_rbuf_to_size()
367 {
368 if(d_read_buffer_is_mine)
369 delete[] d_read_buffer;
370
371 d_read_buffer = new char[d_size];
372 d_read_buffer_size = d_size;
373 d_read_buffer_is_mine = true;
375 }
376
381 virtual char *get_rbuf()
382 {
383 return d_read_buffer;
384 }
385
386#if 0 // Superseded by Chunk::set_read_buffer(); - ndp 12/17/20
400 virtual void set_rbuf(char *buf, unsigned int buf_size)
401 {
402 if(d_read_buffer_is_mine)
403 delete[] d_read_buffer;
404
405 d_read_buffer = buf;
406 d_read_buffer_size = buf_size;
407
408 set_bytes_read(buf_size);
409 }
410#endif
411
424 char *buf,
425 unsigned long long buf_size,
426 unsigned long long bytes_read = 0,
427 bool assume_ownership = true ){
428
429 if(d_read_buffer_is_mine)
430 delete[] d_read_buffer;
431
432 d_read_buffer_is_mine = assume_ownership;
433 d_read_buffer = buf;
434 d_read_buffer_size = buf_size;
435
436 set_bytes_read(bytes_read);
437 }
438
442 virtual unsigned long long get_rbuf_size() const
443 {
444 return d_read_buffer_size;
445 }
446
450 virtual const std::vector<unsigned long long> &get_position_in_array() const
451 {
452 return d_chunk_position_in_array;
453 }
454
455
457
458 void set_position_in_array(const std::string &pia);
459 void set_position_in_array(const std::vector<unsigned long long> &pia);
460
461 virtual void read_chunk();
462
463 virtual void filter_chunk(const std::string &filters, unsigned long long chunk_size, unsigned long long elem_width);
464
465 virtual bool get_is_read() { return d_is_read; }
466 virtual void set_is_read(bool state) { d_is_read = state; }
467
468 //virtual bool get_is_inflated() const { return d_is_inflated; }
469 //virtual void set_is_inflated(bool state) { d_is_inflated = state; }
470
471 virtual std::string get_curl_range_arg_string();
472
473 static void parse_chunk_position_in_array_string(const std::string &pia, std::vector<unsigned long long> &pia_vect);
474
475 virtual void dump(std::ostream & strm) const;
476
477 virtual std::string to_string() const;
478};
479
480#if 0
482struct inflate_chunk_args {
483 Chunk *chunk;
484 bool deflate;
485 bool shuffle;
486 unsigned int chunk_size;
487 unsigned int elem_width;
488
489 inflate_chunk_args(Chunk *c, bool d, bool s, unsigned int c_size, unsigned int e_size):
490 chunk(c), deflate(d), shuffle(s), chunk_size(c_size), elem_width(e_size) {}
491};
492
493void *inflate_chunk(void *args);
494#endif
495
496} // namespace dmrpp
497
498#endif // _Chunk_h
virtual void set_bytes_read(unsigned long long bytes_read)
Set the size of this Chunk's data block.
Definition: Chunk.h:347
virtual void dump(std::ostream &strm) const
Definition: Chunk.cc:880
Chunk(std::string order, unsigned long long size, unsigned long long offset, const std::string &pia_str="")
Get a chunk initialized with values, the data URL will not be set.
Definition: Chunk.h:185
virtual char * get_rbuf()
Definition: Chunk.h:381
virtual void read_chunk()
Definition: Chunk.cc:840
void add_tracking_query_param()
Modify this chunk's data URL so that it includes tracking info.
Definition: Chunk.cc:526
virtual std::string get_response_content_type()
Get the response type of the last response.
Definition: Chunk.h:298
virtual std::string get_byte_order()
Get the chunk byte order.
Definition: Chunk.h:304
virtual std::string get_curl_range_arg_string()
Returns a curl range argument. The libcurl requires a string argument for range-ge activitys,...
Definition: Chunk.cc:509
virtual std::shared_ptr< http::url > get_data_url() const
Get the data url for this Chunk's data block.
Definition: Chunk.cc:903
Chunk(std::shared_ptr< http::url > data_url, std::string order, unsigned long long size, unsigned long long offset, const std::string &pia_str="")
Get a chunk initialized with values.
Definition: Chunk.h:152
Chunk(std::string order, unsigned long long size, unsigned long long offset, const std::vector< unsigned long long > &pia_vec)
Get a chunk initialized with values, the data URl will not be set.
Definition: Chunk.h:250
virtual void set_data_url(std::shared_ptr< http::url > data_url)
Set the data url for this Chunk's data block.
Definition: Chunk.h:330
Chunk()
Get an empty chunk.
Definition: Chunk.h:135
Chunk & operator=(const Chunk &rhs)
Definition: Chunk.h:288
virtual void set_rbuf_to_size()
Allocates the internal read buffer to be d_size bytes.
Definition: Chunk.h:366
virtual unsigned long long get_offset() const
Get the offset to this Chunk's data block.
Definition: Chunk.h:317
virtual unsigned long long get_bytes_read() const
Get the number of bytes read so far for this Chunk.
Definition: Chunk.h:338
void set_position_in_array(const std::string &pia)
parse the chunk position string
Definition: Chunk.cc:458
virtual unsigned long long get_rbuf_size() const
Definition: Chunk.h:442
virtual unsigned long long get_size() const
Get the size of this Chunk's data block on disk.
Definition: Chunk.h:309
Chunk(std::shared_ptr< http::url > data_url, std::string order, unsigned long long size, unsigned long long offset, const std::vector< unsigned long long > &pia_vec)
Get a chunk initialized with values.
Definition: Chunk.h:216
virtual const std::vector< unsigned long long > & get_position_in_array() const
Definition: Chunk.h:450
void set_read_buffer(char *buf, unsigned long long buf_size, unsigned long long bytes_read=0, bool assume_ownership=true)
Set the target read buffer for this chunk.
Definition: Chunk.h:423
virtual void filter_chunk(const std::string &filters, unsigned long long chunk_size, unsigned long long elem_width)
filter data in the chunk
Definition: Chunk.cc:755
void set_response_content_type(const std::string &ct)
Set the response type of the last response.
Definition: Chunk.h:301