bes Updated for version 3.20.10
DmrppCommon.cc
1// -*- mode: c++; c-basic-offset:4 -*-
2
3// This file is part of the BES
4
5// Copyright (c) 2016 OPeNDAP, Inc.
6// Author: James Gallagher <jgallagher@opendap.org>
7//
8// This library is free software; you can redistribute it and/or
9// modify it under the terms of the GNU Lesser General Public
10// License as published by the Free Software Foundation; either
11// version 2.1 of the License, or (at your option) any later version.
12//
13// This library is distributed in the hope that it will be useful,
14// but WITHOUT ANY WARRANTY; without even the implied warranty of
15// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16// Lesser General Public License for more details.
17//
18// You should have received a copy of the GNU Lesser General Public
19// License along with this library; if not, write to the Free Software
20// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21//
22// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
23
24#include "config.h"
25
26#include <string>
27#include <sstream>
28#include <vector>
29#include <iterator>
30#include <cstdio>
31#include <cstdlib>
32#include <cstring>
33
34#include <curl/curl.h>
35
36#include <libdap/BaseType.h>
37#include <libdap/D4Attributes.h>
38#include <libdap/XMLWriter.h>
39#include <libdap/util.h>
40
41#if 1
42#define PUGIXML_NO_XPATH
43#define PUGIXML_HEADER_ONLY
44#include <pugixml.hpp>
45#endif
46
47#include "url_impl.h"
48#include "BESIndent.h"
49#include "BESDebug.h"
50#include "BESUtil.h"
51#include "BESInternalError.h"
52
53#include "DmrppRequestHandler.h"
54#include "DmrppCommon.h"
55#include "Chunk.h"
56#include "byteswap_compat.h"
57
58
59using namespace std;
60using namespace libdap;
61
62#define prolog std::string("DmrppCommon::").append(__func__).append("() - ")
63
64namespace dmrpp {
65
66// Used with BESDEBUG
67static const string dmrpp_3 = "dmrpp:3";
68static const string dmrpp_4 = "dmrpp:4";
69
71string DmrppCommon::d_dmrpp_ns = "http://xml.opendap.org/dap/dmrpp/1.0.0#";
72string DmrppCommon::d_ns_prefix = "dmrpp";
73
85void join_threads(pthread_t threads[], unsigned int num_threads)
86{
87 int status;
88 for (unsigned int i = 0; i < num_threads; ++i) {
89 if (threads[i]) {
90 BESDEBUG(dmrpp_3, "Join thread " << i << " after an exception was caught." << endl);
91 string *error = NULL;
92 if ((status = pthread_join(threads[i], (void **) &error)) < 0) {
93 BESDEBUG(dmrpp_3, "Could not join thread " << i << ", " << strerror(status)<< endl);
94 // ERROR_LOG("Failed to join thread " << i << " during clean up from an exception: " << strerror(status) << endl);
95 }
96 else if (error != NULL) {
97 BESDEBUG(dmrpp_3, "Joined thread " << i << ", error exit: " << *error << endl);
98 // ERROR_LOG("Joined thread " << i << ", error exit" << *error << endl);
99 }
100 else {
101 BESDEBUG(dmrpp_3, "Joined thread " << i << ", successful exit." << endl);
102 }
103 }
104 }
105}
106
108void DmrppCommon::set_filter(const string &value) {
109 if (DmrppRequestHandler::d_emulate_original_filter_order_behavior) {
110 d_filters = "";
111 if (value.find("shuffle") != string::npos)
112 d_filters.append(" shuffle");
113 if (value.find("deflate") != string::npos)
114 d_filters.append(" deflate");
115 if (value.find("fletcher32") != string::npos)
116 d_filters.append(" fletcher32");
117
119 }
120 else {
121 d_filters = value;
122 }
123}
124
134void DmrppCommon::parse_chunk_dimension_sizes(const string &chunk_dims_string)
135{
136 d_chunk_dimension_sizes.clear();
137
138 if (chunk_dims_string.empty()) return;
139
140 string chunk_dims = chunk_dims_string;
141 // If the input is anything other than integers and spaces, throw
142 if (chunk_dims.find_first_not_of("1234567890 ") != string::npos)
143 throw BESInternalError("while processing chunk dimension information, illegal character(s)", __FILE__, __LINE__);
144
145 string space(" ");
146 size_t strPos = 0;
147 string strVal;
148
149 // Are there spaces or multiple values?
150 if (chunk_dims.find(space) != string::npos) {
151 // Process space delimited content
152 while ((strPos = chunk_dims.find(space)) != string::npos) {
153 strVal = chunk_dims.substr(0, strPos);
154
155 d_chunk_dimension_sizes.push_back(strtol(strVal.c_str(), nullptr, 10));
156 chunk_dims.erase(0, strPos + space.length());
157 }
158 }
159
160 // If it's multivalued there's still one more value left to process
161 // If it's single valued the same is true, so let's ingest that.
162 d_chunk_dimension_sizes.push_back(strtol(chunk_dims.c_str(), nullptr, 10));
163}
164
171void DmrppCommon::ingest_compression_type(const string &compression_type_string)
172{
173 if (compression_type_string.empty()) return;
174 set_filter(compression_type_string);
175}
176
182void DmrppCommon::ingest_byte_order(const string &byte_order_string) {
183
184 if (byte_order_string.empty()) return;
185
186 // Process content
187 if (byte_order_string.compare("LE") == 0) {
188 d_byte_order = "LE";
189 d_twiddle_bytes = is_host_big_endian();
190 } else {
191 if (byte_order_string.compare("BE") == 0) {
192 d_byte_order = "BE";
193 d_twiddle_bytes = !(is_host_big_endian());
194 } else {
195 throw BESInternalError("Did not recognize byteOrder.", __FILE__, __LINE__);
196 }
197 }
198}
199
205 std::shared_ptr<http::url> data_url,
206 const string &byte_order,
207 unsigned long long size,
208 unsigned long long offset,
209 const string &position_in_array)
210{
211 vector<unsigned long long> cpia_vector;
212 Chunk::parse_chunk_position_in_array_string(position_in_array, cpia_vector);
213 return add_chunk(move(data_url), byte_order, size, offset, cpia_vector);
214}
215
216unsigned long DmrppCommon::add_chunk(
217 std::shared_ptr<http::url> data_url,
218 const string &byte_order,
219 unsigned long long size,
220 unsigned long long offset,
221 const vector<unsigned long long> &position_in_array)
222{
223 std::shared_ptr<Chunk> chunk(new Chunk(move(data_url), byte_order, size, offset, position_in_array));
224#if 0
225 auto array = dynamic_cast<dmrpp::DmrppArray *>(this);
226 if(!array){
227 stringstream msg;
228 msg << prolog << "ERROR DmrrpCommon::add_chunk() may only be called on an instance of DmrppArray. ";
229 msg << "The variable";
230 auto bt = dynamic_cast<libdap::BaseType *>(this);
231 if(bt){
232 msg << " " << bt->type_name() << " " << bt->name();
233 }
234 msg << " is not an instance of DmrppArray.";
235 msg << "this: " << (void **) this << " ";
236 msg << "byte_order: " << byte_order << " ";
237 msg << "size: " << size << " ";
238 msg << "offset: " << offset << " ";
239 throw BESInternalError(msg.str(),__FILE__, __LINE__);
240 }
241
242 if(d_super_chunks.empty())
243 d_super_chunks.push_back( shared_ptr<SuperChunk>(new SuperChunk()));
244
245 auto currentSuperChunk = d_super_chunks.back();
246
247 bool chunk_was_added = currentSuperChunk->add_chunk(chunk);
248 if(!chunk_was_added){
249 if(currentSuperChunk->empty()){
250 stringstream msg;
251 msg << prolog << "ERROR! Failed to add a Chunk to an empty SuperChunk. This should not happen.";
252 throw BESInternalError(msg.str(),__FILE__,__LINE__);
253 }
254 // This means that the chunk was not contiguous with the currentSuperChunk
255 currentSuperChunk = shared_ptr<SuperChunk>(new SuperChunk());
256 chunk_was_added = currentSuperChunk->add_chunk(chunk);
257 if(!chunk_was_added) {
258 stringstream msg;
259 msg << prolog << "ERROR! Failed to add a Chunk to an empty SuperChunk. This should not happen.";
260 throw BESInternalError(msg.str(),__FILE__,__LINE__);
261 }
262 d_super_chunks.push_back(currentSuperChunk);
263 }
264#endif
265
266 d_chunks.push_back(chunk);
267 return d_chunks.size();
268}
269
270unsigned long DmrppCommon::add_chunk(
271 const string &byte_order,
272 unsigned long long size,
273 unsigned long long offset,
274 const string &position_in_array)
275{
276 vector<unsigned long long> cpia_vector;
277 Chunk::parse_chunk_position_in_array_string(position_in_array, cpia_vector);
278 return add_chunk(byte_order, size, offset, cpia_vector);
279}
280
281unsigned long DmrppCommon::add_chunk(
282 const string &byte_order,
283 unsigned long long size,
284 unsigned long long offset,
285 const vector<unsigned long long> &position_in_array)
286{
287 std::shared_ptr<Chunk> chunk(new Chunk( byte_order, size, offset, position_in_array));
288
289 d_chunks.push_back(chunk);
290 return d_chunks.size();
291}
292
310char *
311DmrppCommon::read_atomic(const string &name)
312{
313 if (get_chunks_size() != 1)
314 throw BESInternalError(string("Expected only a single chunk for variable ") + name, __FILE__, __LINE__);
315
316 auto chunk = get_immutable_chunks()[0];
317
318 chunk->read_chunk();
319
320 return chunk->get_rbuf();
321}
322
329void
330DmrppCommon::print_chunks_element(XMLWriter &xml, const string &name_space)
331{
332 // Start element "chunks" with dmrpp namespace and attributes:
333 if (xmlTextWriterStartElementNS(xml.get_writer(), (const xmlChar*)name_space.c_str(), (const xmlChar*) "chunks", NULL) < 0)
334 throw BESInternalError("Could not start chunks element.", __FILE__, __LINE__);
335
336 if (!d_filters.empty())
337 if (xmlTextWriterWriteAttribute(xml.get_writer(), (const xmlChar*) "compressionType", (const xmlChar*) d_filters.c_str()) < 0)
338 throw BESInternalError("Could not write compression attribute.", __FILE__, __LINE__);
339
340
341 if(get_chunks_size() != 0) { // FIXME !get_chunks().empty()){
342 auto first_chunk = get_immutable_chunks().front();
343 if (!first_chunk->get_byte_order().empty()) {
344 if (xmlTextWriterWriteAttribute(xml.get_writer(), (const xmlChar *) "byteOrder",
345 (const xmlChar *) first_chunk->get_byte_order().c_str()) < 0)
346 throw BESInternalError("Could not write attribute byteOrder", __FILE__, __LINE__);
347 }
348 }
349
350 if (d_chunk_dimension_sizes.size() > 0) {
351 // Write element "chunkDimensionSizes" with dmrpp namespace:
352 ostringstream oss;
353 copy(d_chunk_dimension_sizes.begin(), d_chunk_dimension_sizes.end(), ostream_iterator<unsigned int>(oss, " "));
354 string sizes = oss.str();
355 sizes.erase(sizes.size() - 1, 1); // trim the trailing space
356
357 if (xmlTextWriterWriteElementNS(xml.get_writer(), (const xmlChar*) name_space.c_str(), (const xmlChar*) "chunkDimensionSizes", NULL,
358 (const xmlChar*) sizes.c_str()) < 0) throw BESInternalError("Could not write chunkDimensionSizes attribute.", __FILE__, __LINE__);
359 }
360
361 // Start elements "chunk" with dmrpp namespace and attributes:
362 // for (vector<Chunk>::iterator i = get_chunks().begin(), e = get_chunks().end(); i != e; ++i) {
363
364 for(auto chunk: get_immutable_chunks()){
365
366 if (xmlTextWriterStartElementNS(xml.get_writer(), (const xmlChar*)name_space.c_str(), (const xmlChar*) "chunk", NULL) < 0)
367 throw BESInternalError("Could not start element chunk", __FILE__, __LINE__);
368
369 // Get offset string:
370 ostringstream offset;
371 offset << chunk->get_offset();
372 if (xmlTextWriterWriteAttribute(xml.get_writer(), (const xmlChar*) "offset", (const xmlChar*) offset.str().c_str()) < 0)
373 throw BESInternalError("Could not write attribute offset", __FILE__, __LINE__);
374
375 // Get nBytes string:
376 ostringstream nBytes;
377 nBytes << chunk->get_size();
378 if (xmlTextWriterWriteAttribute(xml.get_writer(), (const xmlChar*) "nBytes", (const xmlChar*) nBytes.str().c_str()) < 0)
379 throw BESInternalError("Could not write attribute nBytes", __FILE__, __LINE__);
380
381 if (chunk->get_position_in_array().size() > 0) {
382 // Get position in array string:
383 vector<unsigned long long> pia = chunk->get_position_in_array();
384 ostringstream oss;
385 oss << "[";
386 copy(pia.begin(), pia.end(), ostream_iterator<unsigned int>(oss, ","));
387 string pia_str = oss.str();
388 if (pia.size() > 0) pia_str.replace(pia_str.size() - 1, 1, "]"); // replace the trailing ',' with ']'
389 if (xmlTextWriterWriteAttribute(xml.get_writer(), (const xmlChar*) "chunkPositionInArray", (const xmlChar*) pia_str.c_str()) < 0)
390 throw BESInternalError("Could not write attribute position in array", __FILE__, __LINE__);
391 }
392
393 // End element "chunk":
394 if (xmlTextWriterEndElement(xml.get_writer()) < 0) throw BESInternalError("Could not end chunk element", __FILE__, __LINE__);
395 }
396
397 if (xmlTextWriterEndElement(xml.get_writer()) < 0) throw BESInternalError("Could not end chunks element", __FILE__, __LINE__);
398}
399
403void
404DmrppCommon::print_compact_element(XMLWriter &xml, const string &name_space, const std::string &encoded)
405{
406 // Write element "compact" with dmrpp namespace:
407 ostringstream oss;
408 copy(encoded.begin(), encoded.end(), ostream_iterator<char>(oss, ""));
409 string sizes = oss.str();
410
411 if (xmlTextWriterWriteElementNS(xml.get_writer(), (const xmlChar *) name_space.c_str(),
412 (const xmlChar *) "compact", NULL,
413 (const xmlChar *) sizes.c_str()) < 0)
414 throw BESInternalError("Could not write compact element.", __FILE__, __LINE__);
415}
416
427void DmrppCommon::print_dmrpp(XMLWriter &xml, bool constrained /*false*/)
428{
429 BaseType &bt = dynamic_cast<BaseType&>(*this);
430 if (constrained && !bt.send_p())
431 return;
432
433 if (xmlTextWriterStartElement(xml.get_writer(), (const xmlChar*)bt.type_name().c_str()) < 0)
434 throw InternalErr(__FILE__, __LINE__, "Could not write " + bt.type_name() + " element");
435
436 if (!bt.name().empty())
437 if (xmlTextWriterWriteAttribute(xml.get_writer(), (const xmlChar*) "name", (const xmlChar*)bt.name().c_str()) < 0)
438 throw InternalErr(__FILE__, __LINE__, "Could not write attribute for name");
439
440 if (bt.is_dap4())
441 bt.attributes()->print_dap4(xml);
442
443 if (!bt.is_dap4() && bt.get_attr_table().get_size() > 0)
444 bt.get_attr_table().print_xml_writer(xml);
445
446 // This is the code added to libdap::BaseType::print_dap4(). jhrg 5/10/18
449
450 if (xmlTextWriterEndElement(xml.get_writer()) < 0)
451 throw InternalErr(__FILE__, __LINE__, "Could not end " + bt.type_name() + " element");
452}
453
454void DmrppCommon::dump(ostream & strm) const
455{
456 strm << BESIndent::LMarg << "is_filters_empty: " << (is_filters_empty() ? "true" : "false") << endl;
457 strm << BESIndent::LMarg << "filters: " << (d_filters.c_str()) << endl;
458
459 const vector<unsigned long long> &chunk_dim_sizes = get_chunk_dimension_sizes();
460
461 strm << BESIndent::LMarg << "chunk dimension sizes: [";
462 for (unsigned int i = 0; i < chunk_dim_sizes.size(); i++) {
463 strm << (i ? "][" : "") << chunk_dim_sizes[i];
464 }
465 strm << "]" << endl;
466
467 auto chunk_refs = get_immutable_chunks();
468 strm << BESIndent::LMarg << "Chunks (aka chunks):" << (chunk_refs.size() ? "" : "None Found.") << endl;
469 BESIndent::Indent();
470 for (auto & chunk_ref : chunk_refs) {
471 strm << BESIndent::LMarg;
472 chunk_ref->dump(strm);
473 strm << endl;
474 }
475}
476
481void
483 d_dmz->load_chunks(btp);
484}
485
490void
491DmrppCommon::load_attributes(libdap::BaseType *btp)
492{
493 d_dmz->load_attributes(btp);
494}
495
496} // namespace dmrpp
497
exception thrown if internal error encountered
static void removeLeadingAndTrailingBlanks(std::string &key)
Definition: BESUtil.cc:485
Extend libdap::Array so that a handler can read data using a DMR++ file.
Definition: DmrppArray.h:68
static std::string d_ns_prefix
The XML namespace prefix to use.
Definition: DmrppCommon.h:120
static bool d_print_chunks
if true, print_dap4() prints chunk elements
Definition: DmrppCommon.h:118
virtual void ingest_compression_type(const std::string &compression_type_string)
Parses the text content of the XML element h4:chunkDimensionSizes into the internal vector<unsigned i...
Definition: DmrppCommon.cc:171
virtual void load_attributes(libdap::BaseType *btp)
Load the attribute information for this variable.
Definition: DmrppCommon.cc:491
void print_compact_element(libdap::XMLWriter &xml, const std::string &name_space="", const std::string &encoded="")
Print the Compact base64-encoded information.
Definition: DmrppCommon.cc:404
virtual size_t get_chunks_size() const
Use this when the number of chunks is needed.
Definition: DmrppCommon.h:175
static std::string d_dmrpp_ns
The DMR++ XML namespace.
Definition: DmrppCommon.h:119
void print_chunks_element(libdap::XMLWriter &xml, const std::string &name_space="")
Print the Chunk information.
Definition: DmrppCommon.cc:330
virtual void parse_chunk_dimension_sizes(const std::string &chunk_dim_sizes_string)
Set the dimension sizes for a chunk.
Definition: DmrppCommon.cc:134
virtual const std::vector< std::shared_ptr< Chunk > > & get_immutable_chunks() const
A const reference to the vector of chunks.
Definition: DmrppCommon.h:169
virtual unsigned long add_chunk(std::shared_ptr< http::url > d_data_url, const std::string &byte_order, unsigned long long size, unsigned long long offset, const std::string &position_in_array)
Add a new chunk as defined by an h4:byteStream element.
Definition: DmrppCommon.cc:204
void set_filter(const std::string &value)
Set the value of the filters property.
Definition: DmrppCommon.cc:108
virtual void ingest_byte_order(const std::string &byte_order_string)
Parses the text content of the XML element chunks:byteOrder.
Definition: DmrppCommon.cc:182
virtual const std::vector< unsigned long long > & get_chunk_dimension_sizes() const
The chunk dimension sizes held in a const vector.
Definition: DmrppCommon.h:179
void print_dmrpp(libdap::XMLWriter &writer, bool constrained=false)
Print the DMR++ response for the Scalar types.
Definition: DmrppCommon.cc:427
virtual void load_chunks(libdap::BaseType *btp)
Load chunk information for this variable.
Definition: DmrppCommon.cc:482
virtual char * read_atomic(const std::string &name)
read method for the atomic types
Definition: DmrppCommon.cc:311