bes Updated for version 3.20.10
DmrppRequestHandler.cc
1// DmrppRequestHandler.cc
2
3// Copyright (c) 2016 OPeNDAP, Inc. Author: James Gallagher
4// <jgallagher@opendap.org>, Patrick West <pwest@opendap.org>
5// Nathan Potter <npotter@opendap.org>
6//
7// modify it under the terms of the GNU Lesser General Public License
8// as published by the Free Software Foundation; either version 2.1 of
9// the License, or (at your option) any later version.
10//
11// This library is distributed in the hope that it will be useful, but
12// WITHOUT ANY WARRANTY; without even the implied warranty of
13// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14// Lesser General Public License for more details.
15//
16// License along with this library; if not, write to the Free Software
17// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
18// 02110-1301 U\ SA
19//
20// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI.
21// 02874-0112.
22
23#include "config.h"
24
25#include <string>
26#include <memory>
27#include <sstream>
28
29#include <curl/curl.h>
30
31#include <libdap/Ancillary.h>
32#include <libdap/DMR.h>
33#include <libdap/D4Group.h>
34#include <libdap/DAS.h>
35
36#include <libdap/InternalErr.h>
37#include <libdap/mime_util.h> // for name_path
38
39#include <BESResponseHandler.h>
40#include <BESResponseNames.h>
41#include <BESDapNames.h>
42#include <BESDataNames.h>
43#include <BESDASResponse.h>
44#include <BESDDSResponse.h>
45#include <BESDataDDSResponse.h>
46#include <BESVersionInfo.h>
47#include <BESContainer.h>
48#include <ObjMemCache.h>
49
50#include <BESDMRResponse.h>
51
52#include <BESConstraintFuncs.h>
53#include <BESServiceRegistry.h>
54#include <BESUtil.h>
55#include <BESLog.h>
56#include <TheBESKeys.h>
57
58#include <BESDapError.h>
59#include <BESInternalFatalError.h>
60#include <BESDebug.h>
61#include <BESStopWatch.h>
62
63#if 1
64#define PUGIXML_NO_XPATH
65#define PUGIXML_HEADER_ONLY
66#include <pugixml.hpp>
67#endif
68
69#include "DmrppNames.h"
70//#include "DMRpp.h"
71#include "DmrppTypeFactory.h"
72//#include "DmrppParserSax2.h"
73#include "DmrppRequestHandler.h"
74#include "CurlHandlePool.h"
75//#include "DmrppMetadataStore.h"
76#include "CredentialsManager.h"
77
78using namespace bes;
79using namespace libdap;
80using namespace std;
81
82#define MODULE_NAME "dmrpp_module"
83#ifndef MODULE_VERSION
84#define MODULE_VERSION "unset" // Set this in the Makefile.am
85#endif
86
87#define prolog std::string("DmrppRequestHandler::").append(__func__).append("() - ")
88
89#define USE_DMZ_TO_MANAGE_XML 1
90
91namespace dmrpp {
92
93ObjMemCache *DmrppRequestHandler::das_cache = 0;
94ObjMemCache *DmrppRequestHandler::dds_cache = 0;
95ObjMemCache *DmrppRequestHandler::dmr_cache = 0;
96
97shared_ptr<DMZ> DmrppRequestHandler::dmz(nullptr);
98
99// This is used to maintain a pool of reusable curl handles that enable connection
100// reuse. jhrg
101CurlHandlePool *DmrppRequestHandler::curl_handle_pool = 0;
102
103bool DmrppRequestHandler::d_use_transfer_threads = true;
104unsigned int DmrppRequestHandler::d_max_transfer_threads = 8;
105
106bool DmrppRequestHandler::d_use_compute_threads = true;
107unsigned int DmrppRequestHandler::d_max_compute_threads = 8;
108
109// Default minimum value is 2MB: 2 * (1024*1024)
110unsigned long long DmrppRequestHandler::d_contiguous_concurrent_threshold = DMRPP_DEFAULT_CONTIGUOUS_CONCURRENT_THRESHOLD;
111
112// This behavior mirrors the SAX2 parser behavior where the software doesn't require that
113// a variable actually have chunks (that is, some variable might not have any data).
114// We could make this a run-time option if needed. jhrg 11/4/21
115bool DmrppRequestHandler::d_require_chunks = false;
116
117// See the comment in the header for more about this kludge. jhrg 11/9/21
118bool DmrppRequestHandler::d_emulate_original_filter_order_behavior = false;
119
120static void read_key_value(const std::string &key_name, bool &key_value)
121{
122 bool key_found = false;
123 string value;
124 TheBESKeys::TheKeys()->get_value(key_name, value, key_found);
125 if (key_found) {
126 value = BESUtil::lowercase(value);
127 key_value = (value == "true" || value == "yes");
128 }
129}
130
131static void read_key_value(const std::string &key_name, unsigned int &key_value)
132{
133 bool key_found = false;
134 string value;
135 TheBESKeys::TheKeys()->get_value(key_name, value, key_found);
136 if (key_found) {
137 istringstream iss(value);
138 iss >> key_value;
139 }
140}
141static void read_key_value(const std::string &key_name, unsigned long long &key_value)
142{
143 bool key_found = false;
144 string value;
145 TheBESKeys::TheKeys()->get_value(key_name, value, key_found);
146 if (key_found) {
147 istringstream iss(value);
148 iss >> key_value;
149 }
150}
151
158{
159 add_method(DMR_RESPONSE, dap_build_dmr);
160 add_method(DAP4DATA_RESPONSE, dap_build_dap4data);
161 add_method(DAS_RESPONSE, dap_build_das);
162 add_method(DDS_RESPONSE, dap_build_dds);
163 add_method(DATA_RESPONSE, dap_build_dap2data);
164
165 add_method(VERS_RESPONSE, dap_build_vers);
166 add_method(HELP_RESPONSE, dap_build_help);
167
168 stringstream msg;
169 read_key_value(DMRPP_USE_TRANSFER_THREADS_KEY, d_use_transfer_threads);
170 read_key_value(DMRPP_MAX_TRANSFER_THREADS_KEY, d_max_transfer_threads);
171 msg << prolog << "Concurrent Transfer Threads: ";
172 if(DmrppRequestHandler::d_use_transfer_threads){
173 msg << "Enabled. max_transfer_threads: " << DmrppRequestHandler::d_max_transfer_threads << endl;
174 }
175 else{
176 msg << "Disabled." << endl;
177 }
178
179 INFO_LOG(msg.str() );
180 msg.str(std::string());
181
182 read_key_value(DMRPP_USE_COMPUTE_THREADS_KEY, d_use_compute_threads);
183 read_key_value(DMRPP_MAX_COMPUTE_THREADS_KEY, d_max_compute_threads);
184 msg << prolog << "Concurrent Compute Threads: ";
185 if(DmrppRequestHandler::d_use_compute_threads){
186 msg << "Enabled. max_compute_threads: " << DmrppRequestHandler::d_max_compute_threads << endl;
187 }
188 else{
189 msg << "Disabled." << endl;
190 }
191
192 INFO_LOG(msg.str() );
193 msg.str(std::string());
194
195 // DMRPP_CONTIGUOUS_CONCURRENT_THRESHOLD_KEY
196 read_key_value(DMRPP_CONTIGUOUS_CONCURRENT_THRESHOLD_KEY, d_contiguous_concurrent_threshold);
197 msg << prolog << "Contiguous Concurrency Threshold: " << d_contiguous_concurrent_threshold << " bytes." << endl;
198 INFO_LOG(msg.str() );
199
200#if !HAVE_CURL_MULTI_API
201 if (DmrppRequestHandler::d_use_transfer_threads)
202 ERROR_LOG("The DMR++ handler is configured to use parallel transfers, but the libcurl Multi API is not present, defaulting to serial transfers");
203#endif
204
206
207 if (!curl_handle_pool)
208 curl_handle_pool = new CurlHandlePool(d_max_transfer_threads);
209
210 // This and the matching cleanup function can be called many times as long as
211 // they are called in balanced pairs. jhrg 9/3/20
212 // TODO 10/8/21 move this into the http at the top level of the BES. That is, all
213 // calls to this should be moved out of handlers and handlers can/should
214 // assume that curl is present and init'd. jhrg
215 curl_global_init(CURL_GLOBAL_DEFAULT);
216}
217
218DmrppRequestHandler::~DmrppRequestHandler()
219{
220 delete curl_handle_pool;
221 curl_global_cleanup();
222}
223
229void
230handle_exception(const string &file, int line)
231 try {
232 throw;
233 }
234 catch (BESError &e) {
235 throw e;
236 }
237 catch (InternalErr &e) {
238 throw BESDapError(e.get_error_message(), true, e.get_error_code(), file, line);
239 }
240 catch (Error &e) {
241 throw BESDapError(e.get_error_message(), false, e.get_error_code(), file, line);
242 }
243 catch (std::exception &e) {
244 BESInternalFatalError(string("C++ exception: ").append(e.what()), file, line);
245 }
246 catch (...) {
247 throw BESInternalFatalError("Unknown exception caught building DAP4 Data response", file, line);
248 }
249
250
251void DmrppRequestHandler::build_dmr_from_file(BESContainer *container, DMR* dmr)
252{
253 string data_pathname = container->access();
254
255 dmr->set_filename(data_pathname);
256 dmr->set_name(name_path(data_pathname));
257
258#if USE_DMZ_TO_MANAGE_XML
259 dmz = shared_ptr<DMZ>(new DMZ);
260
261 // Enable adding the DMZ to the BaseTypes built by the factory
262 DmrppTypeFactory BaseFactory(dmz);
263 dmr->set_factory(&BaseFactory);
264
265 dmz->parse_xml_doc(data_pathname);
266 dmz->build_thin_dmr(dmr);
267
268 dmz->load_all_attributes(dmr);
269#else
270 DmrppTypeFactory BaseFactory; // Use the factory for this handler's types
271 dmr->set_factory(&BaseFactory);
272
273 DmrppParserSax2 parser;
274 ifstream in(data_pathname.c_str(), ios::in);
275 parser.intern(in, dmr);
276
277 dmr->set_factory(0);
278#endif
279}
280
294{
295 BESDEBUG(MODULE, prolog << "BEGIN" << endl);
296
297 BESResponseObject *response = dhi.response_handler->get_response_object();
298 BESDMRResponse *bdmr = dynamic_cast<BESDMRResponse *>(response);
299 if (!bdmr) throw BESInternalError("Cast error, expected a BESDMRResponse object.", __FILE__, __LINE__);
300
301 try {
302 build_dmr_from_file(dhi.container, bdmr->get_dmr());
303 bdmr->set_dap4_constraint(dhi);
304 bdmr->set_dap4_function(dhi);
305 }
306 catch (...) {
307 handle_exception(__FILE__, __LINE__);
308 }
309
310 BESDEBUG(MODULE, prolog << "END" << endl);
311
312 return true;
313}
314
321{
322 BESStopWatch sw;
323 if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start(prolog + "timer" , dhi.data[REQUEST_ID]);
324
325 BESDEBUG(MODULE, prolog << "BEGIN" << endl);
326
327 BESResponseObject *response = dhi.response_handler->get_response_object();
328 auto *bdmr = dynamic_cast<BESDMRResponse *>(response);
329 if (!bdmr) throw BESInternalError("Cast error, expected a BESDMRResponse object.", __FILE__, __LINE__);
330
331 try {
332 build_dmr_from_file(dhi.container, bdmr->get_dmr());
333
334 // We don't need all the attributes, so use the lazy-load feature implemented
335 // using overloads of the BaseType::set_send_p() method.
336
337 bdmr->set_dap4_constraint(dhi);
338 bdmr->set_dap4_function(dhi);
339 }
340 catch (...) {
341 handle_exception(__FILE__, __LINE__);
342 }
343
344 BESDEBUG(MODULE, prolog << "END" << endl);
345
346 return true;
347}
348
355template <class T>
356void DmrppRequestHandler::get_dds_from_dmr_or_cache(BESDataHandlerInterface &dhi, T *bdds) {
357 string container_name_str = bdds->get_explicit_containers() ? dhi.container->get_symbolic_name() : "";
358
359 DDS *dds = bdds->get_dds();
360 if (!container_name_str.empty()) dds->container_name(container_name_str);
361 string accessed = dhi.container->access();
362
363 // Look in memory cache, if it's initialized
364 DDS *cached_dds_ptr = 0;
365 if (dds_cache && (cached_dds_ptr = static_cast<DDS*>(dds_cache->get(accessed)))) {
366 BESDEBUG(MODULE, prolog << "DDS Cached hit for : " << accessed << endl);
367 *dds = *cached_dds_ptr;
368 }
369 else {
370 DMR dmr;
371 build_dmr_from_file(dhi.container, &dmr);
372
373 delete dds; // delete the current one;
374 dds = dmr.getDDS(); // assign the new one.
375
376 // Stuff it into the response.
377 bdds->set_dds(dds);
378
379 // Cache it, if the cache is active.
380 if (dds_cache) {
381 dds_cache->add(new DDS(*dds), accessed);
382 }
383 }
384}
385
390{
391 BESStopWatch sw;
392 if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start(prolog + "timer" , dhi.data[REQUEST_ID]);
393
394 BESDEBUG(MODULE, prolog << "BEGIN" << endl);
395
396 BESResponseObject *response = dhi.response_handler->get_response_object();
397 BESDataDDSResponse *bdds = dynamic_cast<BESDataDDSResponse *>(response);
398 if (!bdds) throw BESInternalError("Cast error, expected a BESDataDDSResponse object.", __FILE__, __LINE__);
399
400 try {
401 get_dds_from_dmr_or_cache<BESDataDDSResponse>(dhi, bdds);
402 bdds->set_constraint(dhi);
403 bdds->clear_container();
404 }
405 catch (...) {
406 handle_exception(__FILE__, __LINE__);
407 }
408
409 BESDEBUG(MODULE, prolog << "END" << endl);
410 return true;
411}
412
413
418{
419 BESStopWatch sw;
420 if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start(prolog + "timer" , dhi.data[REQUEST_ID]);
421
422 BESDEBUG(MODULE, prolog << "BEGIN" << endl);
423
424 BESResponseObject *response = dhi.response_handler->get_response_object();
425 BESDDSResponse *bdds = dynamic_cast<BESDDSResponse *>(response);
426 if (!bdds) throw BESInternalError("Cast error, expected a BESDDSResponse object.", __FILE__, __LINE__);
427
428 try {
429 get_dds_from_dmr_or_cache<BESDDSResponse>(dhi, bdds);
430
431 bdds->set_constraint(dhi);
432 bdds->clear_container();
433 }
434 catch (...) {
435 handle_exception(__FILE__, __LINE__);
436 }
437
438 BESDEBUG(MODULE, prolog << "END" << endl);
439 return true;
440}
441
447{
448 BESStopWatch sw;
449 if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start(prolog + "timer" , dhi.data[REQUEST_ID]);
450
451 BESResponseObject *response = dhi.response_handler->get_response_object();
452 BESDASResponse *bdas = dynamic_cast<BESDASResponse *>(response);
453 if (!bdas) throw BESInternalError("Cast error, expected a BESDASResponse object.", __FILE__, __LINE__);
454
455 try {
456 string container_name_str = bdas->get_explicit_containers() ? dhi.container->get_symbolic_name() : "";
457
458 DAS *das = bdas->get_das();
459 if (!container_name_str.empty()) das->container_name(container_name_str);
460 string accessed = dhi.container->access();
461
462 // Look in memory cache (if it's initialized)
463 DAS *cached_das_ptr = 0;
464 if (das_cache && (cached_das_ptr = static_cast<DAS*>(das_cache->get(accessed)))) {
465 // copy the cached DAS into the BES response object
466 *das = *cached_das_ptr;
467 }
468 else {
469 DMR dmr;
470 build_dmr_from_file(dhi.container, &dmr);
471
472 // Get a DDS from the DMR, getDDS() allocates all new objects. Use unique_ptr
473 // to ensure this is deleted. jhrg 11/12/21
474 // TODO Add a getDAS() method to DMR so we don't have to go the long way?
475 // Or not and drop the DAP2 stuff until the code is higher up the chain?
476 // jhrg 11/12/21
477 unique_ptr<DDS> dds(dmr.getDDS());
478
479 // Load the BESDASResponse DAS from the DDS
480 dds->get_das(das);
481 Ancillary::read_ancillary_das(*das, accessed);
482
483 // Add to cache if cache is active
484 if (das_cache) {
485 // copy because the BES deletes the DAS held by the DHI.
486 // TODO Change the DHI to use shared_ptr objects. I think ... jhrg 11/12/21
487 das_cache->add(new DAS(*das), accessed);
488 }
489 }
490
491 bdas->clear_container();
492 }
493 catch (...) {
494 handle_exception(__FILE__, __LINE__);
495 }
496
497 BESDEBUG(MODULE, prolog << "END" << endl);
498 return true;
499}
500
501
502bool DmrppRequestHandler::dap_build_vers(BESDataHandlerInterface &dhi)
503{
504 BESVersionInfo *info = dynamic_cast<BESVersionInfo *>(dhi.response_handler->get_response_object());
505 if (!info) throw BESInternalFatalError("Expected a BESVersionInfo instance.", __FILE__, __LINE__);
506
507 info->add_module(MODULE_NAME, MODULE_VERSION);
508 return true;
509}
510
511bool DmrppRequestHandler::dap_build_help(BESDataHandlerInterface &dhi)
512{
513 BESInfo *info = dynamic_cast<BESInfo *>(dhi.response_handler->get_response_object());
514 if (!info) throw BESInternalFatalError("Expected a BESVersionInfo instance.", __FILE__, __LINE__);
515
516 // This is an example. If you had a help file you could load it like
517 // this and if your handler handled the following responses.
518 map<string, string> attrs;
519 attrs["name"] = MODULE_NAME;
520 attrs["version"] = MODULE_VERSION;
521 list<string> services;
522 BESServiceRegistry::TheRegistry()->services_handled(MODULE, services);
523 if (services.size() > 0) {
524 string handles = BESUtil::implode(services, ',');
525 attrs["handles"] = handles;
526 }
527 info->begin_tag("module", &attrs);
528 info->end_tag("module");
529
530 return true;
531}
532
533void DmrppRequestHandler::dump(ostream &strm) const
534{
535 strm << BESIndent::LMarg << "DmrppRequestHandler::dump - (" << (void *) this << ")" << endl;
536 BESIndent::Indent();
538 BESIndent::UnIndent();
539}
540
541} // namespace dmrpp
A container is something that holds data. E.G., a netcdf file or a database entry.
Definition: BESContainer.h:65
std::string get_symbolic_name() const
retrieve the symbolic name for this container
Definition: BESContainer.h:221
virtual std::string access()=0
returns the true name of this container
Represents an OPeNDAP DAS DAP2 data object within the BES.
virtual void clear_container()
clear the container in the DAP response object
Holds a DDS object within the BES.
virtual void clear_container()
clear the container in the DAP response object
Represents an OPeNDAP DMR DAP4 data object within the BES.
error object created from libdap error objects and can handle those errors
Definition: BESDapError.h:59
virtual void set_dap4_function(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_dap4_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
bool get_explicit_containers() const
Should containers be explicitly represented in the DD* responses?
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
virtual void clear_container()
clear the container in the DAP response object
Structure storing information used by the BES to handle the request.
std::map< std::string, std::string > data
the map of string data that will be required for the current request.
BESContainer * container
pointer to current container in this interface
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
Definition: BESDebug.h:168
Abstract exception class for the BES with basic string message.
Definition: BESError.h:58
informational response object
Definition: BESInfo.h:63
exception thrown if internal error encountered
exception thrown if an internal error is found and is fatal to the BES
Represents a specific data type request handler.
virtual bool add_method(const std::string &name, p_request_handler_method method)
add a handler method to the request handler that knows how to fill in a specific response object
virtual void dump(std::ostream &strm) const
dumps information about this object
virtual BESResponseObject * get_response_object()
return the current response object
Abstract base class representing a specific set of information in response to a request to the BES.
virtual void services_handled(const std::string &handler, std::list< std::string > &services)
returns the list of servies provided by the handler in question
virtual bool start(std::string name)
Definition: BESStopWatch.cc:67
static std::string lowercase(const std::string &s)
Definition: BESUtil.cc:206
static std::string implode(const std::list< std::string > &values, char delim)
Definition: BESUtil.cc:657
static CredentialsManager * theCM()
Returns the singleton instance of the CrednetialsManager.
An in-memory cache for DapObj (DAS, DDS, ...) objects.
Definition: ObjMemCache.h:84
virtual void add(libdap::DapObj *obj, const std::string &key)
Add an object to the cache and associate it with a key.
Definition: ObjMemCache.cc:63
virtual libdap::DapObj * get(const std::string &key)
Get the cached pointer.
Definition: ObjMemCache.cc:105
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:340
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:71
static bool dap_build_dds(BESDataHandlerInterface &dhi)
void dump(std::ostream &strm) const override
dumps information about this object
static bool dap_build_dap2data(BESDataHandlerInterface &dhi)
static bool dap_build_dmr(BESDataHandlerInterface &dhi)
static bool dap_build_das(BESDataHandlerInterface &dhi)
static bool dap_build_dap4data(BESDataHandlerInterface &dhi)
Build a DAP4 data response. Adds timing to dap_build_dmr()
DmrppRequestHandler(const std::string &name)