bes Updated for version 3.20.10
HDF5CF.cc
Go to the documentation of this file.
1// This file is part of the hdf5_handler implementing for the CF-compliant
2// Copyright (c) 2011-2016 The HDF Group, Inc. and OPeNDAP, Inc.
3//
4// This is free software; you can redistribute it and/or modify it under the
5// terms of the GNU Lesser General Public License as published by the Free
6// Software Foundation; either version 2.1 of the License, or (at your
7// option) any later version.
8//
9// This software is distributed in the hope that it will be useful, but
10// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
11// or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
12// License for more details.
13//
14// You should have received a copy of the GNU Lesser General Public
15// License along with this library; if not, write to the Free Software
16// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17//
18// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
19// You can contact The HDF Group, Inc. at 1800 South Oak Street,
20// Suite 203, Champaign, IL 61820
21
36
37#include <sstream>
38#include <algorithm>
39#include <functional>
40#include <climits>
41#include "HDF5CF.h"
42#include "h5cfdaputil.h"
43#include "HDF5RequestHandler.h"
44#include "h5apicompatible.h"
45#include "BESDebug.h"
46
47using namespace HDF5CF;
48
49Var::Var(Var *var)
50{
51
52 newname = var->newname;
53 name = var->name;
54 fullpath = var->fullpath;
55 rank = var->rank;
56 total_elems = var->total_elems;
57 zero_storage_size = var->zero_storage_size;
58 dtype = var->dtype;
59 comp_ratio = var->comp_ratio;
60 unsupported_attr_dtype = var->unsupported_attr_dtype;
61 unsupported_attr_dspace = var->unsupported_attr_dspace;
62 unsupported_dspace = var->unsupported_dspace;
63 unsupported_attr_dspace = var->unsupported_attr_dspace;
64 dimnameflag = var->dimnameflag;
65 coord_attr_add_path = var->coord_attr_add_path;
66
67 for (vector<Attribute*>::iterator ira = var->attrs.begin(); ira != var->attrs.end(); ++ira) {
68 Attribute* attr = new Attribute();
69 attr->name = (*ira)->name;
70 attr->newname = (*ira)->newname;
71 attr->dtype = (*ira)->dtype;
72 attr->count = (*ira)->count;
73 attr->strsize = (*ira)->strsize;
74 attr->fstrsize = (*ira)->fstrsize;
75 attr->value = (*ira)->value;
76 attrs.push_back(attr);
77 }
78
79 for (vector<Dimension*>::iterator ird = var->dims.begin(); ird != var->dims.end(); ++ird) {
80 Dimension *dim = new Dimension((*ird)->size);
81 dim->name = (*ird)->name;
82 dim->newname = (*ird)->newname;
83 dim->unlimited_dim = (*ird)->unlimited_dim;
84 dims.push_back(dim);
85 }
86
87}
88
89bool CVar::isLatLon() const
90{
91
92 bool ret_value = false;
93 if (CV_EXIST == this->cvartype || CV_MODIFY == this->cvartype || CV_SPECIAL == this->cvartype) {
94 string attr_name = "units";
95 string lat_unit_value = "degrees_north";
96 string lon_unit_value = "degrees_east";
97
98 for (vector<Attribute *>::const_iterator ira = this->attrs.begin(); ira != this->attrs.end(); ira++) {
99
100 if ((H5FSTRING == (*ira)->getType()) || (H5VSTRING == (*ira)->getType())) {
101 if (attr_name == (*ira)->newname) {
102 string attr_value1((*ira)->getValue().begin(), (*ira)->getValue().end());
103
104 if ((*ira)->getCount() == 1) {
105 string attr_value((*ira)->getValue().begin(), (*ira)->getValue().end());
106 if (attr_value.compare(0, lat_unit_value.size(), lat_unit_value) == 0) {
107 if (attr_value.size() == lat_unit_value.size()) {
108 ret_value = true;
109 break;
110 }
111 else if (attr_value.size() == (lat_unit_value.size() + 1)) {
112 if (attr_value[attr_value.size() - 1] == '\0'
113 || attr_value[attr_value.size() - 1] == ' ') {
114 ret_value = true;
115 break;
116 }
117 }
118 }
119 else if (attr_value.compare(0, lon_unit_value.size(), lon_unit_value) == 0) {
120 if (attr_value.size() == lon_unit_value.size()) {
121 ret_value = true;
122 break;
123 }
124 else if (attr_value.size() == (lon_unit_value.size() + 1)) {
125 if (attr_value[attr_value.size() - 1] == '\0'
126 || attr_value[attr_value.size() - 1] == ' ') {
127 ret_value = true;
128 break;
129 }
130 }
131
132 }
133
134 }
135 }
136 }
137 }
138 }
139 else if (this->cvartype == CV_LAT_MISS || this->cvartype == CV_LON_MISS) ret_value = true;
140 return ret_value;
141
142}
143File::~File()
144{
145
146 if (this->fileid >= 0) {
147 if (this->rootid >= 0) {
148 for_each(this->groups.begin(), this->groups.end(), delete_elem());
149 for_each(this->vars.begin(), this->vars.end(), delete_elem());
150 for_each(this->root_attrs.begin(), this->root_attrs.end(), delete_elem());
151 H5Gclose(rootid);
152 }
153 }
154}
155
156Group::~Group()
157{
158 for_each(this->attrs.begin(), this->attrs.end(), delete_elem());
159}
160
161Var::~Var()
162{
163 for_each(this->dims.begin(), this->dims.end(), delete_elem());
164 for_each(this->attrs.begin(), this->attrs.end(), delete_elem());
165}
166
167Attribute::~Attribute()
168{
169}
170
171void File::Retrieve_H5_Info(const char * /*path*/, hid_t file_id, bool include_attr)
172{
173
174 BESDEBUG("h5", "coming to Retrieve_H5_Info" <<endl);
175
176 if (true == include_attr) {
177
178 // Obtain the BES key to check the ignored objects
179 // We will only use DAS to output these information.
180 this->check_ignored = HDF5RequestHandler::get_check_ignore_obj();
181 if (true == this->check_ignored) this->add_ignored_info_page_header();
182
183 }
184
185 hid_t root_id;
186 if ((root_id = H5Gopen(file_id, "/", H5P_DEFAULT)) < 0) {
187 throw1("Cannot open the HDF5 root group ");
188 }
189 this->rootid = root_id;
190 try {
191 this->Retrieve_H5_Obj(root_id, "/", include_attr);
192 }
193 catch (...) {
194 throw;
195 }
196
197 // Obtain attributes only necessary
198 if (true == include_attr) {
199
200 // Find the file(root group) attribute
201
202 // Obtain the object type, such as group or dataset.
203 H5O_info_t oinfo;
204 int num_attrs = 0;
205
206 if (H5OGET_INFO(root_id, &oinfo) < 0)
207 throw1("Error obtaining the info for the root group");
208
209 num_attrs = oinfo.num_attrs;
210 bool temp_unsup_attr_atype = false;
211 bool temp_unsup_attr_dspace = false;
212
213 for (int j = 0; j < num_attrs; j++) {
214 Attribute * attr = new Attribute();
215 try {
216 this->Retrieve_H5_Attr_Info(attr, root_id, j, temp_unsup_attr_atype, temp_unsup_attr_dspace);
217 }
218 catch (...) {
219 delete attr;
220 throw;
221
222 }
223 this->root_attrs.push_back(attr);
224 }
225
226 this->unsupported_attr_dtype = temp_unsup_attr_atype;
227 this->unsupported_attr_dspace = temp_unsup_attr_dspace;
228 }
229}
230
231void File::Retrieve_H5_Obj(hid_t grp_id, const char*gname, bool include_attr)
232{
233
234 // Iterate through the file to see the members of the group from the root.
235 H5G_info_t g_info;
236 hsize_t nelems = 0;
237
238 if (H5Gget_info(grp_id, &g_info) < 0)
239 throw2("Counting hdf5 group elements error for ", gname);
240 nelems = g_info.nlinks;
241
242 ssize_t oname_size = 0;
243 for (hsize_t i = 0; i < nelems; i++) {
244
245 hid_t cgroup = -1;
246 hid_t cdset = -1;
247 Group *group = NULL;
248 Var *var = NULL;
249 Attribute *attr = NULL;
250
251 try {
252
253 size_t dummy_name_len = 1;
254
255 // Query the length of object name.
256 oname_size = H5Lget_name_by_idx(grp_id, ".", H5_INDEX_NAME, H5_ITER_NATIVE, i, NULL, dummy_name_len,
257 H5P_DEFAULT);
258 if (oname_size <= 0)
259 throw2("Error getting the size of the hdf5 object from the group: ", gname);
260
261 // Obtain the name of the object
262 vector<char> oname;
263 oname.resize((size_t) oname_size + 1);
264
265 if (H5Lget_name_by_idx(grp_id, ".", H5_INDEX_NAME, H5_ITER_NATIVE, i, &oname[0], (size_t) (oname_size + 1),
266 H5P_DEFAULT) < 0)
267 throw2("Error getting the hdf5 object name from the group: ", gname);
268
269 // Check if it is a hard link or a soft link
270 H5L_info_t linfo;
271 if (H5Lget_info(grp_id, &oname[0], &linfo, H5P_DEFAULT) < 0)
272 throw2("HDF5 link name error from ", gname);
273
274 // We ignore soft links and external links for the CF options
275 if (H5L_TYPE_SOFT == linfo.type || H5L_TYPE_EXTERNAL == linfo.type) {
276 if (true == include_attr && true == check_ignored) {
277 this->add_ignored_info_links_header();
278 string full_path_name;
279 string temp_oname(oname.begin(), oname.end());
280 full_path_name = (
281 (string(gname) != "/") ?
282 (string(gname) + "/" + temp_oname.substr(0, temp_oname.size() - 1)) :
283 ("/" + temp_oname.substr(0, temp_oname.size() - 1)));
284 this->add_ignored_info_links(full_path_name);
285
286 }
287 continue;
288 }
289
290 // Obtain the object type, such as group or dataset.
291 H5O_info_t oinfo;
292
293 if (H5OGET_INFO_BY_IDX(grp_id, ".", H5_INDEX_NAME, H5_ITER_NATIVE, i, &oinfo, H5P_DEFAULT) < 0)
294 throw2("Error obtaining the info for the object ", string(oname.begin(), oname.end()));
295
296 H5O_type_t obj_type = oinfo.type;
297
298 switch (obj_type) {
299
300 case H5O_TYPE_GROUP: {
301
302 // Obtain the full path name
303 string full_path_name;
304 string temp_oname(oname.begin(), oname.end());
305
306 full_path_name = (
307 (string(gname) != "/") ?
308 (string(gname) + "/" + temp_oname.substr(0, temp_oname.size() - 1)) :
309 ("/" + temp_oname.substr(0, temp_oname.size() - 1)));
310
311 cgroup = H5Gopen(grp_id, full_path_name.c_str(), H5P_DEFAULT);
312 if (cgroup < 0)
313 throw2("Error opening the group ", full_path_name);
314
315 group = new Group();
316 group->path = full_path_name;
317 group->newname = full_path_name;
318
319 // Retrieve group attribute if the attribute flag is true
320 if (true == include_attr) {
321
322 int num_attrs = oinfo.num_attrs;
323 bool temp_unsup_attr_dtype = false;
324 bool temp_unsup_attr_dspace = false;
325
326 for (int j = 0; j < num_attrs; j++) {
327
328 attr = new Attribute();
329 Retrieve_H5_Attr_Info(attr, cgroup, j, temp_unsup_attr_dtype, temp_unsup_attr_dspace);
330 group->attrs.push_back(attr);
331 attr = NULL;
332 }
333
334 group->unsupported_attr_dtype = temp_unsup_attr_dtype;
335 group->unsupported_attr_dspace = temp_unsup_attr_dspace;
336 }
337 this->groups.push_back(group);
338 Retrieve_H5_Obj(cgroup, full_path_name.c_str(), include_attr);
339 if (H5Gclose(cgroup) < 0)
340 throw2("Error closing the group ", full_path_name);
341 }
342 break;
343 case H5O_TYPE_DATASET: {
344
345 // Obtain the absolute path of the HDF5 dataset
346 string temp_oname(oname.begin(), oname.end());
347 string full_path_name = (
348 (string(gname) != "/") ?
349 (string(gname) + "/" + temp_oname.substr(0, temp_oname.size() - 1)) :
350 ("/" + temp_oname.substr(0, temp_oname.size() - 1)));
351
352 var = new Var();
353 var->name = temp_oname.substr(0, temp_oname.size() - 1);
354 var->fullpath = full_path_name;
355
356 // newname is for the final CF name
357 var->newname = full_path_name;
358
359 cdset = H5Dopen(grp_id, full_path_name.c_str(), H5P_DEFAULT);
360 if (cdset < 0)
361 throw2("Error opening the HDF5 dataset ", full_path_name);
362
363 // Retrieve the HDF5 dataset datatype, return the flag for unsupported types.
364 bool temp_unsup_var_dtype = false;
365 Retrieve_H5_VarType(var, cdset, full_path_name, temp_unsup_var_dtype);
366
367 // Update the unsupported datatype flag
368 if (!this->unsupported_var_dtype && temp_unsup_var_dtype) this->unsupported_var_dtype = true;
369
370 // Retrieve the HDF5 dataset data space, return the flag for unsupported dataspaces.
371 bool temp_unsup_var_dspace = false;
372 Retrieve_H5_VarDim(var, cdset, full_path_name, temp_unsup_var_dspace);
373
374 // Update the unsupported data space flag
375 if (!this->unsupported_var_dspace && temp_unsup_var_dspace) this->unsupported_var_dspace = true;
376
377 hsize_t d_storage_size = H5Dget_storage_size(cdset);
378 var->zero_storage_size =(d_storage_size ==0);
379 var->comp_ratio = Retrieve_H5_VarCompRatio(var, cdset);
380
381 // Retrieve the attribute info. if asked
382 if (true == include_attr) {
383
384 int num_attrs = oinfo.num_attrs;
385 bool temp_unsup_attr_dtype = false;
386 bool temp_unsup_attr_dspace = false;
387
388 for (int j = 0; j < num_attrs; j++) {
389
390 attr = new Attribute();
391
392 Retrieve_H5_Attr_Info(attr, cdset, j, temp_unsup_attr_dtype, temp_unsup_attr_dspace);
393 var->attrs.push_back(attr);
394 attr = NULL;
395 }
396
397 var->unsupported_attr_dtype = temp_unsup_attr_dtype;
398 var->unsupported_attr_dspace = temp_unsup_attr_dspace;
399
400 if (!this->unsupported_var_attr_dspace && temp_unsup_attr_dspace)
401 this->unsupported_var_attr_dspace = true;
402 }
403
404 this->vars.push_back(var);
405 if (H5Dclose(cdset) < 0)
406 throw2("Error closing the HDF5 dataset ", full_path_name);
407 }
408 break;
409
410 case H5O_TYPE_NAMED_DATATYPE: {
411 // ignore the named datatype
412 if (true == include_attr && true == check_ignored) {
413 this->add_ignored_info_namedtypes(string(gname), string(oname.begin(), oname.end()));
414 }
415 }
416 break;
417 default:
418 break;
419 } // "switch (obj_type)"
420 } // try
421 catch (...) {
422
423 if (attr != NULL) {
424 delete attr;
425 attr = NULL;
426 }
427
428 if (var != NULL) {
429 delete var;
430 var = NULL;
431 }
432
433 if (group != NULL) {
434 delete group;
435 group = NULL;
436 }
437
438 if (cgroup != -1) H5Gclose(cgroup);
439
440 if (cdset != -1) H5Dclose(cdset);
441 throw;
442
443 } // catch
444 } // "for (hsize_t i = 0; i < nelems; i++)"
445
446}
447
448// Retrieve HDF5 dataset datatype
449float File::Retrieve_H5_VarCompRatio(Var *var, hid_t dset_id)
450{
451
452 float comp_ratio = 1.0;
453 // Obtain the data type of the variable.
454 hid_t dset_create_plist = H5Dget_create_plist(dset_id);
455 if (dset_create_plist < 0)
456 throw1("unable to obtain hdf5 dataset creation property list ");
457 H5D_layout_t dset_layout = H5Pget_layout(dset_create_plist);
458 if (dset_layout < 0) {
459 H5Pclose(dset_create_plist);
460 throw1("unable to obtain hdf5 dataset creation property list storage layout");
461 }
462
463 if (dset_layout == H5D_CHUNKED) {
464
465 hsize_t dstorage_size = H5Dget_storage_size(dset_id);
466 if (dstorage_size > 0 && var->total_elems > 0) {
467 hid_t ty_id = -1;
468
469 // Obtain the data type of the variable.
470 if ((ty_id = H5Dget_type(dset_id)) < 0)
471 throw1("unable to obtain hdf5 datatype for the dataset ");
472 size_t type_size = H5Tget_size(ty_id);
473 comp_ratio = ((float) (var->total_elems) * type_size) / dstorage_size;
474 H5Tclose(ty_id);
475 }
476
477 }
478 H5Pclose(dset_create_plist);
479 return comp_ratio;
480
481}
482// Retrieve HDF5 dataset datatype
483void File::Retrieve_H5_VarType(Var *var, hid_t dset_id, const string & varname, bool &unsup_var_dtype)
484{
485
486 hid_t ty_id = -1;
487
488 // Obtain the data type of the variable.
489 if ((ty_id = H5Dget_type(dset_id)) < 0)
490 throw2("unable to obtain hdf5 datatype for the dataset ", varname);
491
492 // The following datatype class and datatype will not be supported for the CF option.
493 // H5T_TIME, H5T_BITFIELD
494 // H5T_OPAQUE, H5T_ENUM
495 // H5T_REFERENCE, H5T_COMPOUND
496 // H5T_VLEN,H5T_ARRAY
497 // 64-bit integer
498
499 // Note: H5T_REFERENCE H5T_COMPOUND and H5T_ARRAY can be mapped to DAP2 DDS for the default option.
500 // H5T_COMPOUND, H5T_ARRAY can be mapped to DAP2 DAS for the default option.
501 // 1-D variable length of string can also be mapped for the CF option..
502 // The variable length string class is H5T_STRING rather than H5T_VLEN,
503 // We also ignore the mapping of integer 64 bit since DAP2 doesn't
504 // support 64-bit integer. In theory, DAP2 doesn't support long double
505 // (128-bit or 92-bit floating point type).
506 //
507
508 var->dtype = HDF5CFUtil::H5type_to_H5DAPtype(ty_id);
509 if (false == HDF5CFUtil::cf_strict_support_type(var->dtype,_is_dap4))
510 unsup_var_dtype = true;
511
512 if (H5Tclose(ty_id) < 0)
513 throw1("Unable to close the HDF5 datatype ");;
514}
515
516// Retrieve the HDF5 dataset dimension information
517void File::Retrieve_H5_VarDim(Var *var, hid_t dset_id, const string & varname, bool &unsup_var_dspace)
518{
519
520 vector<hsize_t> dsize;
521 vector<hsize_t> maxsize;
522
523 hid_t dspace_id = -1;
524 hid_t ty_id = -1;
525
526 try {
527 if ((dspace_id = H5Dget_space(dset_id)) < 0)
528 throw2("Cannot get hdf5 dataspace id for the variable ", varname);
529
530 H5S_class_t space_class = H5S_NO_CLASS;
531 if ((space_class = H5Sget_simple_extent_type(dspace_id)) < 0)
532 throw2("Cannot obtain the HDF5 dataspace class for the variable ", varname);
533
534 if (H5S_NULL == space_class)
535 unsup_var_dspace = true;
536 else {
537 if (false == unsup_var_dspace) {
538
539 hssize_t h5_total_elms = H5Sget_simple_extent_npoints(dspace_id);
540 if (h5_total_elms < 0)
541 throw2("Cannot get the total number of elements of HDF5 dataset ", varname);
542 else
543 var->total_elems = (size_t) h5_total_elms;
544 int ndims = H5Sget_simple_extent_ndims(dspace_id);
545 if (ndims < 0)
546 throw2("Cannot get the hdf5 dataspace number of dimension for the variable ", varname);
547
548 var->rank = ndims;
549 if (ndims != 0) {
550 dsize.resize(ndims);
551 maxsize.resize(ndims);
552 }
553
554 // The netcdf DAP client supports the representation of the unlimited dimension.
555 // So we need to check.
556 if (H5Sget_simple_extent_dims(dspace_id, &dsize[0], &maxsize[0]) < 0)
557 throw2("Cannot obtain the dim. info for the variable ", varname);
558
559 for (int i = 0; i < ndims; i++) {
560 Dimension * dim = new Dimension(dsize[i]);
561 if (maxsize[i] == H5S_UNLIMITED) {
562 dim->unlimited_dim = true;
563 if (false == have_udim) have_udim = true;
564 }
565 var->dims.push_back(dim);
566 }
567 }
568 }
569
570 var->unsupported_dspace = unsup_var_dspace;
571
572 if (H5Sclose(dspace_id) < 0)
573 throw1("Cannot close the HDF5 dataspace .");
574
575 }
576
577 catch (...) {
578
579 if (dspace_id != -1) H5Sclose(dspace_id);
580
581 if (ty_id != -1) H5Tclose(ty_id);
582 throw;
583 }
584
585}
586
587// Retrieve the HDF5 attribute information.
588void File::Retrieve_H5_Attr_Info(Attribute * attr, hid_t obj_id, const int j, bool &unsup_attr_dtype,
589 bool &unsup_attr_dspace)
590
591{
592
593 hid_t attrid = -1;
594 hid_t ty_id = -1;
595 hid_t aspace_id = -1;
596 hid_t memtype = -1;
597
598 try {
599
600 // Obtain the attribute ID.
601 if ((attrid = H5Aopen_by_idx(obj_id, ".", H5_INDEX_CRT_ORDER, H5_ITER_INC, (hsize_t) j, H5P_DEFAULT,
602 H5P_DEFAULT)) < 0)
603 throw1("Unable to open attribute by index ");
604
605 // Obtain the size of attribute name.
606 ssize_t name_size = H5Aget_name(attrid, 0, NULL);
607 if (name_size < 0)
608 throw1("Unable to obtain the size of the hdf5 attribute name ");
609
610 string attr_name;
611 attr_name.resize(name_size + 1);
612
613 // Obtain the attribute name.
614 if ((H5Aget_name(attrid, name_size + 1, &attr_name[0])) < 0)
615 throw1("unable to obtain the hdf5 attribute name ");
616
617 // Obtain the type of the attribute.
618 if ((ty_id = H5Aget_type(attrid)) < 0)
619 throw2("unable to obtain hdf5 datatype for the attribute ", attr_name);
620
621 // The following datatype class and datatype will not be supported for the CF option.
622 // H5T_TIME, H5T_BITFIELD
623 // H5T_OPAQUE, H5T_ENUM
624 // H5T_REFERENCE, H5T_COMPOUND
625 // H5T_VLEN,H5T_ARRAY
626 // 64-bit integer
627
628 // Note: H5T_REFERENCE H5T_COMPOUND and H5T_ARRAY can be mapped to DAP2 DDS for the default option.
629 // H5T_COMPOUND, H5T_ARRAY can be mapped to DAP2 DAS for the default option.
630 // 1-D variable length of string can also be mapped for the CF option..
631 // The variable length string class is H5T_STRING rather than H5T_VLEN,
632 // We also ignore the mapping of integer 64 bit since DAP2 doesn't
633 // support 64-bit integer. In theory, DAP2 doesn't support long double
634 // (128-bit or 92-bit floating point type).
635 //
636 attr->dtype = HDF5CFUtil::H5type_to_H5DAPtype(ty_id);
637 if (false == HDF5CFUtil::cf_strict_support_type(attr->dtype,_is_dap4))
638 unsup_attr_dtype = true;
639
640 if(H5VSTRING == attr->dtype || H5FSTRING == attr->dtype) {
641 H5T_cset_t c_set_type = H5Tget_cset(ty_id);
642 if(c_set_type <0)
643 throw2("Cannot get hdf5 character set type for the attribute ", attr_name);
644 // This is a UTF-8 string
645 if(c_set_type == 1)
646 attr->is_cset_ascii = false;
647 }
648
649 if ((aspace_id = H5Aget_space(attrid)) < 0)
650 throw2("Cannot get hdf5 dataspace id for the attribute ", attr_name);
651
652 int ndims = H5Sget_simple_extent_ndims(aspace_id);
653 if (ndims < 0)
654 throw2("Cannot get the hdf5 dataspace number of dimension for attribute ", attr_name);
655
656 hsize_t nelmts = 1;
657
658 // if it is a scalar attribute, just define number of elements to be 1.
659 if (ndims != 0) {
660
661 vector<hsize_t> asize;
662 vector<hsize_t> maxsize;
663 asize.resize(ndims);
664 maxsize.resize(ndims);
665
666 // Obtain the attribute data space information.
667 if (H5Sget_simple_extent_dims(aspace_id, &asize[0], &maxsize[0]) < 0)
668 throw2("Cannot obtain the dim. info for the attribute ", attr_name);
669
670 // Here we need to take care of 0-length attribute. This is legal in HDF5.
671 for (int dim_count = 0;dim_count < ndims; dim_count ++) {
672 // STOP adding unsupported_attr_dspace!
673 if (asize[dim_count] == 0) {
674 unsup_attr_dspace = true;
675 break;
676 }
677 }
678
679 if (false == unsup_attr_dspace) {
680 // Return ndims and size[ndims].
681 for (int dim_count = 0; dim_count< ndims; dim_count++)
682 nelmts *= asize[dim_count];
683 }
684 else
685 nelmts = 0;
686 } // "if(ndims != 0)"
687
688 size_t ty_size = H5Tget_size(ty_id);
689 if (0 == ty_size)
690 throw2("Cannot obtain the dtype size for the attribute ", attr_name);
691
692 memtype = H5Tget_native_type(ty_id, H5T_DIR_ASCEND);
693 if (memtype < 0)
694 throw2("Cannot obtain the memory datatype for the attribute ", attr_name);
695
696 // Store the name and the count
697 string temp_aname(attr_name.begin(), attr_name.end());
698 attr->name = temp_aname.substr(0, temp_aname.size() - 1);
699 attr->newname = attr->name;
700 attr->count = nelmts;
701
702 // Release HDF5 resources.
703 if (H5Tclose(ty_id) < 0)
704 throw1("Cannot successfully close the attribute datatype.");
705 if (H5Tclose(memtype) < 0)
706 throw1("Cannot successfully close the attribute memory datatype.");
707 if (H5Sclose(aspace_id) < 0)
708 throw1("Cannot successfully close the HDF5 dataspace.");
709 if (H5Aclose(attrid) < 0)
710 throw1("Cannot successfully close the HDF5 attribute.");
711
712 } // try
713 catch (...) {
714
715 if (ty_id != -1) H5Tclose(ty_id);
716
717 if (memtype != -1) H5Tclose(memtype);
718
719 if (aspace_id != -1) H5Sclose(aspace_id);
720
721 if (attrid != -1) H5Aclose(attrid);
722
723 throw;
724 }
725
726}
727
728// Retrieve all HDF5 supported attribute values.
730{
731
732 for (vector<Attribute *>::iterator ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira)
733 Retrieve_H5_Attr_Value(*ira, "/");
734
735 for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
736 for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end(); ++ira) {
737 Retrieve_H5_Attr_Value(*ira, (*irg)->path);
738 }
739 }
740
741 for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
742 for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
743 Retrieve_H5_Attr_Value(*ira, (*irv)->fullpath);
744 }
745 }
746}
747
749{
750 for (vector<Attribute *>::iterator ira = var->attrs.begin(); ira != var->attrs.end(); ++ira) {
751 Retrieve_H5_Attr_Value(*ira, var->fullpath);
752 }
753}
754
755// Retrieve the values of a specific HDF5 attribute.
756void File::Retrieve_H5_Attr_Value(Attribute *attr, string obj_name)
757{
758
759 // Define HDF5 object Ids.
760 hid_t obj_id = -1;
761 hid_t attr_id = -1;
762 hid_t ty_id = -1;
763 hid_t memtype_id = -1;
764 hid_t aspace_id = -1;
765
766 try {
767
768 // Open the object that hold this attribute
769 obj_id = H5Oopen(this->fileid, obj_name.c_str(), H5P_DEFAULT);
770 if (obj_id < 0)
771 throw2("Cannot open the object ", obj_name);
772
773 attr_id = H5Aopen(obj_id, (attr->name).c_str(), H5P_DEFAULT);
774 if (attr_id < 0)
775 throw4("Cannot open the attribute ", attr->name, " of object ", obj_name);
776
777 ty_id = H5Aget_type(attr_id);
778 if (ty_id < 0)
779 throw4("Cannot obtain the datatype of the attribute ", attr->name, " of object ", obj_name);
780
781 memtype_id = H5Tget_native_type(ty_id, H5T_DIR_ASCEND);
782 if (memtype_id < 0)
783 throw2("Cannot obtain the memory datatype for the attribute ", attr->name);
784
785 size_t ty_size = H5Tget_size(memtype_id);
786 if (0 == ty_size)
787 throw4("Cannot obtain the dtype size for the attribute ", attr->name, " of object ", obj_name);
788
789 size_t total_bytes = attr->count * ty_size;
790
791 // We have to handle variable length string differently.
792 if (H5VSTRING == attr->dtype) {
793
794 // Variable length string attribute values only store pointers of the actual string value.
795 vector<char> temp_buf;
796 temp_buf.resize(total_bytes);
797
798 if (H5Aread(attr_id, memtype_id, &temp_buf[0]) < 0)
799 throw4("Cannot obtain the value of the attribute ", attr->name, " of object ", obj_name);
800
801 char *temp_bp = NULL;
802 char *ptr_1stvlen_ptr = &temp_buf[0];
803 temp_bp = &temp_buf[0];
804 char* onestring = NULL;
805 string total_vstring = "";
806
807 attr->strsize.resize(attr->count);
808
809 for (unsigned int temp_i = 0; temp_i < attr->count; temp_i++) {
810
811 // This line will assure that we get the real variable length string value.
812 onestring = *(char **) temp_bp;
813 if (onestring != NULL) {
814 total_vstring += string(onestring);
815 attr->strsize[temp_i] = (string(onestring)).size();
816 }
817 else
818 attr->strsize[temp_i] = 0;
819
820 // going to the next value.
821 temp_bp += ty_size;
822 }
823
824 if (ptr_1stvlen_ptr != NULL) {
825 aspace_id = H5Aget_space(attr_id);
826 if (aspace_id < 0)
827 throw4("Cannot obtain space id for ", attr->name, " of object ", obj_name);
828
829 // Reclaim any VL memory if necessary.
830 if (H5Dvlen_reclaim(memtype_id, aspace_id, H5P_DEFAULT, &temp_buf[0]) < 0)
831 throw4("Cannot reclaim VL memory for ", attr->name, " of object ", obj_name);
832
833 H5Sclose(aspace_id);
834 }
835
836 if (HDF5CFUtil::H5type_to_H5DAPtype(ty_id) != H5VSTRING)
837 throw4("Error to obtain the VL string type for attribute ", attr->name, " of object ", obj_name);
838
839 attr->value.resize(total_vstring.size());
840
841 copy(total_vstring.begin(), total_vstring.end(), attr->value.begin());
842
843 }
844 else {
845
846 if (attr->dtype == H5FSTRING) {
847 attr->fstrsize = ty_size;
848 }
849
850 attr->value.resize(total_bytes);
851
852 // Read HDF5 attribute data.
853 if (H5Aread(attr_id, memtype_id, (void *) &attr->value[0]) < 0)
854 throw4("Cannot obtain the dtype size for the attribute ", attr->name, " of object ", obj_name);
855
856 if (attr->dtype == H5FSTRING) {
857
858 size_t sect_size = ty_size;
859 int num_sect = 1;
860 if (sect_size > 0)
861 num_sect =
862 (total_bytes % sect_size == 0) ? (total_bytes / sect_size) : (total_bytes / sect_size + 1);
863 else
864 throw4("The attribute datatype size is not a positive integer ", attr->name, " of object ",
865 obj_name);
866
867 vector<size_t> sect_newsize;
868 sect_newsize.resize(num_sect);
869
870 string total_fstring = string(attr->value.begin(), attr->value.end());
871
872 string new_total_fstring = HDF5CFUtil::trim_string(memtype_id, total_fstring, num_sect, sect_size,
873 sect_newsize);
874 attr->value.resize(new_total_fstring.size());
875 copy(new_total_fstring.begin(), new_total_fstring.end(), attr->value.begin());
876 attr->strsize.resize(num_sect);
877 for (int temp_i = 0; temp_i < num_sect; temp_i++)
878 attr->strsize[temp_i] = sect_newsize[temp_i];
879
880#if 0
881 // "h5","new string value " <<string(attr->value.begin(), attr->value.end()) <<endl;
882 cerr<<"Attr name is "<<attr->name <<endl;
883 for (int temp_i = 0; temp_i <num_sect; temp_i ++)
884 cerr<<"string new section size = " << attr->strsize[temp_i] <<endl;
885#endif
886 }
887 }
888
889 if (H5Tclose(memtype_id) < 0)
890 throw1("Fail to close the HDF5 memory datatype ID.");
891 if (H5Tclose(ty_id) < 0)
892 throw1("Fail to close the HDF5 datatype ID.");
893 if (H5Aclose(attr_id) < 0)
894 throw1("Fail to close the HDF5 attribute ID.");
895 if (H5Oclose(obj_id) < 0)
896 throw1("Fail to close the HDF5 object ID.");
897
898 }
899
900 catch (...) {
901
902 if (memtype_id != -1) H5Tclose(memtype_id);
903
904 if (ty_id != -1) H5Tclose(ty_id);
905
906 if (aspace_id != -1) H5Sclose(aspace_id);
907
908 if (attr_id != -1) H5Aclose(attr_id);
909
910 if (obj_id != -1) H5Oclose(obj_id);
911
912 throw;
913 }
914
915}
916
917// Handle the unsupported datatype
918void File::Handle_Unsupported_Dtype(bool include_attr)
919{
920
921 if (true == include_attr) {
922 Handle_Group_Unsupported_Dtype();
923 Handle_VarAttr_Unsupported_Dtype();
924 }
925
926 Handle_Var_Unsupported_Dtype();
927}
928
929//Leave this code here.
930#if 0
931// First the root attributes
932if (true == include_attr) {
933 if (true == this->unsupported_attr_dtype) {
934 for (vector<Attribute *>::iterator ira = this->root_attrs.begin();
935 ira != this->root_attrs.end(); ) {
936 H5DataType temp_dtype = (*ira)->getType();
937 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
938 delete (*ira);
939 ira = this->root_attrs.erase(ira);
940
941 }
942 else {
943 ++ira;
944 }
945 }
946 }
947}
948
949// Then the group attributes
950if (false == this->groups.empty()) {
951 for (vector<Group *>::iterator irg = this->groups.begin();
952 irg != this->groups.end(); ++irg) {
953 if (false == (*irg)->attrs.empty()) {
954 if (true == (*irg)->unsupported_attr_dtype) {
955 for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin();
956 ira != (*irg)->attrs.end(); ) {
957 H5DataType temp_dtype = (*ira)->getType();
958 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
959 delete (*ira);
960 ira = (*irg)->attrs.erase(ira);
961 }
962 else {
963 ++ira;
964 }
965 }
966 }
967 }
968 }
969}
970}
971
972 // Then the variable(HDF5 dataset) and the correponding attributes.
973if (false == this->vars.empty()) {
974if (true == include_attr) {
975 for (vector<Var *>::iterator irv = this->vars.begin();
976 irv != this->vars.end();++irv ) {
977 if (false == (*irv)->attrs.empty()) {
978 if (true == (*irv)->unsupported_attr_dtype) {
979 for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin();
980 ira != (*irv)->attrs.end(); ) {
981 H5DataType temp_dtype = (*ira)->getType();
982 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
983 delete (*ira);
984 ira = (*irv)->attrs.erase(ira);
985 //ira--;
986 }
987 else {
988 ++ira;
989 }
990 }
991 }
992 }
993 }
994}
995if (true == this->unsupported_var_dtype) {
996 // "h5","having unsupported variable datatype" <<endl;
997 for (vector<Var *>::iterator irv = this->vars.begin();
998 irv != this->vars.end(); ) {
999 H5DataType temp_dtype = (*irv)->getType();
1000 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
1001 delete (*irv);
1002 irv = this->vars.erase(irv);
1003 //irv--;
1004 }
1005 else
1006 ++irv;
1007 }
1008}
1009}
1010#endif
1011
1012void File::Handle_Group_Unsupported_Dtype()
1013{
1014
1015 // First root
1016 if (false == this->root_attrs.empty()) {
1017 if (true == this->unsupported_attr_dtype) {
1018 for (vector<Attribute *>::iterator ira = this->root_attrs.begin(); ira != this->root_attrs.end();) {
1019 H5DataType temp_dtype = (*ira)->getType();
1020 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
1021 delete (*ira);
1022 ira = this->root_attrs.erase(ira);
1023 }
1024 else {
1025 ++ira;
1026 }
1027 }
1028 }
1029 }
1030
1031 // Then the group attributes
1032 if (false == this->groups.empty()) {
1033 for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1034 if (false == (*irg)->attrs.empty()) {
1035 if (true == (*irg)->unsupported_attr_dtype) {
1036 for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end();) {
1037 H5DataType temp_dtype = (*ira)->getType();
1038 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
1039 delete (*ira);
1040 ira = (*irg)->attrs.erase(ira);
1041 }
1042 else {
1043 ++ira;
1044 }
1045 }
1046 }
1047 }
1048 }
1049 }
1050}
1051
1052// Generate group unsupported datatype Information, this is for the BES ignored object key
1053void File::Gen_Group_Unsupported_Dtype_Info()
1054{
1055
1056 // First root
1057 if (false == this->root_attrs.empty()) {
1058 //if (true == this->unsupported_attr_dtype) {
1059 for (vector<Attribute *>::iterator ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira) {
1060 H5DataType temp_dtype = (*ira)->getType();
1061 // TODO: Don't know why we still include 64-bit integer here.
1062 //if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype) || temp_dtype == H5INT64 || temp_dtype == H5UINT64) {
1063 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)
1064 || temp_dtype == H5INT64 || temp_dtype == H5UINT64) {
1065 this->add_ignored_info_attrs(true, "/", (*ira)->name);
1066 }
1067 }
1068 //}
1069 }
1070
1071 // Then the group attributes
1072 if (false == this->groups.empty()) {
1073 for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1074 if (false == (*irg)->attrs.empty()) {
1075 //if (true == (*irg)->unsupported_attr_dtype) {
1076 for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end(); ++ira) {
1077 H5DataType temp_dtype = (*ira)->getType();
1078 // TODO: Don't know why we still include 64-bit integer here.
1079 //if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype) || temp_dtype == H5INT64 || temp_dtype==H5UINT64 ) {
1080 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)
1081 || temp_dtype == H5INT64 || temp_dtype==H5UINT64 ) {
1082 this->add_ignored_info_attrs(true, (*irg)->path, (*ira)->name);
1083 }
1084 }
1085 //}
1086 }
1087 }
1088 }
1089}
1090
1091// Handler unsupported variable datatype
1092void File::Handle_Var_Unsupported_Dtype()
1093{
1094 if (false == this->vars.empty()) {
1095 if (true == this->unsupported_var_dtype) {
1096 for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end();) {
1097 H5DataType temp_dtype = (*irv)->getType();
1098 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
1099 delete (*irv);
1100 irv = this->vars.erase(irv);
1101 }
1102 else {
1103 ++irv;
1104
1105 }
1106 }
1107 }
1108 }
1109}
1110
1111// Generate unsupported variable type info. This is for the ignored objects.
1112void File::Gen_Var_Unsupported_Dtype_Info()
1113{
1114
1115 if (false == this->vars.empty()) {
1116 //if (true == this->unsupported_var_dtype) {
1117 for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1118 H5DataType temp_dtype = (*irv)->getType();
1119 //TODO: don't know why 64-bit integer is still listed here.
1120 //if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)||(H5INT64 == temp_dtype) ||(H5UINT64 == temp_dtype)) {
1121 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)
1122 ||(H5INT64 == temp_dtype) ||(H5UINT64 == temp_dtype)) {
1123 this->add_ignored_info_objs(false, (*irv)->fullpath);
1124 }
1125 }
1126 //}
1127 }
1128
1129}
1130
1131// Handling unsupported datatypes for variable(HDF5 dataset) and the correponding attributes.
1132void File::Handle_VarAttr_Unsupported_Dtype()
1133{
1134 if (false == this->vars.empty()) {
1135 for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1136 if (false == (*irv)->attrs.empty()) {
1137 if (true == (*irv)->unsupported_attr_dtype) {
1138 for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end();) {
1139 H5DataType temp_dtype = (*ira)->getType();
1140 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
1141 delete (*ira);
1142 ira = (*irv)->attrs.erase(ira);
1143 }
1144 else {
1145 ++ira;
1146 }
1147 }
1148 }
1149 }
1150 }
1151 }
1152}
1153
1154// Generated unsupported var/attribute unsupported datatype Info when the BES ignored object key is on.
1155void File::Gen_VarAttr_Unsupported_Dtype_Info()
1156{
1157
1158 if (false == this->vars.empty()) {
1159 for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1160 if (false == (*irv)->attrs.empty()) {
1161 //if (true == (*irv)->unsupported_attr_dtype) {
1162 for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
1163 H5DataType temp_dtype = (*ira)->getType();
1164 // TODO: check why 64-bit integer is still listed here.
1165 //if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype) || (temp_dtype==H5INT64) || (temp_dtype == H5UINT64)) {
1166 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)
1167 || (temp_dtype==H5INT64) || (temp_dtype == H5UINT64)) {
1168 this->add_ignored_info_attrs(false, (*irv)->fullpath, (*ira)->name);
1169 }
1170 }
1171 //}
1172 }
1173 }
1174 }
1175}
1176
1177// Generated unsupported datatype information for HDF5 dimension scales.
1178// The datatypes of HDF5 dimension scales("DIMENSION_LIST" and "REFERENCE_LIST")
1179// are not supported. However, the information
1180// are retrieved by the handlers so we don't want to report them as ignored objects.
1181void File::Gen_DimScale_VarAttr_Unsupported_Dtype_Info()
1182{
1183
1184 for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1185
1186 // If the attribute REFERENCE_LIST comes with the attribut CLASS, the
1187 // attribute REFERENCE_LIST is okay to ignore. No need to report.
1188 bool is_ignored = ignored_dimscale_ref_list((*irv));
1189 if (false == (*irv)->attrs.empty()) {
1190 //if (true == (*irv)->unsupported_attr_dtype) {
1191 for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
1192 H5DataType temp_dtype = (*ira)->getType();
1193 // TODO: check why 64-bit is still listed here.
1194 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)
1195 || (temp_dtype == H5INT64) || (temp_dtype == H5UINT64)) {
1196 // "DIMENSION_LIST" is okay to ignore and "REFERENCE_LIST"
1197 // is okay to ignore if the variable has another attribute
1198 // CLASS="DIMENSION_SCALE"
1199 if (("DIMENSION_LIST" != (*ira)->name)
1200 && ("REFERENCE_LIST" != (*ira)->name || true == is_ignored))
1201 this->add_ignored_info_attrs(false, (*irv)->fullpath, (*ira)->name);
1202 }
1203 }
1204 //}
1205 }
1206 }
1207}
1208
1209// Handle unsupported dataspace for group attributes.
1210void File::Handle_GroupAttr_Unsupported_Dspace()
1211{
1212
1213 // First root
1214 if (false == this->root_attrs.empty()) {
1215 if (true == this->unsupported_attr_dspace) {
1216 for (vector<Attribute *>::iterator ira = this->root_attrs.begin(); ira != this->root_attrs.end();) {
1217 // Remove 0-size attribute
1218 if ((*ira)->count == 0) {
1219 delete (*ira);
1220 ira = this->root_attrs.erase(ira);
1221 }
1222 else {
1223 ++ira;
1224 }
1225 }
1226 }
1227 }
1228
1229 // Then the group attributes
1230 if (false == this->groups.empty()) {
1231 for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1232 if (false == (*irg)->attrs.empty()) {
1233 if (true == (*irg)->unsupported_attr_dspace) {
1234 for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end();) {
1235 if ((*ira)->count == 0) {
1236 delete (*ira);
1237 ira = (*irg)->attrs.erase(ira);
1238 }
1239 else {
1240 ++ira;
1241 }
1242 }
1243 }
1244 }
1245 }
1246 }
1247}
1248
1249// Handle unsupported data space information for variable and attribute
1250void File::Handle_VarAttr_Unsupported_Dspace()
1251{
1252
1253 if (false == this->vars.empty()) {
1254 if (true == this->unsupported_var_attr_dspace) {
1255 for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1256 if (false == (*irv)->attrs.empty()) {
1257 if (true == (*irv)->unsupported_attr_dspace) {
1258 for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end();) {
1259 if (0 == (*ira)->count) {
1260 delete (*ira);
1261 ira = (*irv)->attrs.erase(ira);
1262 }
1263 else {
1264 ++ira;
1265 }
1266 }
1267 }
1268 }
1269 }
1270 }
1271 }
1272}
1273
1274// Handle unsupported data space.
1275void File::Handle_Unsupported_Dspace(bool include_attr)
1276{
1277
1278 // The unsupported data space
1279 if (false == this->vars.empty()) {
1280 if (true == this->unsupported_var_dspace) {
1281 for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end();) {
1282 if (true == (*irv)->unsupported_dspace) {
1283 delete (*irv);
1284 irv = this->vars.erase(irv);
1285 }
1286 else {
1287 ++irv;
1288
1289 }
1290 }
1291 }
1292 }
1293
1294 if (true == include_attr) {
1295 Handle_GroupAttr_Unsupported_Dspace();
1296 Handle_VarAttr_Unsupported_Dspace();
1297 }
1298}
1299
1300// Generated unsupported dataspace Info when the BES ignored object key is on.
1301void File::Gen_Unsupported_Dspace_Info()
1302{
1303
1304 // Notice in this function, we deliberately don't put the case when an attribute dimension has 0 length.
1305 // Since doing this requires non-trivial change of the source code and the 0-size attribute case is really, really rare,
1306 // so we just "ignore" this case in the "ignored" information.
1307 // In fact, the zero size variable is allowed in both HDF5 and DAP2. So we don't ignore 0-size HDF5 dataset. So
1308 // the only case this function checks is the H5S_NULL case.
1309 if (false == this->vars.empty()) {
1310 if (true == this->unsupported_var_dspace) {
1311 for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1312 if (true == (*irv)->unsupported_dspace) {
1313 this->add_ignored_info_objs(true, (*irv)->fullpath);
1314 }
1315 }
1316 }
1317 }
1318
1319}
1320
1321// Handle other unsupported information.
1322void File::Handle_Unsupported_Others(bool include_attr)
1323{
1324
1325 if (true == this->check_ignored && true == include_attr) {
1326
1327 if (true == HDF5RequestHandler::get_drop_long_string()) {
1328
1329 // netCDF java doesn't have limitation for attributes
1330#if 0
1331 for (vector<Attribute *>::iterator ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira) {
1332 if (H5FSTRING == (*ira)->dtype || H5VSTRING == (*ira)->dtype) {
1333 if ((*ira)->getBufSize() > NC_JAVA_STR_SIZE_LIMIT) {
1334 this->add_ignored_droplongstr_hdr();
1335 this->add_ignored_grp_longstr_info("/", (*ira)->name);
1336 }
1337 }
1338 }
1339
1340 for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1341 for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end(); ++ira) {
1342 if (H5FSTRING == (*ira)->dtype || H5VSTRING == (*ira)->dtype) {
1343 if ((*ira)->getBufSize() > NC_JAVA_STR_SIZE_LIMIT) {
1344 this->add_ignored_droplongstr_hdr();
1345 this->add_ignored_grp_longstr_info((*irg)->path, (*ira)->name);
1346 }
1347 }
1348 }
1349 }
1350#endif
1351 for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1352 if (true == Check_DropLongStr((*irv), NULL)) {
1353 this->add_ignored_droplongstr_hdr();
1354 this->add_ignored_var_longstr_info((*irv), NULL);
1355 }
1356 // netCDF java doesn't have limitation for attributes
1357#if 0
1358 for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
1359 if (true == Check_DropLongStr((*irv), (*ira))) {
1360 this->add_ignored_droplongstr_hdr();
1361 this->add_ignored_var_longstr_info((*irv), (*ira));
1362 }
1363 }
1364#endif
1365 }
1366 }
1367 }
1368
1369}
1370
1371// Flatten the object name, mainly call get_CF_string.
1372void File::Flatten_Obj_Name(bool include_attr)
1373{
1374
1375 for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1376 (*irv)->newname = get_CF_string((*irv)->newname);
1377
1378 for (vector<Dimension *>::iterator ird = (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird) {
1379 (*ird)->newname = get_CF_string((*ird)->newname);
1380 }
1381 }
1382
1383 if (true == include_attr) {
1384
1385 for (vector<Attribute *>::iterator ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira) {
1386 (*ira)->newname = get_CF_string((*ira)->newname);
1387 }
1388
1389 for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1390 (*irg)->newname = get_CF_string((*irg)->newname);
1391 for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end(); ++ira) {
1392 (*ira)->newname = get_CF_string((*ira)->newname);
1393 }
1394 }
1395
1396 for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1397 for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
1398 (*ira)->newname = get_CF_string((*ira)->newname);
1399 }
1400 }
1401 } // "if (true == include_attr)"
1402}
1403
1404// Variable name clashing
1405void File::Handle_Var_NameClashing(set<string>&objnameset)
1406{
1407
1408 Handle_General_NameClashing(objnameset, this->vars);
1409}
1410
1411// Group name clashing
1412void File::Handle_Group_NameClashing(set<string> &objnameset)
1413{
1414
1415 pair<set<string>::iterator, bool> setret;
1416
1417 // Now for DAS, we need to handle name clashings for
1418 // DAS tables. Namely we need to make sure the global attribute
1419 // table(HDF5_GLOBAL) and the attribute tables mapped from
1420 // HDF5 groups will not have name clashings with the variable name
1421 // lists. If having the name clashings, the global attribute table and the
1422 // the attribute tables generated from the groups will be changed.
1423 // The file attribute name clashing
1424
1425 setret = objnameset.insert(FILE_ATTR_TABLE_NAME);
1426 if (false == setret.second) {
1427
1428 int clash_index = 1;
1429 string fa_clash_name = FILE_ATTR_TABLE_NAME;
1430 HDF5CFUtil::gen_unique_name(fa_clash_name, objnameset, clash_index);
1431 FILE_ATTR_TABLE_NAME = fa_clash_name;
1432 }
1433
1434 // The group attribute name clashing
1435 Handle_General_NameClashing(objnameset, this->groups);
1436
1437}
1438
1439//Object attribute name clashing
1440void File::Handle_Obj_AttrNameClashing()
1441{
1442
1443 // Now handling the possible name clashings for attributes
1444 // For attribute clashings, we only need to resolve the name clashings
1445 // for attributes within each variable, file attributes and attributes
1446 // within each group. The name clashings for attributes should be very rare.
1447 // Potentially the checking and the correcting may be costly.
1448 // This is another reason for special products, we may not even need to check
1449 // the name clashings. KY 2011-12-24
1450
1451 set<string> objnameset;
1452
1453 // For root attributes
1454 Handle_General_NameClashing(objnameset, this->root_attrs);
1455
1456 // For group attributes
1457 for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1458 objnameset.clear();
1459 Handle_General_NameClashing(objnameset, (*irg)->attrs);
1460 }
1461
1462 // For variable attributes
1463 for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1464 objnameset.clear();
1465 Handle_General_NameClashing(objnameset, (*irv)->attrs);
1466 }
1467}
1468
1469// Handle General name clashing
1470//class T must have member string newname. In our case, T is either groups, attributes or vars.
1471template<class T> void File::Handle_General_NameClashing(set<string>&objnameset, vector<T*>& objvec)
1472{
1473
1474// set<string> objnameset;
1475 pair<set<string>::iterator, bool> setret;
1476 set<string>::iterator iss;
1477
1478 vector<string> clashnamelist;
1479 vector<string>::iterator ivs;
1480
1481 map<int, int> cl_to_ol;
1482 int ol_index = 0;
1483 int cl_index = 0;
1484
1485 /*class*/
1486 typename vector<T*>::iterator irv;
1487
1488 for (irv = objvec.begin(); irv != objvec.end(); ++irv) {
1489 setret = objnameset.insert((*irv)->newname);
1490 if (false == setret.second) {
1491 clashnamelist.insert(clashnamelist.end(), (*irv)->newname);
1492 cl_to_ol[cl_index] = ol_index;
1493 cl_index++;
1494 }
1495 ol_index++;
1496 }
1497
1498 // Now change the clashed elements to unique elements,
1499 // Generate the set which has the same size as the original vector.
1500 for (ivs = clashnamelist.begin(); ivs != clashnamelist.end(); ivs++) {
1501 int clash_index = 1;
1502 string temp_clashname = *ivs + '_';
1503 HDF5CFUtil::gen_unique_name(temp_clashname, objnameset, clash_index);
1504 *ivs = temp_clashname;
1505 }
1506
1507 // Now go back to the original vector, make it unique.
1508 for (unsigned int i = 0; i < clashnamelist.size(); i++)
1509 objvec[cl_to_ol[i]]->newname = clashnamelist[i];
1510
1511}
1512
1513// Handle General object name clashing
1514void File::Handle_GeneralObj_NameClashing(bool include_attr, set<string>& objnameset)
1515{
1516
1517 Handle_Var_NameClashing(objnameset);
1518 if (true == include_attr) {
1519 Handle_Group_NameClashing(objnameset);
1520 Handle_Obj_AttrNameClashing();
1521 }
1522}
1523
1524// Get CF name, flatten the path, change the non-alphanumeric letters to underscore.
1525string File::get_CF_string(string s)
1526{
1527
1528 if ("" == s) return s;
1529 string insertString(1, '_');
1530
1531 // Always start with _ if the first character is not a letter
1532 if (true == isdigit(s[0])) s.insert(0, insertString);
1533
1534 for (unsigned int i = 0; i < s.length(); i++)
1535 if ((false == isalnum(s[i])) && (s[i] != '_')) s[i] = '_';
1536
1537 return s;
1538
1539}
1540
1541// For the connection of variable dimensions. Build dimname to dimsize(unlimited) maps
1542void File::Insert_One_NameSizeMap_Element(string name, hsize_t size, bool unlimited)
1543{
1544 pair<map<string, hsize_t>::iterator, bool> mapret;
1545 mapret = dimname_to_dimsize.insert(pair<string, hsize_t>(name, size));
1546 if (false == mapret.second)
1547 throw4("The dimension name ", name, " should map to ", size);
1548
1549 pair<map<string, bool>::iterator, bool> mapret2;
1550 mapret2 = dimname_to_unlimited.insert(pair<string, bool>(name, unlimited));
1551 if (false == mapret2.second)
1552 throw3("The dimension name ", name, " unlimited dimension info. should be provided.");
1553
1554}
1555
1556// Similar to Inset_One_NameSizeMap_Element but the maps are provided as parameters.
1557void File::Insert_One_NameSizeMap_Element2(map<string, hsize_t>& name_to_size, map<string, bool>& name_to_unlimited,
1558 string name, hsize_t size, bool unlimited)
1559{
1560 pair<map<string, hsize_t>::iterator, bool> mapret;
1561 mapret = name_to_size.insert(pair<string, hsize_t>(name, size));
1562 if (false == mapret.second)
1563 throw4("The dimension name ", name, " should map to ", size);
1564
1565 pair<map<string, bool>::iterator, bool> mapret2;
1566 mapret2 = name_to_unlimited.insert(pair<string, bool>(name, unlimited));
1567 if (false == mapret2.second)
1568 throw3("The dimension name ", name, " unlimited dimension info. should be provided.");
1569
1570}
1571
1572// For dimension names added by the handlers, by default,
1573// Each dimension will have a unique dimension name. For example,
1574// Int foo[100][200] will be Int foo[Fakedim1][Fakedim2]
1575// If you have many variables, the dimension names may be too many.
1576// To reduce numbers, we ASSUME that the dimension having the same
1577// size shares the same dimension. In this way, the number of dimension names
1578// will be reduced.
1579// For example, Int foo2[100][300] will be Int foo2[Fakedim1][Fakedim3]
1580// instead of foo2[Fakedim3][Fakedim4]. However, that may impose
1581// another problem. Suppose Int Foosame[100][100] becomes
1582// Int Foosame[FakeDim1][FakeDim1]. This doesn't make sense for some
1583// applications. The fuction Adjust_Duplicate_FakeDim_Name will make sure
1584// this case will not happen.
1585void File::Add_One_FakeDim_Name(Dimension *dim)
1586{
1587
1588 stringstream sfakedimindex;
1589 string fakedimstr = "FakeDim";
1590 pair<set<string>::iterator, bool> setret;
1591 map<hsize_t, string>::iterator im;
1592 pair<map<hsize_t, string>::iterator, bool> mapret;
1593
1594 sfakedimindex << addeddimindex;
1595 string added_dimname = fakedimstr + sfakedimindex.str();
1596
1597 // Build up the size to fakedim map.
1598 mapret = dimsize_to_fakedimname.insert(pair<hsize_t, string>(dim->size, added_dimname));
1599 if (false == mapret.second) { //The dim size exists, use the corresponding name.
1600 dim->name = dimsize_to_fakedimname[dim->size];
1601 dim->newname = dim->name;
1602 }
1603 else { // Insert this (dimsize,dimname) pair to dimsize_to_fakedimname map successfully.
1604 //First make sure this new dim name doesn't have name clashing
1605 // with previous dim names, after the checking, inserting to the
1606 // dimname list set.
1607 // dimnamelist is a private memeber of File.
1608 setret = dimnamelist.insert(added_dimname);
1609 if (false == setret.second) {
1610 int clash_index = 1;
1611 string temp_clashname = added_dimname + '_';
1612 HDF5CFUtil::gen_unique_name(temp_clashname, dimnamelist, clash_index);
1613 dim->name = temp_clashname;
1614 dim->newname = dim->name;
1615 setret = dimnamelist.insert(dim->name);
1616 if (false == setret.second)
1617 throw2("Fail to insert the unique dimsizede name ", dim->name);
1618
1619 // We have to adjust the dim. name of the dimsize_to_fakedimname map, since the
1620 // dimname has been updated for this size.
1621 dimsize_to_fakedimname.erase(dim->size);
1622 mapret = dimsize_to_fakedimname.insert(pair<hsize_t, string>(dim->size, dim->name));
1623 if (false == mapret.second)
1624 throw4("The dimension size ", dim->size, " should map to ", dim->name);
1625 } // "if(false == setret.second)"
1626
1627 // New dim name is inserted successfully, update the dimname_to_dimsize map.
1628 dim->name = added_dimname;
1629 dim->newname = dim->name;
1630 Insert_One_NameSizeMap_Element(dim->name, dim->size, dim->unlimited_dim);
1631
1632 // Increase the dimindex since the new dimname has been inserted.
1633 addeddimindex++;
1634 } // else
1635}
1636
1637// See the function comments of Add_One_FakeDim_Name
1638void File::Adjust_Duplicate_FakeDim_Name(Dimension * dim)
1639{
1640
1641 // No need to adjust the dimsize_to_fakedimname map, only create a new Fakedim
1642 // The simplest way is to increase the dim index and resolve any name clashings with other dim names.
1643 // Note: No need to update the dimsize_to_dimname map since the original "FakeDim??" of this size
1644 // can be used as a dimension name of other variables. But we need to update the dimname_to_dimsize map
1645 // since this is a new dim name.
1646 stringstream sfakedimindex;
1647 pair<set<string>::iterator, bool> setret;
1648
1649 addeddimindex++;
1650 sfakedimindex << addeddimindex;
1651 string added_dimname = "FakeDim" + sfakedimindex.str();
1652 setret = dimnamelist.insert(added_dimname);
1653 if (false == setret.second) {
1654 int clash_index = 1;
1655 string temp_clashname = added_dimname + '_';
1656 HDF5CFUtil::gen_unique_name(temp_clashname, dimnamelist, clash_index);
1657 dim->name = temp_clashname;
1658 dim->newname = dim->name;
1659 setret = dimnamelist.insert(dim->name);
1660 if (false == setret.second)
1661 throw2("Fail to insert the unique dimsizede name ", dim->name);
1662 }
1663 dim->name = added_dimname;
1664 dim->newname = dim->name;
1665 Insert_One_NameSizeMap_Element(dim->name, dim->size, dim->unlimited_dim);
1666
1667 // Need to prepare for the next unique FakeDim.
1668 addeddimindex++;
1669}
1670
1671// Replace all dimension names, this function is currently not used. So comment out. May delete it in the future.
1672#if 0
1673void File::Replace_Dim_Name_All(const string orig_dim_name, const string new_dim_name) {
1674
1675 // The newname of the original dimension should also be replaced by new_dim_name
1676 for (vector<Var *>::iterator irv = this->vars.begin();
1677 irv != this->vars.end(); ++irv) {
1678 for (vector<Dimension *>::iterator ird= (*irv)->dims.begin();
1679 ird != (*irv)->dims.end(); ++ird) {
1680 if((*ird)->name == orig_dim_name) {
1681 (*ird)->name = new_dim_name;
1682 (*ird)->newname = new_dim_name;
1683 }
1684
1685 }
1686 }
1687}
1688#endif
1689
1690#if 0
1691void File::Use_Dim_Name_With_Size_All(const string dim_name, const size_t dim_size) {
1692
1693 // The newname of the original dimension should also be replaced by new_dim_name
1694 for (vector<Var *>::iterator irv = this->vars.begin();
1695 irv != this->vars.end(); ++irv) {
1696 for (vector<Dimension *>::iterator ird= (*irv)->dims.begin();
1697 ird != (*irv)->dims.end(); ++ird) {
1698 if((*ird)->size == orig_dim_name) {
1699 (*ird)->name = new_dim_name;
1700 (*ird)->newname = new_dim_name;
1701 }
1702
1703 }
1704 }
1705}
1706#endif
1707
1708// Often times we need to add a CF attribute with string datatype because some products don't provide them
1709// Examples are units, comment etc.
1710void File::Add_Str_Attr(Attribute* attr, const string &attrname, const string& strvalue)
1711{
1712
1713 attr->name = attrname;
1714 attr->newname = attr->name;
1715 attr->dtype = H5FSTRING;
1716 attr->count = 1;
1717 attr->fstrsize = strvalue.size();
1718 attr->strsize.resize(1);
1719 attr->strsize[0] = attr->fstrsize;
1720 attr->value.resize(strvalue.size());
1721 copy(strvalue.begin(), strvalue.end(), attr->value.begin());
1722}
1723
1724#if 0
1725bool
1726File:: Var_Has_Attr(Var*var,const string &attrname) {
1727
1728 for (vector<Attribute *>:: iterator ira =var->attrs.begin(); ira !=var->attrs.end(); ++ira) {
1729
1730 // We only check the original attribute name
1731 // Remove the original "coordinates" attribute.
1732 if((*ira)->name == attrname || (*ira)->newname == attrname) {
1733 return true;
1734 }
1735 }
1736 return false;
1737}
1738#endif
1739
1740// Rretrieve the variable attribute in string.var_path is the variable path.
1741string File::Retrieve_Str_Attr_Value(Attribute *attr, const string var_path)
1742{
1743
1744 if (attr != NULL && var_path != "") {
1745 Retrieve_H5_Attr_Value(attr, var_path);
1746 string orig_attr_value(attr->value.begin(), attr->value.end());
1747 return orig_attr_value;
1748 }
1749 return "";
1750
1751}
1752
1753//Check if the attribute value of this variable is the input value.
1754bool File::Is_Str_Attr(Attribute* attr, string varfullpath, const string &attrname, const string& strvalue)
1755{
1756 bool ret_value = false;
1757 if (attrname == get_CF_string(attr->newname)) {
1758 Retrieve_H5_Attr_Value(attr, varfullpath);
1759 string attr_value(attr->value.begin(), attr->value.end());
1760 if (attr_value == strvalue) ret_value = true;
1761 }
1762 return ret_value;
1763}
1764
1765// If this latitude or longitude units follows the CF
1766bool File::has_latlon_cf_units(Attribute *attr, const string &varfullpath, bool is_lat)
1767{
1768 string attr_name = "units";
1769 if (true == is_lat) {
1770 string lat_unit_value = "degrees_north";
1771 return Is_Str_Attr(attr, varfullpath, attr_name, lat_unit_value);
1772 }
1773 else {
1774 string lon_unit_value = "degrees_east";
1775 return Is_Str_Attr(attr, varfullpath, attr_name, lon_unit_value);
1776 }
1777}
1778
1779// This function is mainly to add _FillValue.
1780void File::Add_One_Float_Attr(Attribute* attr, const string &attrname, float float_value)
1781{
1782 attr->name = attrname;
1783 attr->newname = attr->name;
1784 attr->dtype = H5FLOAT32;
1785 attr->count = 1;
1786 attr->value.resize(sizeof(float));
1787 memcpy(&(attr->value[0]), (void*) (&float_value), sizeof(float));
1788}
1789
1790// Products like GPM use string type for MissingValue, we need to change them to the corresponding variable datatype and
1791// get the value corrected.
1792void File::Change_Attr_One_Str_to_Others(Attribute* attr, Var*var)
1793{
1794
1795 char *pEnd;
1796 // string to long int number.
1797 long int num_sli = 0;
1798 if (attr->dtype != H5FSTRING)
1799 throw2("Currently we only convert fixed-size string to other datatypes. ", attr->name);
1800 if (attr->count != 1)
1801 throw4("The fixed-size string count must be 1 and the current count is ", attr->count, " for the attribute ",
1802 attr->name);
1803
1804 Retrieve_H5_Attr_Value(attr, var->fullpath);
1805 string attr_value;
1806 attr_value.resize(attr->value.size());
1807 copy(attr->value.begin(), attr->value.end(), attr_value.begin());
1808
1809 switch (var->dtype) {
1810
1811 case H5UCHAR: {
1812 num_sli = strtol(&(attr->value[0]), &pEnd, 10);
1813 if (num_sli < 0 || num_sli > UCHAR_MAX)
1814 throw5("Attribute type is unsigned char, the current attribute ", attr->name, " has the value ", num_sli,
1815 ". It is overflowed. ");
1816 else {
1817 unsigned char num_suc = (unsigned char) num_sli;
1818 attr->dtype = H5UCHAR;
1819 attr->value.resize(sizeof(unsigned char));
1820 memcpy(&(attr->value[0]), (void*) (&num_suc), sizeof(unsigned char));
1821 }
1822
1823 }
1824 break;
1825 case H5CHAR: {
1826 num_sli = strtol(&(attr->value[0]), &pEnd, 10);
1827 if (num_sli < SCHAR_MIN || num_sli > SCHAR_MAX)
1828 throw5("Attribute type is signed char, the current attribute ", attr->name, " has the value ", num_sli,
1829 ". It is overflowed. ");
1830 else {
1831 char num_sc = (char) num_sli;
1832 attr->dtype = H5CHAR;
1833 attr->value.resize(sizeof(char));
1834 memcpy(&(attr->value[0]), (void*) (&num_sc), sizeof(char));
1835 }
1836
1837 }
1838 break;
1839 case H5INT16: {
1840 num_sli = strtol(&(attr->value[0]), &pEnd, 10);
1841 if (num_sli < SHRT_MIN || num_sli > SHRT_MAX)
1842 throw5("Attribute type is 16-bit integer, the current attribute ", attr->name, " has the value ", num_sli,
1843 ". It is overflowed. ");
1844 else {
1845 short num_ss = (short) num_sli;
1846 attr->dtype = H5INT16;
1847 attr->value.resize(sizeof(short));
1848 memcpy(&(attr->value[0]), (void*) (&num_ss), sizeof(short));
1849 }
1850
1851 }
1852 break;
1853 case H5UINT16: {
1854 num_sli = strtol(&(attr->value[0]), &pEnd, 10);
1855 if (num_sli < 0 || num_sli > USHRT_MAX)
1856 throw5("Attribute type is unsigned 16-bit integer, the current attribute ", attr->name, " has the value ",
1857 num_sli, ". It is overflowed. ");
1858 else {
1859 unsigned short num_uss = (unsigned short) num_sli;
1860 attr->dtype = H5UINT16;
1861 attr->value.resize(sizeof(unsigned short));
1862 memcpy(&(attr->value[0]), (void*) (&num_uss), sizeof(unsigned short));
1863 }
1864 }
1865 break;
1866 case H5INT32: {
1867 num_sli = strtol(&(attr->value[0]), &pEnd, 10);
1868 //No need to check overflow, the number will always be in the range.
1869#if 0
1870 //if(num_sli <LONG_MIN || num_sli >LONG_MAX)
1871 // throw5("Attribute type is 32-bit integer, the current attribute ",attr->name, " has the value ",num_sli, ". It is overflowed. ");
1872#endif
1873 attr->dtype = H5INT32;
1874 attr->value.resize(sizeof(long int));
1875 memcpy(&(attr->value[0]), (void*) (&num_sli), sizeof(long int));
1876
1877 }
1878 break;
1879 case H5UINT32: {
1880 unsigned long int num_suli = strtoul(&(attr->value[0]), &pEnd, 10);
1881 // No need to check since num_suli will not be bigger than ULONG_MAX.
1882 attr->dtype = H5UINT32;
1883 attr->value.resize(sizeof(unsigned long int));
1884 memcpy(&(attr->value[0]), (void*) (&num_suli), sizeof(unsigned long int));
1885 }
1886 break;
1887 case H5FLOAT32: {
1888 float num_sf = strtof(&(attr->value[0]), NULL);
1889 // Don't think it is necessary to check if floating-point is oveflowed for this routine. ignore it now. KY 2014-09-22
1890 attr->dtype = H5FLOAT32;
1891 attr->value.resize(sizeof(float));
1892 memcpy(&(attr->value[0]), (void*) (&num_sf), sizeof(float));
1893 }
1894 break;
1895 case H5FLOAT64: {
1896 double num_sd = strtod(&(attr->value[0]), NULL);
1897 // Don't think it is necessary to check if floating-point is oveflowed for this routine. ignore it now. KY 2014-09-22
1898 attr->dtype = H5FLOAT64;
1899 attr->value.resize(sizeof(double));
1900 memcpy(&(attr->value[0]), (void*) (&num_sd), sizeof(double));
1901 }
1902 break;
1903
1904 default:
1905 throw4("Unsupported HDF5 datatype that the string is converted to for the attribute ", attr->name,
1906 " of the variable ", var->fullpath);
1907 } // "switch(var->dtype)"
1908
1909}
1910
1911// Change a string type attribute to the input value
1912void File::Replace_Var_Str_Attr(Var* var, const string &attr_name, const string& strvalue)
1913{
1914
1915 bool rep_attr = true;
1916 bool rem_attr = false;
1917 for (vector<Attribute *>::iterator ira = var->attrs.begin(); ira != var->attrs.end(); ira++) {
1918 if ((*ira)->name == attr_name) {
1919 if (true == Is_Str_Attr(*ira, var->fullpath, attr_name, strvalue))
1920 rep_attr = false;
1921 else
1922 rem_attr = true;
1923 break;
1924 }
1925 }
1926
1927 // Remove the attribute if the attribute value is not strvalue
1928 if (true == rem_attr) {
1929 for (vector<Attribute *>::iterator ira = var->attrs.begin(); ira != var->attrs.end(); ira++) {
1930 if ((*ira)->name == attr_name) {
1931 delete (*ira);
1932 var->attrs.erase(ira);
1933 break;
1934 }
1935 }
1936 }
1937
1938 // Add the attribute with strvalue
1939 if (true == rep_attr) {
1940 Attribute * attr = new Attribute();
1941 Add_Str_Attr(attr, attr_name, strvalue);
1942 var->attrs.push_back(attr);
1943 }
1944}
1945
1946// Check if this variable if latitude,longitude.We check the three name pairs(lat,lon),(latitude,longitude),(Latitude,Longitude)
1947bool File::Is_geolatlon(const string & var_name, bool is_lat)
1948{
1949
1950 bool ret_value = false;
1951 if (true == is_lat) {
1952 string lat1 = "lat";
1953 string lat2 = "latitude";
1954 string lat3 = "Latitude";
1955
1956 if (var_name.compare(lat1) == 0 || var_name.compare(lat2) == 0 || var_name.compare(lat3) == 0) ret_value = true;
1957 }
1958
1959 else {
1960 string lon1 = "lon";
1961 string lon2 = "longitude";
1962 string lon3 = "Longitude";
1963 if (var_name.compare(lon1) == 0 || var_name.compare(lon2) == 0 || var_name.compare(lon3) == 0) ret_value = true;
1964
1965 }
1966 return ret_value;
1967}
1968
1969// Add supplementary attributes.
1970void File::Add_Supplement_Attrs(bool add_path)
1971{
1972
1973 if (false == add_path) return;
1974
1975 // Adding variable original name(origname) and full path(fullpath)
1976 for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1977 Attribute * attr = new Attribute();
1978 const string varname = (*irv)->name;
1979 const string attrname = "origname";
1980 Add_Str_Attr(attr, attrname, varname);
1981 (*irv)->attrs.push_back(attr);
1982 }
1983
1984 for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1985 // Turn off the fullnamepath attribute when zero_storage_size is 0.
1986 // Use the BES key since quite a few testing cases will be affected.
1987 // KY 2020-03-23
1988 if((*irv)->zero_storage_size==false
1989 || HDF5RequestHandler::get_no_zero_size_fullnameattr() == false) {
1990 Attribute * attr = new Attribute();
1991 const string varname = (*irv)->fullpath;
1992 const string attrname = "fullnamepath";
1993 Add_Str_Attr(attr, attrname, varname);
1994 (*irv)->attrs.push_back(attr);
1995 }
1996 }
1997
1998 // Adding group path
1999 for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
2000 // Only when this group has attributes, the original path of the group has some values. So add it.
2001 if (false == (*irg)->attrs.empty()) {
2002
2003 Attribute * attr = new Attribute();
2004 const string varname = (*irg)->path;
2005 const string attrname = "fullnamepath";
2006 Add_Str_Attr(attr, attrname, varname);
2007 (*irg)->attrs.push_back(attr);
2008 }
2009 }
2010
2011}
2012
2013// Variable target will not be deleted, but rather its contents are replaced.
2014// We may make this as an operator = in the future.
2015// Note: the attributes can not be replaced.
2016void File::Replace_Var_Info(Var *src, Var *target)
2017{
2018
2019#if 0
2020 for_each (target->dims.begin (), target->dims.end (),
2021 delete_elem ());
2022 for_each (target->attrs.begin (), target->attrs.end (),
2023 delete_elem ());
2024#endif
2025
2026 target->newname = src->newname;
2027 target->name = src->name;
2028 target->fullpath = src->fullpath;
2029 target->rank = src->rank;
2030 target->dtype = src->dtype;
2031 target->unsupported_attr_dtype = src->unsupported_attr_dtype;
2032 target->unsupported_dspace = src->unsupported_dspace;
2033#if 0
2034 for (vector<Attribute*>::iterator ira = target->attrs.begin();
2035 ira!=target->attrs.end(); ++ira) {
2036 delete (*ira);
2037 target->attrs.erase(ira);
2038 ira--;
2039 }
2040#endif
2041 for (vector<Dimension*>::iterator ird = target->dims.begin(); ird != target->dims.end();) {
2042 delete (*ird);
2043 ird = target->dims.erase(ird);
2044 }
2045
2046 // Somehow attributes cannot be replaced.
2047#if 0
2048 for (vector<Attribute*>::iterator ira = src->attrs.begin();
2049 ira!=src->attrs.end(); ++ira) {
2050 Attribute* attr= new Attribute();
2051 attr->name = (*ira)->name;
2052 attr->newname = (*ira)->newname;
2053 attr->dtype =(*ira)->dtype;
2054 attr->count =(*ira)->count;
2055 attr->strsize = (*ira)->strsize;
2056 attr->fstrsize = (*ira)->fstrsize;
2057 attr->value =(*ira)->value;
2058 target->attrs.push_back(attr);
2059 }
2060#endif
2061
2062 for (vector<Dimension*>::iterator ird = src->dims.begin(); ird != src->dims.end(); ++ird) {
2063 Dimension *dim = new Dimension((*ird)->size);
2064 dim->name = (*ird)->name;
2065 dim->newname = (*ird)->newname;
2066 target->dims.push_back(dim);
2067 }
2068
2069}
2070
2071// Replace the attributes of target with src.
2072void File::Replace_Var_Attrs(Var *src, Var *target)
2073{
2074
2075#if 0
2076 for_each (target->dims.begin (), target->dims.end (),
2077 delete_elem ());
2078 for_each (target->attrs.begin (), target->attrs.end (),
2079 delete_elem ());
2080#endif
2081
2082 for (vector<Attribute*>::iterator ira = target->attrs.begin(); ira != target->attrs.end();) {
2083 delete (*ira);
2084 ira = target->attrs.erase(ira);
2085 }
2086 for (vector<Attribute*>::iterator ira = src->attrs.begin(); ira != src->attrs.end(); ++ira) {
2087 Attribute* attr = new Attribute();
2088 attr->name = (*ira)->name;
2089 attr->newname = (*ira)->newname;
2090 attr->dtype = (*ira)->dtype;
2091 attr->count = (*ira)->count;
2092 attr->strsize = (*ira)->strsize;
2093 attr->fstrsize = (*ira)->fstrsize;
2094 attr->value = (*ira)->value;
2095 target->attrs.push_back(attr);
2096 }
2097
2098}
2099
2100// Check if a variable with a var name is under a specific group with groupname
2101// note: the variable's size at each dimension is also returned. The user must allocate the
2102// memory for the dimension sizes(an array(vector is perferred).
2103bool File::is_var_under_group(const string &varname, const string &grpname, const int var_rank,
2104 vector<size_t> & var_size)
2105{
2106
2107 bool ret_value = false;
2108 for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2109
2110 if ((*irv)->rank == var_rank) {
2111 if ((*irv)->name == varname) {
2112
2113 // Obtain the variable path
2114 string var_path = HDF5CFUtil::obtain_string_before_lastslash((*irv)->fullpath);
2115
2116 // Check if we find the variable under this group
2117 if (grpname == var_path) {
2118 ret_value = true;
2119 for (int i = 0; i < var_rank; i++)
2120 var_size[i] = (*irv)->getDimensions()[i]->size;
2121 break;
2122 }
2123 }
2124 } // "if((*irv)->rank == var_rank)"
2125 } // "for (vector<Var *>::iterator irv = this->vars.begin()"
2126
2127
2128 return ret_value;
2129
2130}
2132
2133 bool ret_value = false;
2134 for (vector<Var *>::iterator irv = this->vars.begin();
2135 irv != this->vars.end(); ++irv) {
2136 for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin();
2137 ira != (*irv)->attrs.end(); ++ira) {
2138 if((*ira)->name =="grid_mapping") {
2139 ret_value = true;
2140 break;
2141 }
2142 }
2143 if(true == ret_value)
2144 break;
2145 }
2146
2147 return ret_value;
2148
2149}
2150
2152
2153 for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2154 string attr_value;
2155 for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
2156 if((*ira)->name =="grid_mapping") {
2157 Retrieve_H5_Attr_Value(*ira, (*irv)->fullpath);
2158 attr_value.resize((*ira)->value.size());
2159 copy((*ira)->value.begin(), (*ira)->value.end(), attr_value.begin());
2160 break;
2161 }
2162
2163 }
2164 if(attr_value.find('/') ==string::npos){
2165 string new_name = Check_Grid_Mapping_VarName(attr_value,(*irv)->fullpath);
2166 if(new_name != "")
2167 Replace_Var_Str_Attr((*irv),"grid_mapping",new_name);
2168
2169 }
2170 else {
2171 string new_name = Check_Grid_Mapping_FullPath(attr_value);
2172 //Using new_name as the attribute value
2173 if(new_name != "")
2174 Replace_Var_Str_Attr((*irv),"grid_mapping",new_name);
2175 }
2176 }
2177
2178}
2179
2180string File::Check_Grid_Mapping_VarName(const string & a_value,const string & var_fpath) {
2181
2182 string var_path = HDF5CFUtil::obtain_string_before_lastslash(var_fpath);
2183 string gmap_new_name;
2184 for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2185 if((*irv)->name == a_value){
2186 if(var_path == HDF5CFUtil::obtain_string_before_lastslash((*irv)->fullpath)) {
2187 gmap_new_name = (*irv)->newname;
2188 break;
2189 }
2190 }
2191 }
2192 return gmap_new_name;
2193}
2194
2195
2196string File::Check_Grid_Mapping_FullPath(const string & a_value) {
2197
2198 string gmap_new_name;
2199 for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2200 if((*irv)->fullpath == a_value){
2201 gmap_new_name = (*irv)->newname;
2202 break;
2203 }
2204 }
2205
2206 return gmap_new_name;
2207}
2208
2209void File::remove_netCDF_internal_attributes(bool include_attr) {
2210
2211 if(true == include_attr) {
2212 for (vector<Var *>::iterator irv = this->vars.begin();
2213 irv != this->vars.end(); ++irv) {
2214 bool var_has_dimscale = false;
2215
2216 for(vector<Attribute *>::iterator ira = (*irv)->attrs.begin();
2217 ira != (*irv)->attrs.end();) {
2218 if((*ira)->name == "CLASS") {
2219 string class_value = Retrieve_Str_Attr_Value(*ira,(*irv)->fullpath);
2220
2221 // Compare the attribute "CLASS" value with "DIMENSION_SCALE". We only compare the string with the size of
2222 // "DIMENSION_SCALE", which is 15.
2223 if (0 == class_value.compare(0,15,"DIMENSION_SCALE")) {
2224 delete(*ira);
2225 ira = (*irv)->attrs.erase(ira);
2226 var_has_dimscale = true;
2227
2228 }
2229#if 0
2230 else if(1) {// Add a BES key,also delete
2231
2232 }
2233#endif
2234 else {
2235 ++ira;
2236 }
2237 }
2238#if 0
2239 else if((*ira)->name == "NAME") {// Add a BES Key
2240 string name_value = Retrieve_Str_Attr_Value(*ira,(*irv)->fullpath);
2241 if( 0 == name_value.compare(0,(*irv)->name.size(),(*irv)->name)) {
2242 delete(*ira);
2243 ira =(*irv)->attrs.erase(ira);
2244 }
2245 else {
2246 string netcdf_dim_mark= "This is a netCDF dimension but not a netCDF variable";
2247 if( 0 == name_value.compare(0,netcdf_dim_mark.size(),netcdf_dim_mark)) {
2248 delete(*ira);
2249 ira =(*irv)->attrs.erase(ira);
2250 }
2251 else {
2252 ++ira;
2253 }
2254 }
2255
2256 }
2257#endif
2258 else if((*ira)->name == "_Netcdf4Dimid") {
2259 delete(*ira);
2260 ira =(*irv)->attrs.erase(ira);
2261 }
2262 else if((*ira)->name == "_Netcdf4Coordinates") {
2263 delete(*ira);
2264 ira =(*irv)->attrs.erase(ira);
2265 }
2266#if 0
2267 else if((*ira)->name == "_nc3_strict") {
2268 delete((*ira));
2269 ira =(*irv)->attrs.erase(ira);
2270 }
2271#endif
2272 else {
2273 ++ira;
2274 }
2275 }
2276
2277 if(true == var_has_dimscale) {
2278 for(vector<Attribute *>::iterator ira = (*irv)->attrs.begin();
2279 ira != (*irv)->attrs.end();++ira) {
2280 if((*ira)->name == "NAME") {// Add a BES Key
2281 delete(*ira);
2282 ira =(*irv)->attrs.erase(ira);
2283 break;
2284 }
2285 }
2286 }
2287 }
2288 }
2289
2290}
2291// Add ignored page header info. Mainly a helper message.
2292void File::add_ignored_info_page_header()
2293{
2294 ignored_msg =
2295 " \n This page is for HDF5 CF hyrax data providers or distributors to check if any HDF5 object or attribute information are ignored during the mapping. \n\n";
2296}
2297
2298// Add ignored object header info. Mainly a helper message.
2299void File::add_ignored_info_obj_header()
2300{
2301
2302 ignored_msg += " Some HDF5 objects or the object information are ignored when mapping to DAP2 by the HDF5 OPeNDAP";
2303 ignored_msg += " handler due to the restrictions of DAP2, CF conventions or CF tools.";
2304 ignored_msg += " Please use HDF5 tools(h5dump or HDFView) to check carefully and make sure that these objects";
2305 ignored_msg +=
2306 " are OK to ignore for your service. For questions or requests to find a way to handle the ignored objects, please";
2307 ignored_msg += " contact the HDF5 OPeNDAP handler developer or send an email to help@hdfgroup.org.\n";
2308
2309 ignored_msg += " \n In general, ignored HDF5 objects include HDF5 soft links, external links and named datatype.\n";
2310 ignored_msg +=
2311 " \n The HDF5 datasets(variables in the CF term) and attributes that have the following datatypes are ignored: \n";
2312 ignored_msg +=
2313 " Signed and unsigned 64-bit integers, HDF5 compound, HDF5 variable length(excluding variable length string),";
2314 ignored_msg += " HDF5 reference, HDF5 enum, HDF5 opaque , HDF5 bitfield, HDF5 Array and HDF5 Time datatypes.\n";
2315
2316 ignored_msg +=
2317 " \n The HDF5 datasets(variables in the CF term) and attributes associated with the following dimensions are ignored: \n";
2318 ignored_msg += " 1) variables that have HDF5 NULL dataspace(H5S_NULL)(rarely occurred)\n";
2319 ignored_msg += " 2) attributes that have any zero size dimensions(not reported due to extreme rarity and non-trivial coding)\n\n";
2320
2321}
2322
2323// Add the ignored links information.Mainly a helper message.
2324void File::add_ignored_info_links_header()
2325{
2326
2327 if (false == this->have_ignored) {
2328 add_ignored_info_obj_header();
2329 have_ignored = true;
2330 }
2331 // Add ignored datatype header.
2332 string lh_msg = "******WARNING******\n";
2333 lh_msg += "IGNORED soft links or external links are: ";
2334 if (ignored_msg.rfind(lh_msg) == string::npos) ignored_msg += lh_msg + "\n";
2335
2336}
2337
2338// Leave the code for the time being.
2339#if 0
2340void
2341File:: add_ignored_info_obj_dtype_header() {
2342
2343 // Add ignored datatype header.
2344 ignored_msg += " \n Variables and attributes ignored due to the unsupported datatypes. \n";
2345 ignored_msg += " In general, the unsupported datatypes include: \n";
2346 ignored_msg += " Signed and unsigned 64-bit integers, HDF5 compound, HDF5 variable length(excluding variable length string),";
2347 ignored_msg += " HDF5 reference, HDF5 enum, HDF5 opaque , HDF5 bitfield, HDF5 Array and HDF5 Time datatypes.\n";
2348
2349}
2350
2351void
2352File:: add_ignored_info_obj_dspace_header() {
2353
2354 // Add ignored dataspace header.
2355 ignored_msg += " \n Variables and attributes ignored due to the unsupported dimensions. \n";
2356 ignored_msg += " In general, the unsupported dimensions include: \n";
2357 ignored_msg += " 1) variables that have HDF5 NULL dataspace(H5S_NULL)(rarely occurred)\n";
2358 ignored_msg += " 2) variables that have any zero size dimensions\n";
2359
2360}
2361#endif
2362
2363// Add the ignored link info.
2364void File::add_ignored_info_links(const string & link_path)
2365{
2366 if (ignored_msg.find("Link paths: ") == string::npos)
2367 ignored_msg += " Link paths: " + link_path;
2368 else
2369 ignored_msg += " " + link_path;
2370}
2371
2372// Add the ignored name datatype info.
2373void File::add_ignored_info_namedtypes(const string& grp_name, const string& named_dtype_name)
2374{
2375
2376 if (false == this->have_ignored) {
2377 add_ignored_info_obj_header();
2378 have_ignored = true;
2379 }
2380
2381 string ignored_HDF5_named_dtype_hdr = "\n******WARNING******";
2382 ignored_HDF5_named_dtype_hdr += "\n IGNORED HDF5 named datatype objects:\n";
2383 string ignored_HDF5_named_dtype_msg = " Group name: " + grp_name + " HDF5 named datatype name: " + named_dtype_name.substr(0,named_dtype_name.size()-1)
2384 + "\n";
2385 if (ignored_msg.find(ignored_HDF5_named_dtype_hdr) == string::npos)
2386 ignored_msg += ignored_HDF5_named_dtype_hdr + ignored_HDF5_named_dtype_msg;
2387 else
2388 ignored_msg += ignored_HDF5_named_dtype_msg;
2389
2390}
2391
2392// Add the ignored attribute information. When is_grp is true, the ignored group attribute names are added.
2393// Otherwise, the ignored dataset attribute names are added.
2394void File::add_ignored_info_attrs(bool is_grp, const string & obj_path, const string & attr_name)
2395{
2396
2397 if (false == this->have_ignored) {
2398 add_ignored_info_obj_header();
2399 have_ignored = true;
2400 }
2401
2402
2403 string ignored_warning_str = "\n******WARNING******";
2404 string ignored_HDF5_grp_hdr = ignored_warning_str + "\n Ignored attributes under root and groups:\n";
2405 string ignored_HDF5_grp_msg = " Group path: " + obj_path + " Attribute names: " + attr_name + "\n";
2406 string ignored_HDF5_var_hdr = ignored_warning_str + "\n Ignored attributes for variables:\n";
2407 string ignored_HDF5_var_msg = " Variable path: " + obj_path + " Attribute names: " + attr_name + "\n";
2408
2409
2410 if (true == is_grp) {
2411 if (ignored_msg.find(ignored_HDF5_grp_hdr) == string::npos)
2412 ignored_msg += ignored_HDF5_grp_hdr + ignored_HDF5_grp_msg;
2413 else
2414 ignored_msg += ignored_HDF5_grp_msg;
2415 }
2416 else {
2417 if (ignored_msg.find(ignored_HDF5_var_hdr) == string::npos)
2418 ignored_msg += ignored_HDF5_var_hdr + ignored_HDF5_var_msg;
2419 else
2420 ignored_msg += ignored_HDF5_var_msg;
2421 }
2422
2423}
2424
2425//Ignored object information. When is_dim_related is true, ignored data space info. is present.
2426//When is_dim_related is false, ignored data type info. is present.
2427void File::add_ignored_info_objs(bool is_dim_related, const string & obj_path)
2428{
2429
2430 if (false == this->have_ignored) {
2431 add_ignored_info_obj_header();
2432 have_ignored = true;
2433 }
2434
2435 string ignored_warning_str = "\n******WARNING******";
2436 string ignored_HDF5_dtype_var_hdr = ignored_warning_str + "\n IGNORED variables due to unsupported datatypes:\n";
2437 string ignored_HDF5_dspace_var_hdr = ignored_warning_str + "\n IGNORED variables due to unsupported dimensions:\n";
2438 string ignored_HDF5_var_msg = " Variable path: " + obj_path + "\n";
2439
2440 if (true == is_dim_related) {
2441 if (ignored_msg.find(ignored_HDF5_dspace_var_hdr) == string::npos)
2442 ignored_msg += ignored_HDF5_dspace_var_hdr + ignored_HDF5_var_msg;
2443 else
2444 ignored_msg += ignored_HDF5_var_msg;
2445
2446 }
2447 else {
2448 if (ignored_msg.find(ignored_HDF5_dtype_var_hdr) == string::npos)
2449 ignored_msg += ignored_HDF5_dtype_var_hdr + ignored_HDF5_var_msg;
2450 else
2451 ignored_msg += ignored_HDF5_var_msg;
2452 }
2453
2454}
2455
2456// No ignored info.
2457void File::add_no_ignored_info()
2458{
2459
2460 ignored_msg += "There are no ignored HDF5 objects or attributes.";
2461
2462}
2463
2464// This function should only be used when the HDF5 file is following the netCDF data model.
2465// Check if we should not report the Dimension scale related attributes as ignored.
2466bool File::ignored_dimscale_ref_list(Var *var)
2467{
2468
2469 bool ignored_dimscale = true;
2470 // Only when "General_Product == this->product_type && GENERAL_DIMSCALE== this->gproduct_pattern)"
2471
2472 bool has_dimscale = false;
2473 bool has_reference_list = false;
2474 for (vector<Attribute *>::iterator ira = var->attrs.begin(); ira != var->attrs.end(); ira++) {
2475 if ((*ira)->name == "REFERENCE_LIST" && false == HDF5CFUtil::cf_strict_support_type((*ira)->getType(),_is_dap4))
2476 has_reference_list = true;
2477 if ((*ira)->name == "CLASS") {
2478 Retrieve_H5_Attr_Value(*ira, var->fullpath);
2479 string class_value;
2480 class_value.resize((*ira)->value.size());
2481 copy((*ira)->value.begin(), (*ira)->value.end(), class_value.begin());
2482
2483 // Compare the attribute "CLASS" value with "DIMENSION_SCALE". We only compare the string with the size of
2484 // "DIMENSION_SCALE", which is 15.
2485 if (0 == class_value.compare(0, 15, "DIMENSION_SCALE")) {
2486 has_dimscale = true;
2487 }
2488 }
2489
2490 if (true == has_dimscale && true == has_reference_list) {
2491 ignored_dimscale = false;
2492 break;
2493 }
2494
2495 }
2496 //}
2497 return ignored_dimscale;
2498}
2499
2500// Check if the long string can should be dropped from a dataset or an attribute. Users can set up a BES key to turn it off or on.
2501bool File::Check_DropLongStr(Var *var, Attribute * attr)
2502{
2503
2504 bool drop_longstr = false;
2505 if (NULL == attr) {
2506 if (H5FSTRING == var->dtype || H5VSTRING == var->dtype) {
2507 try {
2508 drop_longstr = Check_VarDropLongStr(var->fullpath, var->dims, var->dtype);
2509 }
2510 catch (...) {
2511 throw1("Check_VarDropLongStr fails ");
2512 }
2513 }
2514 }
2515 // No limitation for the attributes. KY 2018-02-26
2516#if 0
2517 else {
2518 if (H5FSTRING == attr->dtype || H5VSTRING == attr->dtype) {
2519 if (attr->getBufSize() > NC_JAVA_STR_SIZE_LIMIT) {
2520 drop_longstr = true;
2521 }
2522 }
2523
2524 }
2525#endif
2526 return drop_longstr;
2527}
2528
2529// Check if a long string dataset should be dropped. Users can turn on a BES key not to drop the long string.
2530// However, the Java clients may not access.
2531//
2532bool File::Check_VarDropLongStr(const string & varpath, const vector<Dimension *>& dims, H5DataType dtype) const
2533
2534{
2535
2536 bool drop_longstr = false;
2537
2538 hid_t dset_id = H5Dopen2(this->fileid, varpath.c_str(), H5P_DEFAULT);
2539 if (dset_id < 0)
2540 throw2("Cannot open the dataset ", varpath);
2541
2542 hid_t dtype_id = -1;
2543 if ((dtype_id = H5Dget_type(dset_id)) < 0) {
2544 H5Dclose(dset_id);
2545 throw2("Cannot obtain the datatype of the dataset ", varpath);
2546 }
2547
2548 size_t ty_size = H5Tget_size(dtype_id);
2549 if (ty_size == 0) {
2550 H5Tclose(dtype_id);
2551 H5Dclose(dset_id);
2552 throw2("Cannot obtain the datatype size of the dataset ", varpath);
2553 }
2554
2555 if (H5FSTRING == dtype) { // Fixed-size, just check the number of elements.
2556 if (ty_size > NC_JAVA_STR_SIZE_LIMIT) drop_longstr = true;
2557 }
2558 else if (H5VSTRING == dtype) {
2559
2560 unsigned long long total_elms = 1;
2561 if (dims.size() != 0) {
2562 for (unsigned int i = 0; i < dims.size(); i++)
2563 total_elms = total_elms * ((dims[i])->size);
2564 }
2565 vector<char> strval;
2566 strval.resize(total_elms * ty_size);
2567 hid_t read_ret = H5Dread(dset_id, dtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, (void*) &strval[0]);
2568 if (read_ret < 0) {
2569 H5Tclose(dtype_id);
2570 H5Dclose(dset_id);
2571 throw2("Cannot read the data of the dataset ", varpath);
2572 }
2573
2574 vector<string> finstrval;
2575 finstrval.resize(total_elms);
2576 char*temp_bp = &strval[0];
2577 char*onestring = NULL;
2578 for (unsigned long long i = 0; i < total_elms; i++) {
2579 onestring = *(char**) temp_bp;
2580 if (onestring != NULL) {
2581 finstrval[i] = string(onestring);
2582 if(finstrval[i].size()>NC_JAVA_STR_SIZE_LIMIT) {
2583 drop_longstr = true;
2584 break;
2585 }
2586 }
2587 temp_bp += ty_size;
2588 }
2589
2590 if (false == strval.empty()) {
2591 herr_t ret_vlen_claim;
2592 hid_t dspace_id = H5Dget_space(dset_id);
2593 if (dspace_id < 0) {
2594 H5Tclose(dtype_id);
2595 H5Dclose(dset_id);
2596 throw2("Cannot obtain the dataspace id.", varpath);
2597 }
2598 ret_vlen_claim = H5Dvlen_reclaim(dtype_id, dspace_id, H5P_DEFAULT, (void*) &strval[0]);
2599 if (ret_vlen_claim < 0) {
2600 H5Tclose(dtype_id);
2601 H5Sclose(dspace_id);
2602 H5Dclose(dset_id);
2603 throw2("Cannot reclaim the vlen space ", varpath);
2604 }
2605 if (H5Sclose(dspace_id) < 0) {
2606 H5Tclose(dtype_id);
2607 H5Dclose(dset_id);
2608 throw2("Cannot close the HDF5 data space.", varpath);
2609 }
2610 }
2611 }
2612 if (H5Tclose(dtype_id) < 0) {
2613 H5Dclose(dset_id);
2614 throw2("Cannot close the HDF5 data type.", varpath);
2615 }
2616 if (H5Dclose(dset_id) < 0)
2617 throw2("Cannot close the HDF5 data type.", varpath);
2618
2619 return drop_longstr;
2620}
2621#if 0
2622bool File::Check_VarDropLongStr(const string & varpath, const vector<Dimension *>& dims, H5DataType dtype)
2623
2624{
2625
2626 bool drop_longstr = false;
2627
2628 unsigned long long total_elms = 1;
2629 if (dims.size() != 0) {
2630 for (unsigned int i = 0; i < dims.size(); i++)
2631 total_elms = total_elms * ((dims[i])->size);
2632 }
2633
2634 if (total_elms > NC_JAVA_STR_SIZE_LIMIT)
2635 drop_longstr = true;
2636
2637 else { // We need to check both fixed-size and variable-length strings.
2638
2639 hid_t dset_id = H5Dopen2(this->fileid, varpath.c_str(), H5P_DEFAULT);
2640 if (dset_id < 0)
2641 throw2("Cannot open the dataset ", varpath);
2642
2643 hid_t dtype_id = -1;
2644 if ((dtype_id = H5Dget_type(dset_id)) < 0) {
2645 H5Dclose(dset_id);
2646 throw2("Cannot obtain the datatype of the dataset ", varpath);
2647 }
2648
2649 size_t ty_size = H5Tget_size(dtype_id);
2650 if (ty_size == 0) {
2651 H5Tclose(dtype_id);
2652 H5Dclose(dset_id);
2653 throw2("Cannot obtain the datatype size of the dataset ", varpath);
2654 }
2655
2656 if (H5FSTRING == dtype) { // Fixed-size, just check the number of elements.
2657 if ((ty_size * total_elms) > NC_JAVA_STR_SIZE_LIMIT) drop_longstr = true;
2658 }
2659 else if (H5VSTRING == dtype) {
2660
2661 vector<char> strval;
2662 strval.resize(total_elms * ty_size);
2663 hid_t read_ret = H5Dread(dset_id, dtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, (void*) &strval[0]);
2664 if (read_ret < 0) {
2665 H5Tclose(dtype_id);
2666 H5Dclose(dset_id);
2667 throw2("Cannot read the data of the dataset ", varpath);
2668 }
2669
2670 vector<string> finstrval;
2671 finstrval.resize(total_elms);
2672 char*temp_bp = &strval[0];
2673 char*onestring = NULL;
2674 for (unsigned long long i = 0; i < total_elms; i++) {
2675 onestring = *(char**) temp_bp;
2676 if (onestring != NULL)
2677 finstrval[i] = string(onestring);
2678 else
2679 // We will add a NULL if onestring is NULL.
2680 finstrval[i] = "";
2681 temp_bp += ty_size;
2682 }
2683
2684 if (false == strval.empty()) {
2685 herr_t ret_vlen_claim;
2686 hid_t dspace_id = H5Dget_space(dset_id);
2687 if (dspace_id < 0) {
2688 H5Tclose(dtype_id);
2689 H5Dclose(dset_id);
2690 throw2("Cannot obtain the dataspace id.", varpath);
2691 }
2692 ret_vlen_claim = H5Dvlen_reclaim(dtype_id, dspace_id, H5P_DEFAULT, (void*) &strval[0]);
2693 if (ret_vlen_claim < 0) {
2694 H5Tclose(dtype_id);
2695 H5Sclose(dspace_id);
2696 H5Dclose(dset_id);
2697 throw2("Cannot reclaim the vlen space ", varpath);
2698 }
2699 if (H5Sclose(dspace_id) < 0) {
2700 H5Tclose(dtype_id);
2701 H5Dclose(dset_id);
2702 throw2("Cannot close the HDF5 data space.", varpath);
2703 }
2704 }
2705 unsigned long long total_str_size = 0;
2706 for (unsigned long long i = 0; i < total_elms; i++) {
2707 total_str_size += finstrval[i].size();
2708 if (total_str_size > NC_JAVA_STR_SIZE_LIMIT) {
2709 drop_longstr = true;
2710 break;
2711 }
2712 }
2713 }
2714 if (H5Tclose(dtype_id) < 0) {
2715 H5Dclose(dset_id);
2716 throw2("Cannot close the HDF5 data type.", varpath);
2717 }
2718 if (H5Dclose(dset_id) < 0)
2719 throw2("Cannot close the HDF5 data type.", varpath);
2720 }
2721 return drop_longstr;
2722}
2723#endif
2724
2725
2726// Provide if the long string is dropped.
2727void File::add_ignored_grp_longstr_info(const string& grp_path, const string & attr_name)
2728{
2729
2730 ignored_msg += "The HDF5 group: " + grp_path + " has an empty-set string attribute: " + attr_name + "\n";
2731
2732 return;
2733}
2734
2735// Provide if the long variable string is dropped.
2736void File::add_ignored_var_longstr_info(Var *var, Attribute *attr)
2737{
2738
2739 if (NULL == attr)
2740 ignored_msg += "String variable: " + var->fullpath + " value is set to empty.\n";
2741 else {
2742 ignored_msg += "The variable: " + var->fullpath + " has an empty-set string attribute: " + attr->name + "\n";
2743
2744 }
2745 return;
2746}
2747
2748// The warnings of the drop of the long string header
2749void File::add_ignored_droplongstr_hdr()
2750{
2751
2752 if (false == this->have_ignored) this->have_ignored = true;
2753 string hdr = "\n\n The values of the following string variables ";
2754 hdr += " are set to empty because at least one string size in this variable exceeds netCDF Java string limit(32767 bytes).\n";
2755 hdr += "To obtain the values, change the BES key H5.EnableDropLongString=true at the handler BES";
2756 hdr += " configuration file(h5.conf)\nto H5.EnableDropLongString=false.\n\n";
2757
2758 if (ignored_msg.rfind(hdr) == string::npos) ignored_msg += hdr;
2759
2760}
2761
2762// Sometimes, we need to release the temporary added resources.
2763void File::release_standalone_var_vector(vector<Var*>&temp_vars)
2764{
2765
2766 for (vector<Var *>::iterator i = temp_vars.begin(); i != temp_vars.end();) {
2767 delete (*i);
2768 i = temp_vars.erase(i);
2769 }
2770
2771}
This class specifies the core engineering of mapping HDF5 to DAP by following CF.
#define throw1(a1)
The followings are convenient functions to throw exceptions with different.
Definition: HDF5CF.h:128
include the entry functions to execute the handlers
This class represents one attribute.
Definition: HDF5CF.h:189
This class repersents one dimension of an HDF5 dataset(variable).
Definition: HDF5CF.h:145
std::vector< Group * > groups
Non-root group vectors.
Definition: HDF5CF.h:816
virtual void Retrieve_H5_Var_Attr_Values(Var *var)
Retrieve attribute values for a variable.
Definition: HDF5CF.cc:748
virtual void Handle_Unsupported_Dspace(bool)
Handle unsupported HDF5 dataspaces for datasets.
Definition: HDF5CF.cc:1275
std::map< hsize_t, std::string > dimsize_to_fakedimname
Handle added dimension names.
Definition: HDF5CF.h:833
virtual void Handle_Grid_Mapping_Vars()
Handle Grid Mapping Vars.
Definition: HDF5CF.cc:2151
virtual void Handle_Unsupported_Others(bool)
Handle other unmapped objects/attributes.
Definition: HDF5CF.cc:1322
virtual void Retrieve_H5_Supported_Attr_Values()
Retrieve attribute values for the supported HDF5 datatypes.
Definition: HDF5CF.cc:729
std::vector< Var * > vars
Var vectors.
Definition: HDF5CF.h:810
virtual void Add_Supplement_Attrs(bool)
Add supplemental attributes such as fullpath and original name.
Definition: HDF5CF.cc:1970
virtual void Retrieve_H5_Info(const char *path, hid_t file_id, bool)
Definition: HDF5CF.cc:171
std::vector< Attribute * > root_attrs
Root attribute vectors.
Definition: HDF5CF.h:813
virtual void Handle_Unsupported_Dtype(bool)
Handle unsupported HDF5 datatypes.
Definition: HDF5CF.cc:918
virtual void Flatten_Obj_Name(bool)
Flatten the object name.
Definition: HDF5CF.cc:1372
virtual bool Have_Grid_Mapping_Attrs()
Check if having Grid Mapping Attrs.
Definition: HDF5CF.cc:2131
This class represents an HDF5 group. The group will be flattened according to the CF conventions.
Definition: HDF5CF.h:552
This class represents one HDF5 dataset(CF variable)
Definition: HDF5CF.h:259
Helper functions for generating DAS attributes and a function to check BES Key.
static H5DataType H5type_to_H5DAPtype(hid_t h5_type_id)
Map HDF5 Datatype to the intermediate H5DAPtype for the future use.
Definition: HDF5CFUtil.cc:54
static std::string trim_string(hid_t dtypeid, const std::string s, int num_sect, size_t section_size, std::vector< size_t > &sect_newsize)
Definition: HDF5CFUtil.cc:235