bes Updated for version 3.20.10
HDF5CFArray.cc
Go to the documentation of this file.
1// This file is part of the hdf5_handler implementing for the CF-compliant
2// Copyright (c) 2011-2016 The HDF Group, Inc. and OPeNDAP, Inc.
3//
4// This is free software; you can redistribute it and/or modify it under the
5// terms of the GNU Lesser General Public License as published by the Free
6// Software Foundation; either version 2.1 of the License, or (at your
7// option) any later version.
8//
9// This software is distributed in the hope that it will be useful, but
10// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
11// or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
12// License for more details.
13//
14// You should have received a copy of the GNU Lesser General Public
15// License along with this library; if not, write to the Free Software
16// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17//
18// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
19// You can contact The HDF Group, Inc. at 1800 South Oak Street,
20// Suite 203, Champaign, IL 61820
21
30
31#include "config_hdf5.h"
32#include <iostream>
33#include <sstream>
34#include <cassert>
35#include <BESDebug.h>
36#include <sys/stat.h>
37#include <libdap/InternalErr.h>
38
39#include <libdap/Str.h>
40#include "HDF5RequestHandler.h"
41#include "HDF5CFArray.h"
42#include "h5cfdaputil.h"
43#include "ObjMemCache.h"
44
45using namespace std;
46using namespace libdap;
47
48
49BaseType *HDF5CFArray::ptr_duplicate()
50{
51 return new HDF5CFArray(*this);
52}
53
54// Read in an HDF5 Array
55bool HDF5CFArray::read()
56{
57
58 BESDEBUG("h5","Coming to HDF5CFArray read "<<endl);
59 if(length() == 0)
60 return true;
61
62 if((NULL == HDF5RequestHandler::get_lrdata_mem_cache()) &&
63 NULL == HDF5RequestHandler::get_srdata_mem_cache()){
65 return true;
66 }
67
68 // Flag to check if using large raw data cache or small raw data cache.
69 short use_cache_flag = 0;
70
71 // The small data cache is checked first to reduce the resources to operate the big data cache.
72 if(HDF5RequestHandler::get_srdata_mem_cache() != NULL) {
73 if(((cvtype == CV_EXIST) && (islatlon != true)) || (cvtype == CV_NONLATLON_MISS)
74 || (cvtype == CV_FILLINDEX) ||(cvtype == CV_MODIFY) ||(cvtype == CV_SPECIAL)){
75
76 if(HDF5CFUtil::cf_dap2_support_numeric_type(dtype,is_dap4)==true)
77 use_cache_flag = 1;
78 }
79 }
80
81 // If this varible doesn't fit the small data cache, let's check if it fits the large data cache.
82 if(use_cache_flag !=1) {
83
84 if(HDF5RequestHandler::get_lrdata_mem_cache() != NULL) {
85
86 // This is the trival case.
87 // If no information is provided in the configuration file of large data cache,
88 // just cache the lat/lon varible per file.
89 if(HDF5RequestHandler::get_common_cache_dirs() == false) {
90 if(cvtype == CV_LAT_MISS || cvtype == CV_LON_MISS
91 || (cvtype == CV_EXIST && islatlon == true)) {
92#if 0
93//cerr<<"coming to use_cache_flag =2 "<<endl;
94#endif
95 // Only the data with the numeric datatype DAP2 and CF support are cached.
96 if(HDF5CFUtil::cf_dap2_support_numeric_type(dtype,is_dap4)==true)
97 use_cache_flag = 2;
98 }
99 }
100 else {// Have large data cache configuration info.
101
102 // Need to check if we don't want to cache some CVs, now
103 // this only applies to lat/lon CV.
104 if(cvtype == CV_LAT_MISS || cvtype == CV_LON_MISS
105 || (cvtype == CV_EXIST && islatlon == true)) {
106
107 vector<string> cur_lrd_non_cache_dir_list;
108 HDF5RequestHandler::get_lrd_non_cache_dir_list(cur_lrd_non_cache_dir_list);
109
110 // Check if this file is included in the non-cache directory
111 if( (cur_lrd_non_cache_dir_list.size() == 0) ||
112 ("" == check_str_sect_in_list(cur_lrd_non_cache_dir_list,filename,'/'))) {
113
114 // Only data with the numeric datatype DAP2 and CF support are cached.
115 if(HDF5CFUtil::cf_dap2_support_numeric_type(dtype,is_dap4)==true)
116 use_cache_flag = 3;
117 }
118 }
119 // Here we allow all the variable names to be cached.
120 // The file path that includes the variables can also included.
121 vector<string> cur_lrd_var_cache_file_list;
122 HDF5RequestHandler::get_lrd_var_cache_file_list(cur_lrd_var_cache_file_list);
123 if(cur_lrd_var_cache_file_list.size() >0){
124#if 0
126//cerr<<"lrd var cache is "<<cur_lrd_var_cache_file_list[i]<<endl;
127#endif
128 if(true == check_var_cache_files(cur_lrd_var_cache_file_list,filename,varname)){
129#if 0
130//cerr<<"varname is "<<varname <<endl;
131//cerr<<"have var cached "<<endl;
132#endif
133
134 // Only the data with the numeric datatype DAP2 and CF support are cached.
135 if(HDF5CFUtil::cf_dap2_support_numeric_type(dtype,is_dap4)==true)
136 use_cache_flag = 4;
137 }
138 }
139 }
140 }
141 }
142
143 if(0 == use_cache_flag)
145 else {// memory cache cases
146
147 string cache_key;
148
149 // Possibly we have common lat/lon dirs,so check here.
150 if( 3 == use_cache_flag){
151 vector<string> cur_cache_dlist;
152 HDF5RequestHandler::get_lrd_cache_dir_list(cur_cache_dlist);
153 string cache_dir = check_str_sect_in_list(cur_cache_dlist,filename,'/');
154 if(cache_dir != "")
155 cache_key = cache_dir + varname;
156 else {
157 cache_key = filename + varname;
158 // If this lat/lon is not in the common dir. list, it is still cached as a general lat/lon.
159 // Change the flag to 2.
160 use_cache_flag = 2;
161 }
162
163 }
164 else
165 cache_key = filename + varname;
166
167 handle_data_with_mem_cache(dtype,total_elems,use_cache_flag,cache_key,is_dap4);
168
169 }
170
171 return true;
172}
173
174// Reading data not from memory cache: The data can be read from the disk cache or can be read via the HDF5 APIs
175void HDF5CFArray::read_data_NOT_from_mem_cache(bool add_mem_cache,void*buf) {
176
177 vector<int>offset;
178 vector<int>count;
179 vector<int>step;
180 vector<hsize_t> hoffset;
181 vector<hsize_t>hcount;
182 vector<hsize_t>hstep;
183 int nelms = 1;
184
185 if (rank <= 0)
186 throw InternalErr (__FILE__, __LINE__,
187 "The number of dimension of the variable is <=0 for an array.");
188 else {
189
190 offset.resize(rank);
191 count.resize(rank);
192 step.resize(rank);
193 hoffset.resize(rank);
194 hcount.resize(rank);
195 hstep.resize(rank);
196 nelms = format_constraint (&offset[0], &step[0], &count[0]);
197 for (int i = 0; i <rank; i++) {
198 hoffset[i] = (hsize_t) offset[i];
199 hcount[i] = (hsize_t) count[i];
200 hstep[i] = (hsize_t) step[i];
201 }
202 }
203
204 hid_t dsetid = -1;
205 hid_t dspace = -1;
206 hid_t mspace = -1;
207 hid_t dtypeid = -1;
208 hid_t memtype = -1;
209
210 bool data_from_disk_cache = false;
211 bool data_to_disk_cache = false;
212
213 // Check if the disk cache can be applied.
214 bool use_disk_cache = valid_disk_cache();
215
216 string cache_fpath;
217
218 if(true == use_disk_cache) {
219
220 BESDEBUG("h5","Coming to use disk cache "<<endl);
221
222 unsigned long long disk_cache_size = HDF5RequestHandler::get_disk_cache_size();
223 string diskcache_dir = HDF5RequestHandler::get_disk_cache_dir();
224 string diskcache_prefix = HDF5RequestHandler::get_disk_cachefile_prefix();
225
226 string cache_fname=HDF5CFUtil::obtain_cache_fname(diskcache_prefix,filename,varname);
227 cache_fpath = diskcache_dir + "/"+ cache_fname;
228
229 int temp_total_elems = 1;
230 for (unsigned int i = 0; i <dimsizes.size();i++)
231 temp_total_elems = temp_total_elems*dimsizes[i];
232 short dtype_size = HDF5CFUtil::H5_numeric_atomic_type_size(dtype);
233 // CHECK: I think when signed 8-bit needs to be converted to int16, dtype_size should also change.
234 if(is_dap4 == false && dtype==H5CHAR)
235 dtype_size = 2;
236
237 int expected_file_size = dtype_size *temp_total_elems;
238 int fd = 0;
239 HDF5DiskCache *disk_cache = HDF5DiskCache::get_instance(disk_cache_size,diskcache_dir,diskcache_prefix);
240 if( true == disk_cache->get_data_from_cache(cache_fpath, expected_file_size,fd)) {
241
242 vector<size_t> offset_size_t;
243 offset_size_t.resize(rank);
244 for(int i = 0; i <rank;i++)
245 offset_size_t[i] = (size_t)offset[i];
246 size_t offset_1st = INDEX_nD_TO_1D(dimsizes,offset_size_t);
247 vector<size_t>end;
248 end.resize(rank);
249 for (int i = 0; i < rank; i++)
250 end[i] = offset[i] +(count[i]-1)*step[i];
251 size_t offset_last = INDEX_nD_TO_1D(dimsizes,end);
252#if 0
253//cerr<<"offset_1d is "<<offset_1st <<endl;
254//cerr<<"offset_last is "<<offset_last <<endl;
255#endif
256 size_t total_read = dtype_size*(offset_last-offset_1st+1);
257
258 off_t fpos = lseek(fd,dtype_size*offset_1st,SEEK_SET);
259 if (-1 == fpos) {
260 disk_cache->unlock_and_close(cache_fpath);
261 disk_cache->purge_file(cache_fpath);
262 }
263
265 else
266 data_from_disk_cache = obtain_cached_data(disk_cache,cache_fpath,fd, step,count,total_read,dtype_size);
267
268 }
269
270 if(true == data_from_disk_cache)
271 return;
272 else
273 data_to_disk_cache = true;
274
275 }
276
277// END CACHE
278
279 bool pass_fileid = HDF5RequestHandler::get_pass_fileid();
280 if(false == pass_fileid) {
281 if ((fileid = H5Fopen(filename.c_str(),H5F_ACC_RDONLY,H5P_DEFAULT))<0) {
282 ostringstream eherr;
283 eherr << "HDF5 File " << filename
284 << " cannot be opened. "<<endl;
285 throw InternalErr (__FILE__, __LINE__, eherr.str ());
286 }
287 }
288
289 if ((dsetid = H5Dopen(fileid,varname.c_str(),H5P_DEFAULT))<0) {
290 HDF5CFUtil::close_fileid(fileid,pass_fileid);
291 ostringstream eherr;
292 eherr << "HDF5 dataset " << varname
293 << " cannot be opened. "<<endl;
294 throw InternalErr (__FILE__, __LINE__, eherr.str ());
295 }
296
297 if ((dspace = H5Dget_space(dsetid))<0) {
298
299 H5Dclose(dsetid);
300 HDF5CFUtil::close_fileid(fileid,pass_fileid);
301 ostringstream eherr;
302 eherr << "Space id of the HDF5 dataset " << varname
303 << " cannot be obtained. "<<endl;
304 throw InternalErr (__FILE__, __LINE__, eherr.str ());
305 }
306
307 if (H5Sselect_hyperslab(dspace, H5S_SELECT_SET,
308 &hoffset[0], &hstep[0],
309 &hcount[0], NULL) < 0) {
310
311 H5Sclose(dspace);
312 H5Dclose(dsetid);
313 HDF5CFUtil::close_fileid(fileid,pass_fileid);
314 ostringstream eherr;
315 eherr << "The selection of hyperslab of the HDF5 dataset " << varname
316 << " fails. "<<endl;
317 throw InternalErr (__FILE__, __LINE__, eherr.str ());
318 }
319
320 mspace = H5Screate_simple(rank, &hcount[0],NULL);
321 if (mspace < 0) {
322 H5Sclose(dspace);
323 H5Dclose(dsetid);
324 HDF5CFUtil::close_fileid(fileid,pass_fileid);
325 ostringstream eherr;
326 eherr << "The creation of the memory space of the HDF5 dataset " << varname
327 << " fails. "<<endl;
328 throw InternalErr (__FILE__, __LINE__, eherr.str ());
329 }
330
331
332 if ((dtypeid = H5Dget_type(dsetid)) < 0) {
333
334 H5Sclose(mspace);
335 H5Sclose(dspace);
336 H5Dclose(dsetid);
337 HDF5CFUtil::close_fileid(fileid,pass_fileid);
338 ostringstream eherr;
339 eherr << "Obtaining the datatype of the HDF5 dataset " << varname
340 << " fails. "<<endl;
341 throw InternalErr (__FILE__, __LINE__, eherr.str ());
342
343 }
344
345 if ((memtype = H5Tget_native_type(dtypeid, H5T_DIR_ASCEND))<0) {
346
347 H5Sclose(mspace);
348 H5Tclose(dtypeid);
349 H5Sclose(dspace);
350 H5Dclose(dsetid);
351 HDF5CFUtil::close_fileid(fileid,pass_fileid);
352 ostringstream eherr;
353 eherr << "Obtaining the memory type of the HDF5 dataset " << varname
354 << " fails. "<<endl;
355 throw InternalErr (__FILE__, __LINE__, eherr.str ());
356
357 }
358
359 hid_t read_ret = -1;
360
361 // Before reading the data, we will check if the memory cache is turned on,
362 // The add_mem_cache is only true when the data memory cache keys are on and used.
363 if(true == add_mem_cache) {
364 if(buf== NULL) {
365 H5Sclose(mspace);
366 H5Tclose(dtypeid);
367 H5Sclose(dspace);
368 H5Dclose(dsetid);
369 HDF5CFUtil::close_fileid(fileid,pass_fileid);
370 throw InternalErr(__FILE__,__LINE__,"The memory data cache buffer needs to be set");
371 }
372 read_ret= H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf);
373 if(read_ret <0){
374 H5Sclose(mspace);
375 H5Tclose(dtypeid);
376 H5Sclose(dspace);
377 H5Dclose(dsetid);
378 HDF5CFUtil::close_fileid(fileid,pass_fileid);
379 throw InternalErr(__FILE__,__LINE__,"Cannot read the data to the buffer.");
380 }
381 }
382
383
384 // Now reading the data, note dtype is not dtypeid.
385 // dtype is an enum defined by the handler.
386
387 switch (dtype) {
388
389 case H5CHAR:
390 {
391
392 vector<char> val;
393 val.resize(nelms);
394
395 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
396 if (read_ret < 0) {
397
398 H5Sclose(mspace);
399 H5Tclose(memtype);
400 H5Tclose(dtypeid);
401 H5Sclose(dspace);
402 H5Dclose(dsetid);
403 HDF5CFUtil::close_fileid(fileid,pass_fileid);
404 ostringstream eherr;
405 eherr << "Cannot read the HDF5 dataset " << varname
406 << " with the type of H5T_NATIVE_CHAR "<<endl;
407 throw InternalErr (__FILE__, __LINE__, eherr.str ());
408
409 }
410
411 if(is_dap4 == true)
412 set_value((dods_int8 *)&val[0],nelms);
413 else {
414
415 vector<short>newval;
416 newval.resize(nelms);
417
418 for (int counter = 0; counter < nelms; counter++)
419 newval[counter] = (short) (val[counter]);
420
421 set_value ((dods_int16 *) &newval[0], nelms);
422 }
423
424 if(true == data_to_disk_cache) {
425 try {
426 BESDEBUG("h5","writing data to disk cache "<<endl);
427 write_data_to_cache(dsetid,dspace,mspace,memtype,cache_fpath,2,val,nelms);
428 }
429 catch(...) {
430 H5Sclose(mspace);
431 H5Tclose(memtype);
432 H5Tclose(dtypeid);
433 H5Sclose(dspace);
434 H5Dclose(dsetid);
435 HDF5CFUtil::close_fileid(fileid,pass_fileid);
436 ostringstream eherr;
437 eherr << "write data to cache failed.";
438 throw InternalErr (__FILE__, __LINE__, eherr.str ());
439
440 }
441 }
442
443 } // case H5CHAR
444 break;
445
446 // Note: for DAP2, H5INT64,H5UINT64 will be ignored.
447 case H5UCHAR:
448 case H5UINT16:
449 case H5INT16:
450 case H5INT32:
451 case H5UINT32:
452 case H5INT64:
453 case H5UINT64:
454 case H5FLOAT32:
455 case H5FLOAT64:
456
457
458 {
459 size_t dtype_size = HDF5CFUtil::H5_numeric_atomic_type_size(dtype);
460 vector<char> val;
461 val.resize(nelms*dtype_size);
462
463 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
464 if (read_ret < 0) {
465 H5Sclose(mspace);
466 H5Tclose(memtype);
467 H5Tclose(dtypeid);
468 H5Sclose(dspace);
469 H5Dclose(dsetid);
470 HDF5CFUtil::close_fileid(fileid,pass_fileid);
471 ostringstream eherr;
472 eherr << "Cannot read the HDF5 dataset " << varname
473 << " with the type of H5T_NATIVE_UCHAR "<<endl;
474 throw InternalErr (__FILE__, __LINE__, eherr.str ());
475
476 }
477 // Not sure if "set_value ((dods_byte *) &val[0], nelms);" works.
478 val2buf(&val[0]);
479 set_read_p(true);
480
481 if(true == data_to_disk_cache) {
482 BESDEBUG("h5","writing data to disk cache "<<endl);
483 try {
484 write_data_to_cache(dsetid,dspace,mspace,memtype,cache_fpath,dtype_size,val,nelms);
485 }
486 catch(...) {
487 H5Sclose(mspace);
488 H5Tclose(memtype);
489 H5Tclose(dtypeid);
490 H5Sclose(dspace);
491 H5Dclose(dsetid);
492 HDF5CFUtil::close_fileid(fileid,pass_fileid);
493 ostringstream eherr;
494 eherr << "write data to cache failed.";
495 throw InternalErr (__FILE__, __LINE__, eherr.str ());
496
497 }
498
499 }
500 } // case H5UCHAR...
501 break;
502
503
504
505#if 0
506 case H5INT16:
507 {
508 vector<short>val;
509 val.resize(nelms);
510
511 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
512 if (read_ret < 0) {
513
514 H5Sclose(mspace);
515 H5Tclose(memtype);
516 H5Tclose(dtypeid);
517 H5Sclose(dspace);
518 H5Dclose(dsetid);
519 HDF5CFUtil::close_fileid(fileid,pass_fileid);
520 //H5Fclose(fileid);
521 ostringstream eherr;
522 eherr << "Cannot read the HDF5 dataset " << varname
523 << " with the type of H5T_NATIVE_SHORT "<<endl;
524 throw InternalErr (__FILE__, __LINE__, eherr.str ());
525
526 }
527 set_value ((dods_int16 *) &val[0], nelms);
528 }// H5INT16
529 break;
530
531
532 case H5UINT16:
533 {
534 vector<unsigned short> val;
535 val.resize(nelms);
536 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
537 if (read_ret < 0) {
538
539 H5Sclose(mspace);
540 H5Tclose(memtype);
541 H5Tclose(dtypeid);
542 H5Sclose(dspace);
543 H5Dclose(dsetid);
544 HDF5CFUtil::close_fileid(fileid,pass_fileid);
545 ostringstream eherr;
546 eherr << "Cannot read the HDF5 dataset " << varname
547 << " with the type of H5T_NATIVE_USHORT "<<endl;
548 throw InternalErr (__FILE__, __LINE__, eherr.str ());
549
550 }
551 set_value ((dods_uint16 *) &val[0], nelms);
552 } // H5UINT16
553 break;
554
555
556 case H5INT32:
557 {
558 vector<int>val;
559 val.resize(nelms);
560 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
561 if (read_ret < 0) {
562 H5Sclose(mspace);
563 H5Tclose(memtype);
564 H5Tclose(dtypeid);
565 H5Sclose(dspace);
566 H5Dclose(dsetid);
567 HDF5CFUtil::close_fileid(fileid,pass_fileid);
568 ostringstream eherr;
569 eherr << "Cannot read the HDF5 dataset " << varname
570 << " with the type of H5T_NATIVE_INT "<<endl;
571 throw InternalErr (__FILE__, __LINE__, eherr.str ());
572
573 }
574 set_value ((dods_int32 *) &val[0], nelms);
575 } // case H5INT32
576 break;
577
578 case H5UINT32:
579 {
580 vector<unsigned int>val;
581 val.resize(nelms);
582 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
583 if (read_ret < 0) {
584 H5Sclose(mspace);
585 H5Tclose(memtype);
586 H5Tclose(dtypeid);
587 H5Sclose(dspace);
588 H5Dclose(dsetid);
589 HDF5CFUtil::close_fileid(fileid,pass_fileid);
590 ostringstream eherr;
591 eherr << "Cannot read the HDF5 dataset " << varname
592 << " with the type of H5T_NATIVE_UINT "<<endl;
593 throw InternalErr (__FILE__, __LINE__, eherr.str ());
594
595 }
596 set_value ((dods_uint32 *) &val[0], nelms);
597 }
598 break;
599
600 case H5FLOAT32:
601 {
602
603 vector<float>val;
604 val.resize(nelms);
605
606 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
607 if (read_ret < 0) {
608 H5Sclose(mspace);
609 H5Tclose(memtype);
610 H5Tclose(dtypeid);
611 H5Sclose(dspace);
612 H5Dclose(dsetid);
613 HDF5CFUtil::close_fileid(fileid,pass_fileid);
614 ostringstream eherr;
615 eherr << "Cannot read the HDF5 dataset " << varname
616 << " with the type of H5T_NATIVE_FLOAT "<<endl;
617 throw InternalErr (__FILE__, __LINE__, eherr.str ());
618
619 }
620 set_value ((dods_float32 *) &val[0], nelms);
621 }
622 break;
623
624
625 case H5FLOAT64:
626 {
627
628 vector<double>val;
629 val.resize(nelms);
630 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
631
632 if (read_ret < 0) {
633 H5Sclose(mspace);
634 H5Tclose(memtype);
635 H5Tclose(dtypeid);
636 H5Sclose(dspace);
637 H5Dclose(dsetid);
638 HDF5CFUtil::close_fileid(fileid,pass_fileid);
639 ostringstream eherr;
640 eherr << "Cannot read the HDF5 dataset " << varname
641 << " with the type of H5T_NATIVE_DOUBLE "<<endl;
642 throw InternalErr (__FILE__, __LINE__, eherr.str ());
643
644 }
645 set_value ((dods_float64 *) &val[0], nelms);
646 } // case H5FLOAT64
647 break;
648
649#endif
650
651 case H5FSTRING:
652 {
653 size_t ty_size = H5Tget_size(dtypeid);
654 if (ty_size == 0) {
655 H5Sclose(mspace);
656 H5Tclose(memtype);
657 H5Tclose(dtypeid);
658 H5Sclose(dspace);
659 H5Dclose(dsetid);
660 HDF5CFUtil::close_fileid(fileid,pass_fileid);
661 ostringstream eherr;
662 eherr << "Cannot obtain the size of the fixed size HDF5 string of the dataset "
663 << varname <<endl;
664 throw InternalErr (__FILE__, __LINE__, eherr.str ());
665 }
666
667 vector <char> strval;
668 strval.resize(nelms*ty_size);
669 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,(void*)&strval[0]);
670
671 if (read_ret < 0) {
672 H5Sclose(mspace);
673 H5Tclose(memtype);
674 H5Tclose(dtypeid);
675 H5Sclose(dspace);
676 H5Dclose(dsetid);
677 HDF5CFUtil::close_fileid(fileid,pass_fileid);
678 ostringstream eherr;
679 eherr << "Cannot read the HDF5 dataset " << varname
680 << " with the type of the fixed size HDF5 string "<<endl;
681 throw InternalErr (__FILE__, __LINE__, eherr.str ());
682 }
683
684 string total_string(strval.begin(),strval.end());
685 strval.clear();//save some memory;
686 vector <string> finstrval;
687 finstrval.resize(nelms);
688 for (int i = 0; i<nelms; i++)
689 finstrval[i] = total_string.substr(i*ty_size,ty_size);
690
691 // Check if we should drop the long string
692
693 // If the size of an individual element is longer than the current netCDF JAVA
694 // string and the "EnableDropLongString" key is turned on,
695 // No string is generated.
696 if ((true == HDF5RequestHandler::get_drop_long_string()) &&
697 ty_size > NC_JAVA_STR_SIZE_LIMIT) {
698 for (int i = 0; i<nelms; i++)
699 finstrval[i] = "";
700 }
701 set_value(finstrval,nelms);
702 total_string.clear();
703 }
704 break;
705
706
707 case H5VSTRING:
708 {
709 size_t ty_size = H5Tget_size(memtype);
710 if (ty_size == 0) {
711 H5Sclose(mspace);
712 H5Tclose(memtype);
713 H5Tclose(dtypeid);
714 H5Sclose(dspace);
715 H5Dclose(dsetid);
716 HDF5CFUtil::close_fileid(fileid,pass_fileid);
717 ostringstream eherr;
718 eherr << "Cannot obtain the size of the fixed size HDF5 string of the dataset "
719 << varname <<endl;
720 throw InternalErr (__FILE__, __LINE__, eherr.str ());
721 }
722 vector <char> strval;
723 strval.resize(nelms*ty_size);
724 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,(void*)&strval[0]);
725
726 if (read_ret < 0) {
727 H5Sclose(mspace);
728 H5Tclose(memtype);
729 H5Tclose(dtypeid);
730 H5Sclose(dspace);
731 H5Dclose(dsetid);
732 HDF5CFUtil::close_fileid(fileid,pass_fileid);
733 ostringstream eherr;
734 eherr << "Cannot read the HDF5 dataset " << varname
735 << " with the type of the HDF5 variable length string "<<endl;
736 throw InternalErr (__FILE__, __LINE__, eherr.str ());
737 }
738
739 vector<string>finstrval;
740 finstrval.resize(nelms);
741 char*temp_bp = &strval[0];
742 char*onestring = NULL;
743 for (int i =0;i<nelms;i++) {
744 onestring = *(char**)temp_bp;
745 if(onestring!=NULL )
746 finstrval[i] =string(onestring);
747
748 else // We will add a NULL if onestring is NULL.
749 finstrval[i]="";
750 temp_bp +=ty_size;
751 }
752
753 if (false == strval.empty()) {
754 herr_t ret_vlen_claim;
755 ret_vlen_claim = H5Dvlen_reclaim(memtype,mspace,H5P_DEFAULT,(void*)&strval[0]);
756 if (ret_vlen_claim < 0){
757 H5Sclose(mspace);
758 H5Tclose(memtype);
759 H5Tclose(dtypeid);
760 H5Sclose(dspace);
761 H5Dclose(dsetid);
762 HDF5CFUtil::close_fileid(fileid,pass_fileid);
763 ostringstream eherr;
764 eherr << "Cannot reclaim the memory buffer of the HDF5 variable length string of the dataset "
765 << varname <<endl;
766 throw InternalErr (__FILE__, __LINE__, eherr.str ());
767
768 }
769 }
770
771 // If the size of one string element is longer than the current netCDF JAVA
772 // string and the "EnableDropLongString" key is turned on,
773 // No string is generated.
774 if (true == HDF5RequestHandler::get_drop_long_string()) {
775 bool drop_long_str = false;
776 for (int i =0;i<nelms;i++) {
777 if(finstrval[i].size() >NC_JAVA_STR_SIZE_LIMIT){
778 drop_long_str = true;
779 break;
780 }
781 }
782 if (drop_long_str == true) {
783 for (int i =0;i<nelms;i++)
784 finstrval[i] = "";
785 }
786 }
787 set_value(finstrval,nelms);
788
789 }
790 break;
791
792 default:
793 {
794 H5Tclose(memtype);
795 H5Tclose(dtypeid);
796 H5Sclose(mspace);
797 H5Sclose(dspace);
798 H5Dclose(dsetid);
799 HDF5CFUtil::close_fileid(fileid,pass_fileid);
800 ostringstream eherr;
801 eherr << "Cannot read the HDF5 dataset " << varname
802 << " with the unsupported HDF5 datatype"<<endl;
803 throw InternalErr (__FILE__, __LINE__, eherr.str ());
804 }
805 }
806
807 H5Tclose(memtype);
808 H5Tclose(dtypeid);
809 H5Sclose(mspace);
810 H5Sclose(dspace);
811 H5Dclose(dsetid);
812 HDF5CFUtil::close_fileid(fileid,pass_fileid);
813
814 return;
815}
816
817bool HDF5CFArray::valid_disk_cache() {
818
819 bool ret_value = false;
820 if(true == HDF5RequestHandler::get_use_disk_cache()) {
821
822 BESDEBUG("h5","Coming to disk cache "<<endl);
823 // Check if this is a valid numeric datatype we want to support
824 if(dtype == H5CHAR || dtype ==H5UCHAR || dtype==H5INT16 || dtype ==H5UINT16 ||
825 dtype == H5INT32 || dtype ==H5UINT32 || dtype ==H5FLOAT32 || dtype==H5FLOAT64 ||
826 dtype == H5INT64 || dtype ==H5UINT64){
827
828 BESDEBUG("h5","Coming to disk cache datatype block"<<endl);
829
830 string diskcache_dir = HDF5RequestHandler::get_disk_cache_dir();
831 string diskcache_prefix = HDF5RequestHandler::get_disk_cachefile_prefix();
832 long diskcache_size = HDF5RequestHandler::get_disk_cache_size();
833
834 if(("" == diskcache_dir)||(""==diskcache_prefix)||(diskcache_size <=0))
835 throw InternalErr (__FILE__, __LINE__, "Either the cached dir is empty or the prefix is NULL or the cache size is not set.");
836 else {
837 struct stat sb;
838 if(stat(diskcache_dir.c_str(),&sb) !=0) {
839 string err_mesg="The cached directory " + diskcache_dir;
840 err_mesg = err_mesg + " doesn't exist. ";
841 throw InternalErr(__FILE__,__LINE__,err_mesg);
842 }
843 else {
844 if(true == S_ISDIR(sb.st_mode)) {
845 if(access(diskcache_dir.c_str(),R_OK|W_OK|X_OK) == -1) {
846 string err_mesg="The cached directory " + diskcache_dir;
847 err_mesg = err_mesg + " can NOT be read,written or executable.";
848 throw InternalErr(__FILE__,__LINE__,err_mesg);
849 }
850 }
851 else {
852 string err_mesg="The cached directory " + diskcache_dir;
853 err_mesg = err_mesg + " is not a directory.";
854 throw InternalErr(__FILE__,__LINE__,err_mesg);
855 }
856 }
857 }
858
859 short dtype_size = HDF5CFUtil::H5_numeric_atomic_type_size(dtype);
860 // Check if we only need to cache the specific compressed dat
861 if(true == HDF5RequestHandler::get_disk_cache_comp_data()){
862 BESDEBUG("h5","Compression disk cache key is true"<<endl);
863 ret_value = valid_disk_cache_for_compressed_data(dtype_size);
864 BESDEBUG("h5","variable disk cache passes the compression parameter check"<<endl);
865 }
866 else {
867 BESDEBUG("h5","Compression disk cache key is NOT set, disk cache key is true."<<endl);
868 ret_value = true;
869 }
870
871 }
872
873 }
874 return ret_value;
875}
876
877bool HDF5CFArray:: valid_disk_cache_for_compressed_data(short dtype_size) const {
878
879 bool ret_value = false;
880 // The compression ratio should be smaller then the threshold(hard to compress)
881 // and the total var size should be bigger than the defined size(bigger)
882#if 0
883 size_t total_byte = total_elems*dtype_size;
884#endif
885 if((comp_ratio < HDF5RequestHandler::get_disk_comp_threshold())
886 && (total_elems*dtype_size >= HDF5RequestHandler::get_disk_var_size())) {
887 if( true == HDF5RequestHandler::get_disk_cache_float_only_comp()) {
888 if(dtype==H5FLOAT32 || dtype == H5FLOAT64)
889 ret_value = true;
890 }
891 else
892 ret_value = true;
893 }
894 return ret_value;
895
896}
897
898bool HDF5CFArray::obtain_cached_data(HDF5DiskCache *disk_cache,const string & cache_fpath, int fd,vector<int> &cd_step, vector<int>&cd_count,size_t total_read,short dtype_size) {
899
900 ssize_t ret_read_val = -1;
901 vector<char>buf;
902
903 buf.resize(total_read);
904 ret_read_val = HDF5CFUtil::read_buffer_from_file(fd,(void*)&buf[0],total_read);
905 disk_cache->unlock_and_close(cache_fpath);
906 if((-1 == ret_read_val) || (ret_read_val != (ssize_t)total_read)) {
907 disk_cache->purge_file(cache_fpath);
908 return false;
909 }
910 else {
911 unsigned int nele_to_read = 1;
912 for(int i = 0; i<rank;i++)
913 nele_to_read *=cd_count[i];
914
915 if(nele_to_read == (total_read/dtype_size)) {
916 val2buf(&buf[0]);
917 set_read_p(true);
918 }
919 else { // Need to re-assemble the buffer according to different datatype
920
921 vector<int>cd_start(rank,0);
922 vector<size_t>cd_pos(rank,0);
923 int nelms_to_send = 1;
924 for(int i = 0; i <rank; i++)
925 nelms_to_send = nelms_to_send*cd_count[i];
926
927 switch (dtype) {
928
929 case H5CHAR:
930 {
931#if 0
932 vector<int>total_val;
933 total_val.resize(total_read/dtype_size);
934 memcpy(&total_val[0],(void*)&buf[0],total_read);
935
936 vector<int>final_val;
937 subset<int>(
938 &total_val[0],
939 rank,
940 dimsizes,
941 &cd_start[0],
942 &cd_step[0],
943 &cd_count[0],
944 &final_val,
945 cd_pos,
946 0
947 );
948
949#endif
950
951 if(is_dap4 == false) {
952 vector<short>final_val;
953 subset<short>(
954 &buf[0],
955 rank,
956 dimsizes,
957 &cd_start[0],
958 &cd_step[0],
959 &cd_count[0],
960 &final_val,
961 cd_pos,
962 0
963 );
964 set_value((dods_int16*)&final_val[0],nelms_to_send);
965 }
966 else {
967 vector<char>final_val;
968 subset<char>(
969 &buf[0],
970 rank,
971 dimsizes,
972 &cd_start[0],
973 &cd_step[0],
974 &cd_count[0],
975 &final_val,
976 cd_pos,
977 0
978 );
979 set_value((dods_int8*)&final_val[0],nelms_to_send);
980 }
981
982 }
983
984 break;
985 case H5UCHAR:
986 {
987#if 0
988 vector<unsigned char>total_val;
989 total_val.resize(total_read/dtype_size);
990 memcpy(&total_val[0],(void*)&buf[0],total_read);
991
992 vector<unsigned char>final_val;
993 subset<unsigned char>(
994 &total_val[0],
995 rank,
996 dimsizes,
997 &cd_start[0],
998 &cd_step[0],
999 &cd_count[0],
1000 &final_val,
1001 cd_pos,
1002 0
1003 );
1004
1005#endif
1006 vector<unsigned char>final_val;
1007 subset<unsigned char>(
1008 &buf[0],
1009 rank,
1010 dimsizes,
1011 &cd_start[0],
1012 &cd_step[0],
1013 &cd_count[0],
1014 &final_val,
1015 cd_pos,
1016 0
1017 );
1018
1019 set_value ((dods_byte *) &final_val[0], nelms_to_send);
1020 }
1021 break;
1022
1023 case H5INT16:
1024 {
1025#if 0
1026 vector<short>total_val;
1027 total_val.resize(total_read/dtype_size);
1028 memcpy(&total_val[0],(void*)&buf[0],total_read);
1029
1030 vector<short>final_val;
1031 subset<short>(
1032 &total_val[0],
1033 rank,
1034 dimsizes,
1035 &cd_start[0],
1036 &cd_step[0],
1037 &cd_count[0],
1038 &final_val,
1039 cd_pos,
1040 0
1041 );
1042#endif
1043
1044 vector<short>final_val;
1045 subset<short>(
1046 &buf[0],
1047 rank,
1048 dimsizes,
1049 &cd_start[0],
1050 &cd_step[0],
1051 &cd_count[0],
1052 &final_val,
1053 cd_pos,
1054 0
1055 );
1056
1057 set_value ((dods_int16 *) &final_val[0], nelms_to_send);
1058 }
1059 break;
1060
1061 case H5UINT16:
1062 {
1063#if 0
1064 vector<unsigned short>total_val;
1065 total_val.resize(total_read/dtype_size);
1066 memcpy(&total_val[0],(void*)&buf[0],total_read);
1067
1068 vector<unsigned short>final_val;
1069 subset<unsigned short>(
1070 &total_val[0],
1071 rank,
1072 dimsizes,
1073 &cd_start[0],
1074 &cd_step[0],
1075 &cd_count[0],
1076 &final_val,
1077 cd_pos,
1078 0
1079 );
1080#endif
1081
1082 vector<unsigned short>final_val;
1083 subset<unsigned short>(
1084 &buf[0],
1085 rank,
1086 dimsizes,
1087 &cd_start[0],
1088 &cd_step[0],
1089 &cd_count[0],
1090 &final_val,
1091 cd_pos,
1092 0
1093 );
1094
1095 set_value ((dods_uint16 *) &final_val[0], nelms_to_send);
1096 }
1097 break;
1098
1099 case H5INT32:
1100 {
1101#if 0
1102 vector<int>total_val;
1103 total_val.resize(total_read/dtype_size);
1104 memcpy(&total_val[0],(void*)&buf[0],total_read);
1105
1106 vector<int>final_val;
1107 subset<int>(
1108 &total_val[0],
1109 rank,
1110 dimsizes,
1111 &cd_start[0],
1112 &cd_step[0],
1113 &cd_count[0],
1114 &final_val,
1115 cd_pos,
1116 0
1117 );
1118
1119#endif
1120
1121 vector<int>final_val;
1122 subset<int>(
1123 &buf[0],
1124 rank,
1125 dimsizes,
1126 &cd_start[0],
1127 &cd_step[0],
1128 &cd_count[0],
1129 &final_val,
1130 cd_pos,
1131 0
1132 );
1133
1134
1135 set_value ((dods_int32 *) &final_val[0], nelms_to_send);
1136 }
1137 break;
1138
1139 case H5UINT32:
1140 {
1141#if 0
1142 vector<unsigned int>total_val;
1143 total_val.resize(total_read/dtype_size);
1144 memcpy(&total_val[0],(void*)&buf[0],total_read);
1145
1146 vector<unsigned int>final_val;
1147 subset<unsigned int>(
1148 &total_val[0],
1149 rank,
1150 dimsizes,
1151 &cd_start[0],
1152 &cd_step[0],
1153 &cd_count[0],
1154 &final_val,
1155 cd_pos,
1156 0
1157 );
1158#endif
1159
1160 vector<unsigned int>final_val;
1161 subset<unsigned int>(
1162 &buf[0],
1163 rank,
1164 dimsizes,
1165 &cd_start[0],
1166 &cd_step[0],
1167 &cd_count[0],
1168 &final_val,
1169 cd_pos,
1170 0
1171 );
1172
1173 set_value ((dods_uint32 *) &final_val[0], nelms_to_send);
1174 }
1175 break;
1176
1177 case H5INT64: // Only for DAP4 CF
1178 {
1179#if 0
1180 vector<unsigned int>total_val;
1181 total_val.resize(total_read/dtype_size);
1182 memcpy(&total_val[0],(void*)&buf[0],total_read);
1183
1184 vector<unsigned int>final_val;
1185 subset<unsigned int>(
1186 &total_val[0],
1187 rank,
1188 dimsizes,
1189 &cd_start[0],
1190 &cd_step[0],
1191 &cd_count[0],
1192 &final_val,
1193 cd_pos,
1194 0
1195 );
1196#endif
1197
1198 vector<long long >final_val;
1199 subset<long long >(
1200 &buf[0],
1201 rank,
1202 dimsizes,
1203 &cd_start[0],
1204 &cd_step[0],
1205 &cd_count[0],
1206 &final_val,
1207 cd_pos,
1208 0
1209 );
1210
1211 set_value ((dods_int64 *) &final_val[0], nelms_to_send);
1212 }
1213 break;
1214
1215
1216
1217 case H5UINT64: // Only for DAP4 CF
1218 {
1219#if 0
1220 vector<unsigned int>total_val;
1221 total_val.resize(total_read/dtype_size);
1222 memcpy(&total_val[0],(void*)&buf[0],total_read);
1223
1224 vector<unsigned int>final_val;
1225 subset<unsigned int>(
1226 &total_val[0],
1227 rank,
1228 dimsizes,
1229 &cd_start[0],
1230 &cd_step[0],
1231 &cd_count[0],
1232 &final_val,
1233 cd_pos,
1234 0
1235 );
1236#endif
1237
1238 vector<unsigned long long >final_val;
1239 subset<unsigned long long >(
1240 &buf[0],
1241 rank,
1242 dimsizes,
1243 &cd_start[0],
1244 &cd_step[0],
1245 &cd_count[0],
1246 &final_val,
1247 cd_pos,
1248 0
1249 );
1250
1251 set_value ((dods_uint64 *) &final_val[0], nelms_to_send);
1252 }
1253 break;
1254
1255
1256 case H5FLOAT32:
1257 {
1258#if 0
1259 vector<float>total_val;
1260 total_val.resize(total_read/dtype_size);
1261 memcpy(&total_val[0],(void*)&buf[0],total_read);
1262
1263 vector<float>final_val;
1264 subset<float>(
1265 &total_val[0],
1266 rank,
1267 dimsizes,
1268 &cd_start[0],
1269 &cd_step[0],
1270 &cd_count[0],
1271 &final_val,
1272 cd_pos,
1273 0
1274 );
1275#endif
1276
1277 vector<float>final_val;
1278 subset<float>(
1279 &buf[0],
1280 rank,
1281 dimsizes,
1282 &cd_start[0],
1283 &cd_step[0],
1284 &cd_count[0],
1285 &final_val,
1286 cd_pos,
1287 0
1288 );
1289
1290
1291 set_value ((dods_float32 *) &final_val[0], nelms_to_send);
1292 }
1293 break;
1294 case H5FLOAT64:
1295 {
1296#if 0
1297 vector<double>total_val;
1298 total_val.resize(total_read/dtype_size);
1299 memcpy(&total_val[0],(void*)&buf[0],total_read);
1300
1301 vector<double>final_val;
1302 subset<double>(
1303 &total_val[0],
1304 rank,
1305 dimsizes,
1306 &cd_start[0],
1307 &cd_step[0],
1308 &cd_count[0],
1309 &final_val,
1310 cd_pos,
1311 0
1312 );
1313#endif
1314 vector<double>final_val;
1315 subset<double>(
1316 &buf[0],
1317 rank,
1318 dimsizes,
1319 &cd_start[0],
1320 &cd_step[0],
1321 &cd_count[0],
1322 &final_val,
1323 cd_pos,
1324 0
1325 );
1326
1327 set_value ((dods_float64 *) &final_val[0], nelms_to_send);
1328 }
1329 break;
1330 default:
1331 throw InternalErr (__FILE__, __LINE__, "unsupported data type.");
1332
1333 }// "end switch(dtype)"
1334 }// "end else (stride is not 1)"
1335 return true;
1336 }// "end else(full_read = true)"
1337}
1338
1339
1340void
1341HDF5CFArray::write_data_to_cache(hid_t dset_id, hid_t /*dspace_id*/, hid_t /*mspace_id*/, hid_t memtype,
1342 const string& cache_fpath, short dtype_size, const vector<char> &buf, int nelms) {
1343
1344 unsigned long long disk_cache_size = HDF5RequestHandler::get_disk_cache_size();
1345 string disk_cache_dir = HDF5RequestHandler::get_disk_cache_dir();
1346 string disk_cache_prefix = HDF5RequestHandler::get_disk_cachefile_prefix();
1347 HDF5DiskCache *disk_cache = HDF5DiskCache::get_instance(disk_cache_size,disk_cache_dir,disk_cache_prefix);
1348 int total_nelem = 1;
1349 for(int i = 0; i <rank; i++)
1350 total_nelem = total_nelem*dimsizes[i];
1351
1352 vector<char>val;
1353
1354 if(H5CHAR == dtype && is_dap4 == false) {
1355
1356 vector<short>newval;
1357 newval.resize(total_nelem);
1358 if(total_nelem == nelms) {
1359 for (int i = 0; i < total_nelem;i++)
1360 newval[i] = (short)buf[i];
1361 disk_cache->write_cached_data2(cache_fpath,sizeof(short)*total_nelem,(const void*)&newval[0]);
1362 }
1363 else {
1364 vector<char>val2;
1365 val2.resize(total_nelem);
1366 if(H5Dread(dset_id, memtype, H5S_ALL, H5S_ALL,H5P_DEFAULT, &val2[0])<0)
1367 throw InternalErr (__FILE__, __LINE__, "Cannot read the whole HDF5 dataset for the disk cache.");
1368 for (int i = 0; i < total_nelem;i++)
1369 newval[i] = (short)val2[i];
1370 disk_cache->write_cached_data2(cache_fpath,sizeof(short)*total_nelem,(const void*)&newval[0]);
1371 }
1372 }
1373 else {
1374 if(total_nelem == nelms) {
1375 disk_cache->write_cached_data2(cache_fpath,dtype_size*total_nelem,(const void*)&buf[0]);
1376 }
1377 else {
1378 val.resize(dtype_size*total_nelem);
1379 if(H5Dread(dset_id, memtype, H5S_ALL, H5S_ALL,H5P_DEFAULT, &val[0])<0)
1380 throw InternalErr (__FILE__, __LINE__, "Cannot read the whole SDS for cache.");
1381
1382 disk_cache->write_cached_data2(cache_fpath,dtype_size*total_nelem,(const void*)&val[0]);
1383 }
1384 }
1385}
1386
1387#if 0
1388void HDF5CFArray::read_data_from_mem_cache(void*buf) {
1389
1390 vector<int>offset;
1391 vector<int>count;
1392 vector<int>step;
1393 int nelms = format_constraint (&offset[0], &step[0], &count[0]);
1394 // set the original position to the starting point
1395 vector<int>at_pos(at_ndims,0);
1396 for (int i = 0; i< rank; i++)
1397 at_pos[i] = at_offset[i];
1398
1399
1400 switch (dtype) {
1401
1402 case H5UCHAR:
1403
1404 {
1405 vector<unsigned char> val;
1406 val.resize(nelms);
1407 subset<unsigned char>(
1408 &total_val[0],
1409 rank,
1410 dimsizes,
1411 offset,
1412 step,
1413 count,
1414 &final_val,
1415 at_pos,
1416 0
1417 );
1418
1419
1420 set_value ((dods_byte *) &val[0], nelms);
1421 } // case H5UCHAR
1422 break;
1423
1424
1425 case H5CHAR:
1426 {
1427
1428 vector<char> val;
1429 val.resize(nelms);
1430
1431 if (0 == rank)
1432 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1433 else
1434 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1435
1436 if (read_ret < 0) {
1437
1438 if (rank > 0)
1439 H5Sclose(mspace);
1440 H5Tclose(memtype);
1441 H5Tclose(dtypeid);
1442 H5Sclose(dspace);
1443 H5Dclose(dsetid);
1444 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1445 ostringstream eherr;
1446 eherr << "Cannot read the HDF5 dataset " << varname
1447 << " with the type of H5T_NATIVE_CHAR "<<endl;
1448 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1449
1450 }
1451
1452 vector<short>newval;
1453 newval.resize(nelms);
1454
1455 for (int counter = 0; counter < nelms; counter++)
1456 newval[counter] = (short) (val[counter]);
1457
1458 set_value ((dods_int16 *) &newval[0], nelms);
1459 } // case H5CHAR
1460 break;
1461
1462
1463 case H5INT16:
1464 {
1465 vector<short>val;
1466 val.resize(nelms);
1467
1468 if (0 == rank)
1469 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1470 else
1471 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1472
1473 if (read_ret < 0) {
1474
1475 if (rank > 0)
1476 H5Sclose(mspace);
1477 H5Tclose(memtype);
1478 H5Tclose(dtypeid);
1479 H5Sclose(dspace);
1480 H5Dclose(dsetid);
1481 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1482 //H5Fclose(fileid);
1483 ostringstream eherr;
1484 eherr << "Cannot read the HDF5 dataset " << varname
1485 << " with the type of H5T_NATIVE_SHORT "<<endl;
1486 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1487
1488 }
1489 set_value ((dods_int16 *) &val[0], nelms);
1490 }// H5INT16
1491 break;
1492
1493
1494 case H5UINT16:
1495 {
1496 vector<unsigned short> val;
1497 val.resize(nelms);
1498 if (0 == rank)
1499 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1500 else
1501 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1502
1503 if (read_ret < 0) {
1504
1505 if (rank > 0) H5Sclose(mspace);
1506 H5Tclose(memtype);
1507 H5Tclose(dtypeid);
1508 H5Sclose(dspace);
1509 H5Dclose(dsetid);
1510 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1511 ostringstream eherr;
1512 eherr << "Cannot read the HDF5 dataset " << varname
1513 << " with the type of H5T_NATIVE_USHORT "<<endl;
1514 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1515
1516 }
1517 set_value ((dods_uint16 *) &val[0], nelms);
1518 } // H5UINT16
1519 break;
1520
1521
1522 case H5INT32:
1523 {
1524 vector<int>val;
1525 val.resize(nelms);
1526 if (0 == rank)
1527 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1528 else
1529 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1530
1531 if (read_ret < 0) {
1532 if (rank > 0)
1533 H5Sclose(mspace);
1534 H5Tclose(memtype);
1535 H5Tclose(dtypeid);
1536 H5Sclose(dspace);
1537 H5Dclose(dsetid);
1538 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1539 ostringstream eherr;
1540 eherr << "Cannot read the HDF5 dataset " << varname
1541 << " with the type of H5T_NATIVE_INT "<<endl;
1542 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1543
1544 }
1545 set_value ((dods_int32 *) &val[0], nelms);
1546 } // case H5INT32
1547 break;
1548
1549 case H5UINT32:
1550 {
1551 vector<unsigned int>val;
1552 val.resize(nelms);
1553 if (0 == rank)
1554 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1555 else
1556 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1557
1558 if (read_ret < 0) {
1559
1560 if (rank > 0)
1561 H5Sclose(mspace);
1562 H5Tclose(memtype);
1563 H5Tclose(dtypeid);
1564 H5Sclose(dspace);
1565 H5Dclose(dsetid);
1566 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1567 ostringstream eherr;
1568 eherr << "Cannot read the HDF5 dataset " << varname
1569 << " with the type of H5T_NATIVE_UINT "<<endl;
1570 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1571
1572 }
1573 set_value ((dods_uint32 *) &val[0], nelms);
1574 }
1575 break;
1576
1577 case H5FLOAT32:
1578 {
1579
1580 vector<float>val;
1581 val.resize(nelms);
1582
1583 if (0 == rank)
1584 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1585 else
1586 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1587
1588 if (read_ret < 0) {
1589 if (rank > 0)
1590 H5Sclose(mspace);
1591 H5Tclose(memtype);
1592 H5Tclose(dtypeid);
1593 H5Sclose(dspace);
1594 H5Dclose(dsetid);
1595 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1596 ostringstream eherr;
1597 eherr << "Cannot read the HDF5 dataset " << varname
1598 << " with the type of H5T_NATIVE_FLOAT "<<endl;
1599 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1600
1601 }
1602 set_value ((dods_float32 *) &val[0], nelms);
1603 }
1604 break;
1605
1606
1607 case H5FLOAT64:
1608 {
1609
1610 vector<double>val;
1611 val.resize(nelms);
1612 if (0 == rank)
1613 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1614 else
1615 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1616
1617 if (read_ret < 0) {
1618 if (rank > 0)
1619 H5Sclose(mspace);
1620 H5Tclose(memtype);
1621 H5Tclose(dtypeid);
1622 H5Sclose(dspace);
1623 H5Dclose(dsetid);
1624 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1625 ostringstream eherr;
1626 eherr << "Cannot read the HDF5 dataset " << varname
1627 << " with the type of H5T_NATIVE_DOUBLE "<<endl;
1628 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1629
1630 }
1631 set_value ((dods_float64 *) &val[0], nelms);
1632 } // case H5FLOAT64
1633 break;
1634
1635
1636
1637 // Just see if it works.
1638 val2buf(buf);
1639 set_read_p(true);
1640 return;
1641}
1642#endif
1643
1644#if 0
1645// We don't inherit libdap Array Class's transform_to_dap4 method since it also transforms attributes.
1646BaseType* HDF5CFArray::h5cfdims_transform_to_dap4(D4Group *grp) {
1647
1648 if(grp == NULL)
1649 return NULL;
1650 Array *dest = static_cast<HDF5CFArray*>(ptr_duplicate());
1651
1652 // If there is just a size, don't make
1653 // a D4Dimension (In DAP4 you cannot share a dimension unless it has
1654 // a name). jhrg 3/18/14
1655
1656 D4Dimensions *grp_dims = grp->dims();
1657 for (Array::Dim_iter dap2_dim = dest->dim_begin(), e = dest->dim_end(); dap2_dim != e; ++dap2_dim) {
1658 if (!(*dap2_dim).name.empty()) {
1659
1660 // If a D4Dimension with the name already exists, use it.
1661 D4Dimension *d4_dim = grp_dims->find_dim((*dap2_dim).name);
1662 if (!d4_dim) {
1663 d4_dim = new D4Dimension((*dap2_dim).name, (*dap2_dim).size);
1664 grp_dims->add_dim_nocopy(d4_dim);
1665 }
1666 // At this point d4_dim's name and size == those of (*d) so just set
1667 // the D4Dimension pointer so it matches the one in the D4Group.
1668 (*dap2_dim).dim = d4_dim;
1669 }
1670 }
1671
1672 return dest;
1673
1674}
1675#endif
1676
1677// We don't inherit libdap Array Class's transform_to_dap4 method since CF option is still using it.
1678// This function is used for 64-bit integer mapping to DAP4 for the CF option. largely borrowed from
1679// DAP4 code.
1680BaseType* HDF5CFArray::h5cfdims_transform_to_dap4_int64(D4Group *grp) {
1681
1682 if(grp == NULL)
1683 return NULL;
1684 Array *dest = static_cast<HDF5CFArray*>(ptr_duplicate());
1685
1686 // If there is just a size, don't make
1687 // a D4Dimension (In DAP4 you cannot share a dimension unless it has
1688 // a name). jhrg 3/18/14
1689
1690 for (Array::Dim_iter d = dest->dim_begin(), e = dest->dim_end(); d != e; ++d) {
1691 if (false == (*d).name.empty()) {
1692
1693 D4Group *temp_grp = grp;
1694 D4Dimension *d4_dim = NULL;
1695 while(temp_grp) {
1696
1697 D4Dimensions *temp_dims = temp_grp->dims();
1698
1699 // Check if the dimension is defined in this group
1700 d4_dim = temp_dims->find_dim((*d).name);
1701 if(d4_dim) {
1702 (*d).dim = d4_dim;
1703 break;
1704 }
1705
1706 if(temp_grp->get_parent())
1707 temp_grp = static_cast<D4Group*>(temp_grp->get_parent());
1708 else
1709 temp_grp = 0;
1710
1711 }
1712
1713 // Not find this dimension in any of the ancestor groups, add it to this group.
1714 // The following block is fine, but to avoid the complaint from sonarcloud.
1715 // Use a bool.
1716 bool d4_dim_null = ((d4_dim==NULL)?true:false);
1717#if 0
1718 //if(d4_dim == NULL) {
1719#endif
1720 // Not find this dimension in any of the ancestor groups, add it to this group.
1721 if(d4_dim_null == true) {
1722
1723 d4_dim = new D4Dimension((*d).name, (*d).size);
1724 D4Dimensions * dims = grp->dims();
1725 dims->add_dim_nocopy(d4_dim);
1726 (*d).dim = d4_dim;
1727 }
1728 }
1729 }
1730
1731 dest->set_is_dap4(true);
1732
1733 return dest;
1734
1735}
1736#if 0
1737// parse constraint expr. and make hdf5 coordinate point location.
1738// return number of elements to read.
1739int
1740HDF5CFArray::format_constraint (int *offset, int *step, int *count)
1741{
1742
1743 long nels = 1;
1744 int id = 0;
1745
1746 Dim_iter p = dim_begin ();
1747
1748 while (p != dim_end ()) {
1749
1750 int start = dimension_start (p, true);
1751 int stride = dimension_stride (p, true);
1752 int stop = dimension_stop (p, true);
1753
1754 // Check for illegal constraint
1755 if (start > stop) {
1756 ostringstream oss;
1757
1758 oss << "Array/Grid hyperslab start point "<< start <<
1759 " is greater than stop point " << stop <<".";
1760 throw Error(malformed_expr, oss.str());
1761 }
1762
1763 offset[id] = start;
1764 step[id] = stride;
1765 count[id] = ((stop - start) / stride) + 1; // count of elements
1766 nels *= count[id]; // total number of values for variable
1767
1768 BESDEBUG ("h5",
1769 "=format_constraint():"
1770 << "id=" << id << " offset=" << offset[id]
1771 << " step=" << step[id]
1772 << " count=" << count[id]
1773 << endl);
1774
1775 id++;
1776 p++;
1777 }
1778
1779 return nels;
1780}
1781
1782#endif
This class includes the methods to read data array into DAP buffer from an HDF5 dataset for the CF op...
include the entry functions to execute the handlers
virtual void unlock_and_close(const std::string &target)
virtual void purge_file(const std::string &file)
Purge a single file from the cache.
virtual void read_data_NOT_from_mem_cache(bool add_cache, void *buf)
Definition: HDF5CFArray.cc:175
static HDF5DiskCache * get_instance(const long, const std::string &, const std::string &)
Helper functions for generating DAS attributes and a function to check BES Key.
static ssize_t read_buffer_from_file(int fd, void *buf, size_t)
Getting a subset of a variable.
Definition: HDF5CFUtil.cc:1195