NetCDF 4.9.0
nc4hdf.c
Go to the documentation of this file.
1/* Copyright 2018, University Corporation for Atmospheric
2 * Research. See the COPYRIGHT file for copying and redistribution
3 * conditions. */
17#include "config.h"
18#include "netcdf.h"
19#include "nc4internal.h"
20#include "ncdispatch.h"
21#include "hdf5internal.h"
22#include "hdf5err.h" /* For BAIL2 */
23#include "hdf5debug.h"
24#include <math.h>
25
26#ifdef HAVE_INTTYPES_H
27#define __STDC_FORMAT_MACROS
28#include <inttypes.h>
29#endif
30
31#define NC_HDF5_MAX_NAME 1024
41static int
42flag_atts_dirty(NCindex *attlist) {
43
44 NC_ATT_INFO_T *att = NULL;
45 int i;
46
47 if(attlist == NULL) {
48 return NC_NOERR;
49 }
50
51 for(i=0;i<ncindexsize(attlist);i++) {
52 att = (NC_ATT_INFO_T*)ncindexith(attlist,i);
53 if(att == NULL) continue;
54 att->dirty = NC_TRUE;
55 }
56
57 return NC_NOERR;
58}
59
76int
77rec_reattach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid)
78{
79 NC_VAR_INFO_T *var;
80 NC_GRP_INFO_T *child_grp;
81 int d, i;
82 int retval;
83
84 assert(grp && grp->hdr.name && dimid >= 0 && dimscaleid >= 0);
85 LOG((3, "%s: grp->hdr.name %s", __func__, grp->hdr.name));
86
87 /* If there are any child groups, attach dimscale there, if needed. */
88 for (i = 0; i < ncindexsize(grp->children); i++)
89 {
90 child_grp = (NC_GRP_INFO_T*)ncindexith(grp->children, i);
91 assert(child_grp);
92 if ((retval = rec_reattach_scales(child_grp, dimid, dimscaleid)))
93 return retval;
94 }
95
96 /* Find any vars that use this dimension id. */
97 for (i = 0; i < ncindexsize(grp->vars); i++)
98 {
99 NC_HDF5_VAR_INFO_T *hdf5_var;
100
101 var = (NC_VAR_INFO_T*)ncindexith(grp->vars,i);
102 assert(var && var->format_var_info);
103
104 hdf5_var = (NC_HDF5_VAR_INFO_T*)var->format_var_info;
105 assert(hdf5_var != NULL);
106 for (d = 0; d < var->ndims; d++)
107 {
108 if (var->dimids[d] == dimid && !hdf5_var->dimscale)
109 {
110 LOG((2, "%s: attaching scale for dimid %d to var %s",
111 __func__, var->dimids[d], var->hdr.name));
112 if (var->created)
113 {
114 if (H5DSattach_scale(hdf5_var->hdf_datasetid,
115 dimscaleid, d) < 0)
116 return NC_EDIMSCALE;
117 hdf5_var->dimscale_attached[d] = NC_TRUE;
118 }
119 }
120 }
121 }
122 return NC_NOERR;
123}
124
141int
142rec_detach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid)
143{
144 NC_VAR_INFO_T *var;
145 NC_GRP_INFO_T *child_grp;
146 int d, i;
147 int retval;
148
149 assert(grp && grp->hdr.name && dimid >= 0 && dimscaleid >= 0);
150 LOG((3, "%s: grp->hdr.name %s", __func__, grp->hdr.name));
151
152 /* If there are any child groups, detach dimscale there, if needed. */
153 for(i=0;i<ncindexsize(grp->children);i++) {
154 child_grp = (NC_GRP_INFO_T*)ncindexith(grp->children,i);
155 if(child_grp == NULL) continue;
156 if ((retval = rec_detach_scales(child_grp, dimid, dimscaleid)))
157 return retval;
158 }
159
160 /* Find any vars that use this dimension id. */
161 for (i = 0; i < ncindexsize(grp->vars); i++)
162 {
163 NC_HDF5_VAR_INFO_T *hdf5_var;
164 var = (NC_VAR_INFO_T*)ncindexith(grp->vars, i);
165 assert(var && var->format_var_info);
166 hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
167
168 for (d = 0; d < var->ndims; d++)
169 {
170 if (var->dimids[d] == dimid && !hdf5_var->dimscale)
171 {
172 LOG((2, "%s: detaching scale for dimid %d to var %s",
173 __func__, var->dimids[d], var->hdr.name));
174 if (var->created)
175 {
176 if (hdf5_var->dimscale_attached && hdf5_var->dimscale_attached[d])
177 {
178 if (H5DSdetach_scale(hdf5_var->hdf_datasetid,
179 dimscaleid, d) < 0)
180 return NC_EDIMSCALE;
181 hdf5_var->dimscale_attached[d] = NC_FALSE;
182 }
183 }
184 }
185 }
186 }
187 return NC_NOERR;
188}
189
201int
202nc4_open_var_grp2(NC_GRP_INFO_T *grp, int varid, hid_t *dataset)
203{
204 NC_VAR_INFO_T *var;
205 NC_HDF5_VAR_INFO_T *hdf5_var;
206
207 assert(grp && grp->format_grp_info && dataset);
208
209 /* Find the requested varid. */
210 if (!(var = (NC_VAR_INFO_T *)ncindexith(grp->vars, varid)))
211 return NC_ENOTVAR;
212 assert(var && var->hdr.id == varid && var->format_var_info);
213 hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
214
215 /* Open this dataset if necessary. */
216 if (!hdf5_var->hdf_datasetid)
217 {
218 NC_HDF5_GRP_INFO_T *hdf5_grp;
219 hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
220
221 if ((hdf5_var->hdf_datasetid = H5Dopen2(hdf5_grp->hdf_grpid,
222 var->hdr.name, H5P_DEFAULT)) < 0)
223 return NC_ENOTVAR;
224 }
225
226 *dataset = hdf5_var->hdf_datasetid;
227
228 return NC_NOERR;
229}
230
248int
249nc4_get_hdf_typeid(NC_FILE_INFO_T *h5, nc_type xtype,
250 hid_t *hdf_typeid, int endianness)
251{
252 NC_TYPE_INFO_T *type;
253 hid_t typeid = 0;
254 int retval = NC_NOERR;
255
256 assert(hdf_typeid && h5);
257
258 *hdf_typeid = -1;
259
260 /* Determine an appropriate HDF5 datatype */
261 if (xtype == NC_NAT)
262 return NC_EBADTYPE;
263 else if (xtype == NC_CHAR || xtype == NC_STRING)
264 {
265 /* NC_CHAR & NC_STRING types create a new HDF5 datatype */
266 if (xtype == NC_CHAR)
267 {
268 if ((typeid = H5Tcopy(H5T_C_S1)) < 0)
269 return NC_EHDFERR;
270 if (H5Tset_strpad(typeid, H5T_STR_NULLTERM) < 0)
271 BAIL(NC_EVARMETA);
272 if(H5Tset_cset(typeid, H5T_CSET_ASCII) < 0)
273 BAIL(NC_EVARMETA);
274
275 /* Take ownership of the newly created HDF5 datatype */
276 *hdf_typeid = typeid;
277 typeid = 0;
278 }
279 else
280 {
281 if ((typeid = H5Tcopy(H5T_C_S1)) < 0)
282 return NC_EHDFERR;
283 if (H5Tset_size(typeid, H5T_VARIABLE) < 0)
284 BAIL(NC_EVARMETA);
285 if(H5Tset_cset(typeid, H5T_CSET_UTF8) < 0)
286 BAIL(NC_EVARMETA);
287
288 /* Take ownership of the newly created HDF5 datatype */
289 *hdf_typeid = typeid;
290 typeid = 0;
291 }
292 }
293 else
294 {
295 /* All other types use an existing HDF5 datatype */
296 switch (xtype)
297 {
298 case NC_BYTE: /* signed 1 byte integer */
299 if (endianness == NC_ENDIAN_LITTLE)
300 typeid = H5T_STD_I8LE;
301 else if (endianness == NC_ENDIAN_BIG)
302 typeid = H5T_STD_I8BE;
303 else
304 typeid = H5T_NATIVE_SCHAR;
305 break;
306
307 case NC_SHORT: /* signed 2 byte integer */
308 if (endianness == NC_ENDIAN_LITTLE)
309 typeid = H5T_STD_I16LE;
310 else if (endianness == NC_ENDIAN_BIG)
311 typeid = H5T_STD_I16BE;
312 else
313 typeid = H5T_NATIVE_SHORT;
314 break;
315
316 case NC_INT:
317 if (endianness == NC_ENDIAN_LITTLE)
318 typeid = H5T_STD_I32LE;
319 else if (endianness == NC_ENDIAN_BIG)
320 typeid = H5T_STD_I32BE;
321 else
322 typeid = H5T_NATIVE_INT;
323 break;
324
325 case NC_UBYTE:
326 if (endianness == NC_ENDIAN_LITTLE)
327 typeid = H5T_STD_U8LE;
328 else if (endianness == NC_ENDIAN_BIG)
329 typeid = H5T_STD_U8BE;
330 else
331 typeid = H5T_NATIVE_UCHAR;
332 break;
333
334 case NC_USHORT:
335 if (endianness == NC_ENDIAN_LITTLE)
336 typeid = H5T_STD_U16LE;
337 else if (endianness == NC_ENDIAN_BIG)
338 typeid = H5T_STD_U16BE;
339 else
340 typeid = H5T_NATIVE_USHORT;
341 break;
342
343 case NC_UINT:
344 if (endianness == NC_ENDIAN_LITTLE)
345 typeid = H5T_STD_U32LE;
346 else if (endianness == NC_ENDIAN_BIG)
347 typeid = H5T_STD_U32BE;
348 else
349 typeid = H5T_NATIVE_UINT;
350 break;
351
352 case NC_INT64:
353 if (endianness == NC_ENDIAN_LITTLE)
354 typeid = H5T_STD_I64LE;
355 else if (endianness == NC_ENDIAN_BIG)
356 typeid = H5T_STD_I64BE;
357 else
358 typeid = H5T_NATIVE_LLONG;
359 break;
360
361 case NC_UINT64:
362 if (endianness == NC_ENDIAN_LITTLE)
363 typeid = H5T_STD_U64LE;
364 else if (endianness == NC_ENDIAN_BIG)
365 typeid = H5T_STD_U64BE;
366 else
367 typeid = H5T_NATIVE_ULLONG;
368 break;
369
370 case NC_FLOAT:
371 if (endianness == NC_ENDIAN_LITTLE)
372 typeid = H5T_IEEE_F32LE;
373 else if (endianness == NC_ENDIAN_BIG)
374 typeid = H5T_IEEE_F32BE;
375 else
376 typeid = H5T_NATIVE_FLOAT;
377 break;
378
379 case NC_DOUBLE:
380 if (endianness == NC_ENDIAN_LITTLE)
381 typeid = H5T_IEEE_F64LE;
382 else if (endianness == NC_ENDIAN_BIG)
383 typeid = H5T_IEEE_F64BE;
384 else
385 typeid = H5T_NATIVE_DOUBLE;
386 break;
387
388 default:
389 /* Maybe this is a user defined type? */
390 if (nc4_find_type(h5, xtype, &type))
391 return NC_EBADTYPE;
392 if (!type)
393 return NC_EBADTYPE;
394 typeid = ((NC_HDF5_TYPE_INFO_T *)type->format_type_info)->hdf_typeid;
395 break;
396 }
397 assert(typeid);
398
399 /* Copy the HDF5 datatype, so the function operates uniformly */
400 if ((*hdf_typeid = H5Tcopy(typeid)) < 0)
401 return NC_EHDFERR;
402 typeid = 0;
403 }
404 assert(*hdf_typeid != -1);
405
406exit:
407 if (typeid > 0 && H5Tclose(typeid) < 0)
408 BAIL2(NC_EHDFERR);
409 return retval;
410}
411
426static int
427put_att_grpa(NC_GRP_INFO_T *grp, int varid, NC_ATT_INFO_T *att)
428{
429 NC_HDF5_GRP_INFO_T *hdf5_grp;
430 hid_t datasetid = 0, locid;
431 hid_t attid = 0, spaceid = 0, file_typeid = 0;
432 hid_t existing_att_typeid = 0, existing_attid = 0, existing_spaceid = 0;
433 hsize_t dims[1]; /* netcdf attributes always 1-D. */
434 htri_t attr_exists;
435 void *data;
436 int phoney_data = 99;
437 int retval = NC_NOERR;
438
439 assert(att->hdr.name && grp && grp->format_grp_info);
440 LOG((3, "%s: varid %d att->hdr.id %d att->hdr.name %s att->nc_typeid %d "
441 "att->len %d", __func__, varid, att->hdr.id, att->hdr.name,
442 att->nc_typeid, att->len));
443
444 /* Get HDF5-specific group info. */
445 hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
446
447 /* If the file is read-only, return an error. */
448 if (grp->nc4_info->no_write)
449 BAIL(NC_EPERM);
450
451 /* Get the hid to attach the attribute to, or read it from. */
452 if (varid == NC_GLOBAL)
453 locid = hdf5_grp->hdf_grpid;
454 else
455 {
456 if ((retval = nc4_open_var_grp2(grp, varid, &datasetid)))
457 BAIL(retval);
458 locid = datasetid;
459 }
460
461 /* Get the length ready, and find the HDF type we'll be
462 * writing. */
463 dims[0] = att->len;
464 if ((retval = nc4_get_hdf_typeid(grp->nc4_info, att->nc_typeid,
465 &file_typeid, 0)))
466 BAIL(retval);
467
468 /* Even if the length is zero, HDF5 won't let me write with a
469 * NULL pointer. So if the length of the att is zero, point to
470 * some phoney data (which won't be written anyway.)*/
471 if (!dims[0])
472 data = &phoney_data;
473#ifdef SEPDATA
474 else if (att->vldata)
475 data = att->vldata;
476 else if (att->stdata)
477 data = att->stdata;
478#endif
479 else
480 data = att->data;
481
482 /* NC_CHAR types require some extra work. The space ID is set to
483 * scalar, and the type is told how long the string is. If it's
484 * really zero length, set the size to 1. (The fact that it's
485 * really zero will be marked by the NULL dataspace, but HDF5
486 * doesn't allow me to set the size of the type to zero.)*/
487 if (att->nc_typeid == NC_CHAR)
488 {
489 size_t string_size = dims[0];
490 if (!string_size)
491 {
492 string_size = 1;
493 if ((spaceid = H5Screate(H5S_NULL)) < 0)
494 BAIL(NC_EATTMETA);
495 }
496 else
497 {
498 if ((spaceid = H5Screate(H5S_SCALAR)) < 0)
499 BAIL(NC_EATTMETA);
500 }
501 if (H5Tset_size(file_typeid, string_size) < 0)
502 BAIL(NC_EATTMETA);
503 if (H5Tset_strpad(file_typeid, H5T_STR_NULLTERM) < 0)
504 BAIL(NC_EATTMETA);
505 }
506 else
507 {
508 if (!att->len)
509 {
510 if ((spaceid = H5Screate(H5S_NULL)) < 0)
511 BAIL(NC_EATTMETA);
512 }
513 else
514 {
515 if ((spaceid = H5Screate_simple(1, dims, NULL)) < 0)
516 BAIL(NC_EATTMETA);
517 }
518 }
519
520 /* Does the att exists already? */
521 if ((attr_exists = H5Aexists(locid, att->hdr.name)) < 0)
522 BAIL(NC_EHDFERR);
523 if (attr_exists)
524 {
525 hssize_t npoints;
526
527 /* Open the attribute. */
528 if ((existing_attid = H5Aopen(locid, att->hdr.name, H5P_DEFAULT)) < 0)
529 BAIL(NC_EATTMETA);
530
531 /* Find the type of the existing attribute. */
532 if ((existing_att_typeid = H5Aget_type(existing_attid)) < 0)
533 BAIL(NC_EATTMETA);
534
535 /* How big is the attribute? */
536 if ((existing_spaceid = H5Aget_space(existing_attid)) < 0)
537 BAIL(NC_EATTMETA);
538 if ((npoints = H5Sget_simple_extent_npoints(existing_spaceid)) < 0)
539 BAIL(NC_EATTMETA);
540
541 /* For text attributes the size is specified in the datatype
542 and it is enough to compare types using H5Tequal(). */
543 if (!H5Tequal(file_typeid, existing_att_typeid) ||
544 (att->nc_typeid != NC_CHAR && npoints != att->len))
545 {
546 /* The attribute exists but we cannot re-use it. */
547
548 /* Delete the attribute. */
549 if (H5Adelete(locid, att->hdr.name) < 0)
550 BAIL(NC_EHDFERR);
551
552 /* Re-create the attribute with the type and length
553 reflecting the new value (or values). */
554 if ((attid = H5Acreate1(locid, att->hdr.name, file_typeid, spaceid,
555 H5P_DEFAULT)) < 0)
556 BAIL(NC_EATTMETA);
557
558 /* Write the values, (even if length is zero). */
559 if (H5Awrite(attid, file_typeid, data) < 0)
560 BAIL(NC_EATTMETA);
561 }
562 else
563 {
564 /* The attribute exists and we can re-use it. */
565
566 /* Write the values, re-using the existing attribute. */
567 if (H5Awrite(existing_attid, file_typeid, data) < 0)
568 BAIL(NC_EATTMETA);
569 }
570 }
571 else
572 {
573 /* The attribute does not exist yet. */
574
575 /* Create the attribute. */
576 if ((attid = H5Acreate1(locid, att->hdr.name, file_typeid, spaceid,
577 H5P_DEFAULT)) < 0)
578 BAIL(NC_EATTMETA);
579
580 /* Write the values, (even if length is zero). */
581 if (H5Awrite(attid, file_typeid, data) < 0)
582 BAIL(NC_EATTMETA);
583 }
584
585exit:
586 if (file_typeid && H5Tclose(file_typeid))
587 BAIL2(NC_EHDFERR);
588 if (attid > 0 && H5Aclose(attid) < 0)
589 BAIL2(NC_EHDFERR);
590 if (existing_att_typeid && H5Tclose(existing_att_typeid))
591 BAIL2(NC_EHDFERR);
592 if (existing_attid > 0 && H5Aclose(existing_attid) < 0)
593 BAIL2(NC_EHDFERR);
594 if (spaceid > 0 && H5Sclose(spaceid) < 0)
595 BAIL2(NC_EHDFERR);
596 if (existing_spaceid > 0 && H5Sclose(existing_spaceid) < 0)
597 BAIL2(NC_EHDFERR);
598 return retval;
599}
600
612static int
613write_attlist(NCindex *attlist, int varid, NC_GRP_INFO_T *grp)
614{
615 NC_ATT_INFO_T *att;
616 int retval;
617 int i;
618
619 for(i = 0; i < ncindexsize(attlist); i++)
620 {
621 att = (NC_ATT_INFO_T *)ncindexith(attlist, i);
622 assert(att);
623 if (att->dirty)
624 {
625 LOG((4, "%s: writing att %s to varid %d", __func__, att->hdr.name, varid));
626 if ((retval = put_att_grpa(grp, varid, att)))
627 return retval;
628 att->dirty = NC_FALSE;
629 att->created = NC_TRUE;
630 }
631 }
632 return NC_NOERR;
633}
634
648static int
649write_coord_dimids(NC_VAR_INFO_T *var)
650{
651 NC_HDF5_VAR_INFO_T *hdf5_var;
652 hsize_t coords_len[1];
653 hid_t c_spaceid = -1, c_attid = -1;
654 int retval = NC_NOERR;
655
656 assert(var && var->format_var_info);
657
658 /* Get HDF5-specific var info. */
659 hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
660
661 /* Set up space for attribute. */
662 coords_len[0] = var->ndims;
663 if ((c_spaceid = H5Screate_simple(1, coords_len, coords_len)) < 0)
664 BAIL(NC_EHDFERR);
665
666 /* Create the attribute. */
667 if ((c_attid = H5Acreate1(hdf5_var->hdf_datasetid, COORDINATES,
668 H5T_NATIVE_INT, c_spaceid, H5P_DEFAULT)) < 0)
669 BAIL(NC_EHDFERR);
670
671 /* Write our attribute. */
672 if (H5Awrite(c_attid, H5T_NATIVE_INT, var->dimids) < 0)
673 BAIL(NC_EHDFERR);
674
675exit:
676 if (c_spaceid >= 0 && H5Sclose(c_spaceid) < 0)
677 BAIL2(NC_EHDFERR);
678 if (c_attid >= 0 && H5Aclose(c_attid) < 0)
679 BAIL2(NC_EHDFERR);
680 return retval;
681}
682
693static int
694write_netcdf4_dimid(hid_t datasetid, int dimid)
695{
696 hid_t dimid_spaceid = -1, dimid_attid = -1;
697 htri_t attr_exists;
698 int retval = NC_NOERR;
699
700 /* Create the space. */
701 if ((dimid_spaceid = H5Screate(H5S_SCALAR)) < 0)
702 BAIL(NC_EHDFERR);
703
704 /* Does the attribute already exist? If so, don't try to create it. */
705 if ((attr_exists = H5Aexists(datasetid, NC_DIMID_ATT_NAME)) < 0)
706 BAIL(NC_EHDFERR);
707 if (attr_exists)
708 dimid_attid = H5Aopen_by_name(datasetid, ".", NC_DIMID_ATT_NAME,
709 H5P_DEFAULT, H5P_DEFAULT);
710 else
711 /* Create the attribute if needed. */
712 dimid_attid = H5Acreate1(datasetid, NC_DIMID_ATT_NAME,
713 H5T_NATIVE_INT, dimid_spaceid, H5P_DEFAULT);
714 if (dimid_attid < 0)
715 BAIL(NC_EHDFERR);
716
717
718 /* Write it. */
719 LOG((4, "%s: writing secret dimid %d", __func__, dimid));
720 if (H5Awrite(dimid_attid, H5T_NATIVE_INT, &dimid) < 0)
721 BAIL(NC_EHDFERR);
722
723exit:
724 /* Close stuff*/
725 if (dimid_spaceid >= 0 && H5Sclose(dimid_spaceid) < 0)
726 BAIL2(NC_EHDFERR);
727 if (dimid_attid >= 0 && H5Aclose(dimid_attid) < 0)
728 BAIL2(NC_EHDFERR);
729
730 return retval;
731}
732
747static int
748var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid)
749{
750 NC_HDF5_GRP_INFO_T *hdf5_grp;
751 NC_HDF5_VAR_INFO_T *hdf5_var;
752 hid_t plistid = 0, access_plistid = 0, typeid = 0, spaceid = 0;
753 hsize_t chunksize[H5S_MAX_RANK], dimsize[H5S_MAX_RANK], maxdimsize[H5S_MAX_RANK];
754 int d;
755 void *fillp = NULL;
756 NC_DIM_INFO_T *dim = NULL;
757 char *name_to_use;
758 int retval;
759 unsigned int* params = NULL;
760
761 assert(grp && grp->format_grp_info && var && var->format_var_info);
762
763 LOG((3, "%s:: name %s", __func__, var->hdr.name));
764
765 /* Get HDF5-specific group and var info. */
766 hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
767 hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
768
769 /* Scalar or not, we need a creation property list. */
770 if ((plistid = H5Pcreate(H5P_DATASET_CREATE)) < 0)
771 BAIL(NC_EHDFERR);
772 if ((access_plistid = H5Pcreate(H5P_DATASET_ACCESS)) < 0)
773 BAIL(NC_EHDFERR);
774
775 /* Turn off object tracking times in HDF5. */
776 if (H5Pset_obj_track_times(plistid, 0) < 0)
777 BAIL(NC_EHDFERR);
778
779 /* Find the HDF5 type of the dataset. */
780 if ((retval = nc4_get_hdf_typeid(grp->nc4_info, var->type_info->hdr.id, &typeid,
781 var->type_info->endianness)))
782 BAIL(retval);
783
784 /* Figure out what fill value to set, if any. */
785 if (var->no_fill)
786 {
787 /* Required to truly turn HDF5 fill values off */
788 if (H5Pset_fill_time(plistid, H5D_FILL_TIME_NEVER) < 0)
789 BAIL(NC_EHDFERR);
790 }
791 else
792 {
793 if ((retval = nc4_get_fill_value(grp->nc4_info, var, &fillp)))
794 BAIL(retval);
795
796 /* If there is a fill value, set it. */
797 if (fillp)
798 {
799 if (var->type_info->nc_type_class == NC_STRING)
800 {
801 if (H5Pset_fill_value(plistid, typeid, fillp) < 0)
802 BAIL(NC_EHDFERR);
803 }
804 else
805 {
806 /* The fill value set in HDF5 must always be presented as
807 * a native type, even if the endianness for this dataset
808 * is non-native. HDF5 will translate the fill value to
809 * the target endiannesss. */
810 hid_t fill_typeid = 0;
811
812 if ((retval = nc4_get_hdf_typeid(grp->nc4_info, var->type_info->hdr.id, &fill_typeid,
814 BAIL(retval);
815 if (H5Pset_fill_value(plistid, fill_typeid, fillp) < 0)
816 {
817 if (H5Tclose(fill_typeid) < 0)
818 BAIL(NC_EHDFERR);
819 BAIL(NC_EHDFERR);
820 }
821 if (H5Tclose(fill_typeid) < 0)
822 BAIL(NC_EHDFERR);
823 }
824 }
825 }
826
827 /* If the user wants to compress the data, using either zlib
828 * (a.k.a deflate) or szip, or another filter, set that up now.
829 * Szip and zip can be turned on
830 * either directly with nc_def_var_szip/deflate(), or using
831 * nc_def_var_filter(). If the user
832 * has specified a filter, it will be applied here. */
833 if(var->filters != NULL) {
834 int j;
835 NClist* filters = (NClist*)var->filters;
836 for(j=0;j<nclistlength(filters);j++) {
837 struct NC_HDF5_Filter* fi = (struct NC_HDF5_Filter*)nclistget(filters,j);
838 if(fi->filterid == H5Z_FILTER_FLETCHER32) {
839 if(H5Pset_fletcher32(plistid) < 0)
840 BAIL(NC_EHDFERR);
841 } else if(fi->filterid == H5Z_FILTER_SHUFFLE) {
842 if(H5Pset_shuffle(plistid) < 0)
843 BAIL(NC_EHDFERR);
844 } else if(fi->filterid == H5Z_FILTER_DEFLATE) {/* Handle zip case here */
845 unsigned level;
846 if(fi->nparams != 1)
847 BAIL(NC_EFILTER);
848 level = (int)fi->params[0];
849 if(H5Pset_deflate(plistid, level) < 0)
850 BAIL(NC_EFILTER);
851 } else if(fi->filterid == H5Z_FILTER_SZIP) {/* Handle szip case here */
852 int options_mask;
853 int bits_per_pixel;
854 if(fi->nparams != 2)
855 BAIL(NC_EFILTER);
856 options_mask = (int)fi->params[0];
857 bits_per_pixel = (int)fi->params[1];
858 if(H5Pset_szip(plistid, options_mask, bits_per_pixel) < 0)
859 BAIL(NC_EFILTER);
860 } else {
861 herr_t code = H5Pset_filter(plistid, fi->filterid,
862#if 1
863 H5Z_FLAG_MANDATORY,
864#else
865 H5Z_FLAG_OPTIONAL,
866#endif
867 fi->nparams, fi->params);
868 if(code < 0)
869 BAIL(NC_EFILTER);
870 }
871 }
872 }
873
874 /* If ndims non-zero, get info for all dimensions. We look up the
875 dimids and get the len of each dimension. We need this to create
876 the space for the dataset. In netCDF a dimension length of zero
877 means an unlimited dimension. */
878 if (var->ndims)
879 {
880 int unlimdim = 0;
881
882 /* Check to see if any unlimited dimensions are used in this var. */
883 for (d = 0; d < var->ndims; d++) {
884 dim = var->dim[d];
885 assert(dim && dim->hdr.id == var->dimids[d]);
886 if (dim->unlimited)
887 unlimdim++;
888 }
889
890 /* If there are no unlimited dims, and no filters, and the user
891 * has not specified chunksizes, use contiguous variable for
892 * better performance. */
893 if (nclistlength((NClist*)var->filters) == 0 &&
894 (var->chunksizes == NULL || !var->chunksizes[0]) && !unlimdim)
895 var->storage = NC_CONTIGUOUS;
896
897 /* Gather current & maximum dimension sizes, along with chunk
898 * sizes. */
899 for (d = 0; d < var->ndims; d++)
900 {
901 dim = var->dim[d];
902 assert(dim && dim->hdr.id == var->dimids[d]);
903 dimsize[d] = dim->unlimited ? NC_HDF5_UNLIMITED_DIMSIZE : dim->len;
904 maxdimsize[d] = dim->unlimited ? H5S_UNLIMITED : (hsize_t)dim->len;
905 if (var->storage == NC_CHUNKED)
906 {
907 if (var->chunksizes[d])
908 chunksize[d] = var->chunksizes[d];
909 else
910 {
911 size_t type_size;
912 if (var->type_info->nc_type_class == NC_STRING)
913 type_size = sizeof(char *);
914 else
915 type_size = var->type_info->size;
916
917 /* Unlimited dim always gets chunksize of 1. */
918 if (dim->unlimited)
919 chunksize[d] = 1;
920 else
921 chunksize[d] = pow((double)DEFAULT_CHUNK_SIZE/type_size,
922 1/(double)(var->ndims - unlimdim));
923
924 /* If the chunksize is greater than the dim
925 * length, make it the dim length. */
926 if (!dim->unlimited && chunksize[d] > dim->len)
927 chunksize[d] = dim->len;
928
929 /* Remember the computed chunksize */
930 var->chunksizes[d] = chunksize[d];
931 }
932 }
933 }
934
935 /* Create the dataspace. */
936 if ((spaceid = H5Screate_simple(var->ndims, dimsize, maxdimsize)) < 0)
937 BAIL(NC_EHDFERR);
938 }
939 else
940 {
941 if ((spaceid = H5Screate(H5S_SCALAR)) < 0)
942 BAIL(NC_EHDFERR);
943 }
944
945 /* Set the var storage to contiguous, compact, or chunked. Don't
946 * try to set chunking for scalar vars, they will default to
947 * contiguous if not set to compact. */
948 if (var->storage == NC_CONTIGUOUS)
949 {
950 if (H5Pset_layout(plistid, H5D_CONTIGUOUS) < 0)
951 BAIL(NC_EHDFERR);
952 }
953 else if (var->storage == NC_COMPACT)
954 {
955 if (H5Pset_layout(plistid, H5D_COMPACT) < 0)
956 BAIL(NC_EHDFERR);
957 }
958 else if (var->ndims)
959 {
960 if (H5Pset_chunk(plistid, var->ndims, chunksize) < 0)
961 BAIL(NC_EHDFERR);
962 }
963
964 /* Turn on creation order tracking. */
965 if (!grp->nc4_info->no_attr_create_order) {
966 if (H5Pset_attr_creation_order(plistid, H5P_CRT_ORDER_TRACKED|
967 H5P_CRT_ORDER_INDEXED) < 0)
968 BAIL(NC_EHDFERR);
969 }
970
971 /* Set per-var chunk cache, for chunked datasets. */
972 if (var->storage == NC_CHUNKED && var->chunkcache.size)
973 if (H5Pset_chunk_cache(access_plistid, var->chunkcache.nelems,
974 var->chunkcache.size, var->chunkcache.preemption) < 0)
975 BAIL(NC_EHDFERR);
976
977 /* At long last, create the dataset. */
978 name_to_use = var->alt_name ? var->alt_name : var->hdr.name;
979 LOG((4, "%s: about to H5Dcreate2 dataset %s of type 0x%x", __func__,
980 name_to_use, typeid));
981 if ((hdf5_var->hdf_datasetid = H5Dcreate2(hdf5_grp->hdf_grpid, name_to_use, typeid,
982 spaceid, H5P_DEFAULT, plistid, access_plistid)) < 0)
983 BAIL(NC_EHDFERR);
984 var->created = NC_TRUE;
985 var->is_new_var = NC_FALSE;
986
987 /* Always write the hidden coordinates attribute, which lists the
988 * dimids of this var. When present, this speeds opens. When not
989 * present, dimscale matching is used. */
990 if (var->ndims)
991 if ((retval = write_coord_dimids(var)))
992 BAIL(retval);
993
994 /* If this is a dimscale, mark it as such in the HDF5 file. Also
995 * find the dimension info and store the dataset id of the dimscale
996 * dataset. */
997 if (hdf5_var->dimscale)
998 {
999 if (H5DSset_scale(hdf5_var->hdf_datasetid, var->hdr.name) < 0)
1000 BAIL(NC_EHDFERR);
1001
1002 /* If this is a multidimensional coordinate variable, write a
1003 * coordinates attribute. */
1004 /* if (var->ndims > 1) */
1005 /* if ((retval = write_coord_dimids(var))) */
1006 /* BAIL(retval); */
1007
1008 /* If desired, write the netCDF dimid. */
1009 if (write_dimid)
1010 if ((retval = write_netcdf4_dimid(hdf5_var->hdf_datasetid, var->dimids[0])))
1011 BAIL(retval);
1012 }
1013
1014 /* If quantization is in use, write an attribute indicating it, a
1015 * single integer which is the number of significant digits
1016 * (NSD, for BitGroom and Granular BitRound) or number of significant bits
1017 * (NSB, for BitRound). */
1018 if (var->quantize_mode == NC_QUANTIZE_BITGROOM)
1019 if ((retval = nc4_put_att(var->container, var->hdr.id, NC_QUANTIZE_BITGROOM_ATT_NAME, NC_INT, 1,
1020 &var->nsd, NC_INT, 0)))
1021 BAIL(retval);
1022
1023 if (var->quantize_mode == NC_QUANTIZE_GRANULARBR)
1024 if ((retval = nc4_put_att(var->container, var->hdr.id, NC_QUANTIZE_GRANULARBR_ATT_NAME, NC_INT, 1,
1025 &var->nsd, NC_INT, 0)))
1026 BAIL(retval);
1027
1028 if (var->quantize_mode == NC_QUANTIZE_BITROUND)
1029 if ((retval = nc4_put_att(var->container, var->hdr.id, NC_QUANTIZE_BITROUND_ATT_NAME, NC_INT, 1,
1030 &var->nsd, NC_INT, 0)))
1031 BAIL(retval);
1032
1033 /* Write attributes for this var. */
1034 if ((retval = write_attlist(var->att, var->hdr.id, grp)))
1035 BAIL(retval);
1036
1037 /* The file is now up-to-date with all settings for this var. */
1038 var->attr_dirty = NC_FALSE;
1039
1040exit:
1041 nullfree(params);
1042 if (typeid > 0 && H5Tclose(typeid) < 0)
1043 BAIL2(NC_EHDFERR);
1044 if (plistid > 0 && H5Pclose(plistid) < 0)
1045 BAIL2(NC_EHDFERR);
1046 if (access_plistid > 0 && H5Pclose(access_plistid) < 0)
1047 BAIL2(NC_EHDFERR);
1048 if (spaceid > 0 && H5Sclose(spaceid) < 0)
1049 BAIL2(NC_EHDFERR);
1050 if (fillp)
1051 {
1052 if (var->type_info->nc_type_class == NC_VLEN)
1053 nc_free_vlen((nc_vlen_t *)fillp);
1054 else if (var->type_info->nc_type_class == NC_STRING && *(char **)fillp)
1055 free(*(char **)fillp);
1056 free(fillp);
1057 }
1058
1059 return retval;
1060}
1061
1075int
1076nc4_adjust_var_cache(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var)
1077{
1078 size_t chunk_size_bytes = 1;
1079 int d;
1080 int retval;
1081
1082 /* Nothing to be done for contiguous or compact data. */
1083 if (var->storage != NC_CHUNKED)
1084 return NC_NOERR;
1085
1086#ifdef USE_PARALLEL4
1087 /* Don't set cache for files using parallel I/O. */
1088 if (grp->nc4_info->parallel)
1089 return NC_NOERR;
1090#endif
1091
1092 /* How many bytes in the chunk? */
1093 for (d = 0; d < var->ndims; d++)
1094 chunk_size_bytes *= var->chunksizes[d];
1095 if (var->type_info->size)
1096 chunk_size_bytes *= var->type_info->size;
1097 else
1098 chunk_size_bytes *= sizeof(char *);
1099
1100 /* If the chunk cache is too small, and the user has not changed
1101 * the default value of the chunk cache size, then increase the
1102 * size of the cache. */
1103 if (var->chunkcache.size == CHUNK_CACHE_SIZE)
1104 if (chunk_size_bytes > var->chunkcache.size)
1105 {
1106 var->chunkcache.size = chunk_size_bytes * DEFAULT_CHUNKS_IN_CACHE;
1107 if (var->chunkcache.size > MAX_DEFAULT_CACHE_SIZE)
1108 var->chunkcache.size = MAX_DEFAULT_CACHE_SIZE;
1109 if ((retval = nc4_reopen_dataset(grp, var)))
1110 return retval;
1111 }
1112
1113 return NC_NOERR;
1114}
1115
1131static int
1132commit_type(NC_GRP_INFO_T *grp, NC_TYPE_INFO_T *type)
1133{
1134 NC_HDF5_GRP_INFO_T *hdf5_grp;
1135 NC_HDF5_TYPE_INFO_T *hdf5_type;
1136 hid_t base_hdf_typeid;
1137 int retval;
1138
1139 assert(grp && grp->format_grp_info && type && type->format_type_info);
1140
1141 /* Get HDF5-specific group and type info. */
1142 hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
1143 hdf5_type = (NC_HDF5_TYPE_INFO_T *)type->format_type_info;
1144
1145 /* Did we already record this type? */
1146 if (type->committed)
1147 return NC_NOERR;
1148
1149 /* Is this a compound type? */
1150 if (type->nc_type_class == NC_COMPOUND)
1151 {
1152 NC_FIELD_INFO_T *field;
1153 hid_t hdf_base_typeid, hdf_typeid;
1154 int i;
1155
1156 if ((hdf5_type->hdf_typeid = H5Tcreate(H5T_COMPOUND, type->size)) < 0)
1157 return NC_EHDFERR;
1158 LOG((4, "creating compound type %s hdf_typeid 0x%x", type->hdr.name,
1159 hdf5_type->hdf_typeid));
1160
1161 for(i=0;i<nclistlength(type->u.c.field);i++)
1162 {
1163 field = (NC_FIELD_INFO_T *)nclistget(type->u.c.field, i);
1164 assert(field);
1165 if ((retval = nc4_get_hdf_typeid(grp->nc4_info, field->nc_typeid,
1166 &hdf_base_typeid, type->endianness)))
1167 return retval;
1168
1169 /* If this is an array, create a special array type. */
1170 if (field->ndims)
1171 {
1172 int d;
1173 hsize_t dims[NC_MAX_VAR_DIMS];
1174
1175 for (d = 0; d < field->ndims; d++)
1176 dims[d] = field->dim_size[d];
1177 if ((hdf_typeid = H5Tarray_create1(hdf_base_typeid, field->ndims,
1178 dims, NULL)) < 0)
1179 {
1180 if (H5Tclose(hdf_base_typeid) < 0)
1181 return NC_EHDFERR;
1182 return NC_EHDFERR;
1183 }
1184 if (H5Tclose(hdf_base_typeid) < 0)
1185 return NC_EHDFERR;
1186 }
1187 else
1188 hdf_typeid = hdf_base_typeid;
1189 LOG((4, "inserting field %s offset %d hdf_typeid 0x%x", field->hdr.name,
1190 field->offset, hdf_typeid));
1191 if (H5Tinsert(hdf5_type->hdf_typeid, field->hdr.name, field->offset,
1192 hdf_typeid) < 0)
1193 return NC_EHDFERR;
1194 if (H5Tclose(hdf_typeid) < 0)
1195 return NC_EHDFERR;
1196 }
1197 }
1198 else if (type->nc_type_class == NC_VLEN)
1199 {
1200 /* Find the HDF typeid of the base type of this vlen. */
1201 if ((retval = nc4_get_hdf_typeid(grp->nc4_info, type->u.v.base_nc_typeid,
1202 &base_hdf_typeid, type->endianness)))
1203 return retval;
1204
1205 /* Create a vlen type. */
1206 if ((hdf5_type->hdf_typeid = H5Tvlen_create(base_hdf_typeid)) < 0)
1207 return NC_EHDFERR;
1208 }
1209 else if (type->nc_type_class == NC_OPAQUE)
1210 {
1211 /* Create the opaque type. */
1212 if ((hdf5_type->hdf_typeid = H5Tcreate(H5T_OPAQUE, type->size)) < 0)
1213 return NC_EHDFERR;
1214 }
1215 else if (type->nc_type_class == NC_ENUM)
1216 {
1217 NC_ENUM_MEMBER_INFO_T *enum_m;
1218 int i;
1219
1220 if (nclistlength(type->u.e.enum_member) == 0)
1221 return NC_EINVAL;
1222
1223 /* Find the HDF typeid of the base type of this enum. */
1224 if ((retval = nc4_get_hdf_typeid(grp->nc4_info, type->u.e.base_nc_typeid,
1225 &base_hdf_typeid, type->endianness)))
1226 return retval;
1227
1228 /* Create an enum type. */
1229 if ((hdf5_type->hdf_typeid = H5Tenum_create(base_hdf_typeid)) < 0)
1230 return NC_EHDFERR;
1231
1232 /* Add all the members to the HDF5 type. */
1233 for(i=0;i<nclistlength(type->u.e.enum_member);i++) {
1234 enum_m = (NC_ENUM_MEMBER_INFO_T*)nclistget(type->u.e.enum_member,i);
1235 if (H5Tenum_insert(hdf5_type->hdf_typeid, enum_m->name, enum_m->value) < 0)
1236 return NC_EHDFERR;
1237 }
1238 }
1239 else
1240 {
1241 LOG((0, "Unknown class: %d", type->nc_type_class));
1242 return NC_EBADTYPE;
1243 }
1244
1245 /* Commit the type. */
1246 if (H5Tcommit1(hdf5_grp->hdf_grpid, type->hdr.name, hdf5_type->hdf_typeid) < 0)
1247 return NC_EHDFERR;
1248 type->committed = NC_TRUE;
1249 LOG((4, "just committed type %s, HDF typeid: 0x%x", type->hdr.name,
1250 hdf5_type->hdf_typeid));
1251
1252 /* Later we will always use the native typeid. In this case, it is
1253 * a copy of the same type pointed to by hdf_typeid, but it's
1254 * easier to maintain a copy. */
1255 if ((hdf5_type->native_hdf_typeid = H5Tget_native_type(hdf5_type->hdf_typeid,
1256 H5T_DIR_DEFAULT)) < 0)
1257 return NC_EHDFERR;
1258
1259 return NC_NOERR;
1260}
1261
1272static int
1273write_nc3_strict_att(hid_t hdf_grpid)
1274{
1275 hid_t attid = 0, spaceid = 0;
1276 int one = 1;
1277 int retval = NC_NOERR;
1278 htri_t attr_exists;
1279
1280 /* If the attribute already exists, call that a success and return
1281 * NC_NOERR. */
1282 if ((attr_exists = H5Aexists(hdf_grpid, NC3_STRICT_ATT_NAME)) < 0)
1283 return NC_EHDFERR;
1284 if (attr_exists)
1285 return NC_NOERR;
1286
1287 /* Create the attribute to mark this as a file that needs to obey
1288 * strict netcdf-3 rules. */
1289 if ((spaceid = H5Screate(H5S_SCALAR)) < 0)
1290 BAIL(NC_EFILEMETA);
1291 if ((attid = H5Acreate1(hdf_grpid, NC3_STRICT_ATT_NAME,
1292 H5T_NATIVE_INT, spaceid, H5P_DEFAULT)) < 0)
1293 BAIL(NC_EFILEMETA);
1294 if (H5Awrite(attid, H5T_NATIVE_INT, &one) < 0)
1295 BAIL(NC_EFILEMETA);
1296
1297exit:
1298 if (spaceid > 0 && (H5Sclose(spaceid) < 0))
1299 BAIL2(NC_EFILEMETA);
1300 if (attid > 0 && (H5Aclose(attid) < 0))
1301 BAIL2(NC_EFILEMETA);
1302 return retval;
1303}
1304
1317static int
1318create_group(NC_GRP_INFO_T *grp)
1319{
1320 NC_HDF5_GRP_INFO_T *hdf5_grp, *parent_hdf5_grp;
1321 hid_t gcpl_id = -1;
1322 int retval = NC_NOERR;;
1323
1324 assert(grp && grp->format_grp_info && grp->parent &&
1325 grp->parent->format_grp_info);
1326
1327 /* Get HDF5 specific group info for group and parent. */
1328 hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
1329 parent_hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->parent->format_grp_info;
1330 assert(parent_hdf5_grp->hdf_grpid);
1331
1332 /* Create group, with link_creation_order set in the group
1333 * creation property list. */
1334 if ((gcpl_id = H5Pcreate(H5P_GROUP_CREATE)) < 0)
1335 BAIL(NC_EHDFERR);
1336
1337 /* Set track_times to be FALSE. */
1338 if (H5Pset_obj_track_times(gcpl_id, 0) < 0)
1339 BAIL(NC_EHDFERR);
1340
1341 /* Tell HDF5 to keep track of objects in creation order. */
1342 if (H5Pset_link_creation_order(gcpl_id, H5P_CRT_ORDER_TRACKED|H5P_CRT_ORDER_INDEXED) < 0)
1343 BAIL(NC_EHDFERR);
1344
1345 /* Tell HDF5 to keep track of attributes in creation order. */
1346 if (!grp->nc4_info->no_attr_create_order) {
1347 if (H5Pset_attr_creation_order(gcpl_id, H5P_CRT_ORDER_TRACKED|H5P_CRT_ORDER_INDEXED) < 0)
1348 BAIL(NC_EHDFERR);
1349 }
1350
1351 /* Create the group. */
1352 if ((hdf5_grp->hdf_grpid = H5Gcreate2(parent_hdf5_grp->hdf_grpid, grp->hdr.name,
1353 H5P_DEFAULT, gcpl_id, H5P_DEFAULT)) < 0)
1354 BAIL(NC_EHDFERR);
1355
1356exit:
1357 if (gcpl_id > -1 && H5Pclose(gcpl_id) < 0)
1358 BAIL2(NC_EHDFERR);
1359 if (retval)
1360 if (hdf5_grp->hdf_grpid > 0 && H5Gclose(hdf5_grp->hdf_grpid) < 0)
1361 BAIL2(NC_EHDFERR);
1362 return retval;
1363}
1364
1377static int
1378attach_dimscales(NC_GRP_INFO_T *grp)
1379{
1380 NC_VAR_INFO_T *var;
1381 NC_HDF5_VAR_INFO_T *hdf5_var;
1382 int d, v;
1383
1384 /* Attach dimension scales. */
1385 for (v = 0; v < ncindexsize(grp->vars); v++)
1386 {
1387 /* Get pointer to var and HDF5-specific var info. */
1388 var = (NC_VAR_INFO_T *)ncindexith(grp->vars, v);
1389 assert(var && var->format_var_info);
1390 hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
1391
1392 /* Scales themselves do not attach. But I really wish they
1393 * would. */
1394 if (hdf5_var->dimscale)
1395 continue;
1396
1397 /* Find the scale for each dimension, if any, and attach it. */
1398 for (d = 0; d < var->ndims; d++)
1399 {
1400 /* Is there a dimscale for this dimension? */
1401 if (hdf5_var->dimscale_attached)
1402 {
1403 if (!hdf5_var->dimscale_attached[d])
1404 {
1405 hid_t dsid; /* Dataset ID for dimension */
1406 assert(var->dim[d] && var->dim[d]->hdr.id == var->dimids[d] &&
1407 var->dim[d]->format_dim_info);
1408
1409 LOG((2, "%s: attaching scale for dimid %d to var %s",
1410 __func__, var->dimids[d], var->hdr.name));
1411
1412 /* Find dataset ID for dimension */
1413 if (var->dim[d]->coord_var)
1414 dsid = ((NC_HDF5_VAR_INFO_T *)(var->dim[d]->coord_var->format_var_info))->hdf_datasetid;
1415 else
1416 dsid = ((NC_HDF5_DIM_INFO_T *)var->dim[d]->format_dim_info)->hdf_dimscaleid;
1417 assert(dsid > 0);
1418
1419 /* Attach the scale. */
1420 if (H5DSattach_scale(hdf5_var->hdf_datasetid, dsid, d) < 0)
1421 return NC_EDIMSCALE;
1422 hdf5_var->dimscale_attached[d] = NC_TRUE;
1423 }
1424 }
1425 }
1426 }
1427
1428 return NC_NOERR;
1429}
1430
1441static int
1442var_exists(hid_t grpid, char *name, nc_bool_t *exists)
1443{
1444 htri_t link_exists;
1445
1446 /* Reset the boolean */
1447 *exists = NC_FALSE;
1448
1449 /* Check if the object name exists in the group */
1450 if ((link_exists = H5Lexists(grpid, name, H5P_DEFAULT)) < 0)
1451 return NC_EHDFERR;
1452 if (link_exists)
1453 {
1454 H5G_stat_t statbuf;
1455
1456 /* Get info about the object */
1457 if (H5Gget_objinfo(grpid, name, 1, &statbuf) < 0)
1458 return NC_EHDFERR;
1459
1460 if (H5G_DATASET == statbuf.type)
1461 *exists = NC_TRUE;
1462 }
1463
1464 return NC_NOERR;
1465}
1466
1482static int
1483remove_coord_atts(hid_t hdf_datasetid)
1484{
1485 htri_t attr_exists;
1486
1487 /* If the variable dataset has an optional NC_DIMID_ATT_NAME
1488 * attribute, delete it. */
1489 if ((attr_exists = H5Aexists(hdf_datasetid, NC_DIMID_ATT_NAME)) < 0)
1490 return NC_EHDFERR;
1491 if (attr_exists)
1492 {
1493 if (H5Adelete(hdf_datasetid, NC_DIMID_ATT_NAME) < 0)
1494 return NC_EHDFERR;
1495 }
1496
1497 /* Remove the dimension scale 'CLASS' & 'NAME' attributes. */
1498 if ((attr_exists = H5Aexists(hdf_datasetid,
1499 HDF5_DIMSCALE_CLASS_ATT_NAME)) < 0)
1500 return NC_EHDFERR;
1501 if (attr_exists)
1502 {
1503 if (H5Adelete(hdf_datasetid, HDF5_DIMSCALE_CLASS_ATT_NAME) < 0)
1504 return NC_EHDFERR;
1505 }
1506 if ((attr_exists = H5Aexists(hdf_datasetid,
1507 HDF5_DIMSCALE_NAME_ATT_NAME)) < 0)
1508 return NC_EHDFERR;
1509 if (attr_exists)
1510 {
1511 if (H5Adelete(hdf_datasetid, HDF5_DIMSCALE_NAME_ATT_NAME) < 0)
1512 return NC_EHDFERR;
1513 }
1514 return NC_NOERR;
1515}
1516
1531static int
1532write_var(NC_VAR_INFO_T *var, NC_GRP_INFO_T *grp, nc_bool_t write_dimid)
1533{
1534 NC_HDF5_GRP_INFO_T *hdf5_grp;
1535 NC_HDF5_VAR_INFO_T *hdf5_var;
1536 nc_bool_t replace_existing_var = NC_FALSE;
1537 int retval;
1538
1539 assert(var && var->format_var_info && grp && grp->format_grp_info);
1540
1541 LOG((4, "%s: writing var %s", __func__, var->hdr.name));
1542
1543 /* Get HDF5-specific group and var info. */
1544 hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
1545 hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
1546
1547 /* If the variable has already been created & the fill value changed,
1548 * indicate that the existing variable should be replaced. */
1549 if (var->created && var->fill_val_changed)
1550 {
1551 replace_existing_var = NC_TRUE;
1552 var->fill_val_changed = NC_FALSE;
1553 /* If the variable is going to be replaced, we need to flag any
1554 other attributes associated with the variable as 'dirty', or
1555 else *only* the fill value attribute will be copied over and
1556 the rest will be lost. See
1557 https://github.com/Unidata/netcdf-c/issues/239 */
1558 flag_atts_dirty(var->att);
1559 }
1560
1561 /* Is this a coordinate var that has already been created in
1562 * the HDF5 file as a dimscale dataset? Check for dims with the
1563 * same name in this group. If there is one, check to see if
1564 * this object exists in the HDF group. */
1565 if (var->became_coord_var)
1566 {
1567 if ((NC_DIM_INFO_T *)ncindexlookup(grp->dim, var->hdr.name))
1568 {
1569 nc_bool_t exists;
1570
1571 if ((retval = var_exists(hdf5_grp->hdf_grpid, var->hdr.name, &exists)))
1572 return retval;
1573 if (exists)
1574 {
1575 /* Indicate that the variable already exists, and should
1576 * be replaced. */
1577 replace_existing_var = NC_TRUE;
1578 flag_atts_dirty(var->att);
1579 }
1580 }
1581 }
1582
1583 /* Check dims if the variable will be replaced, so that the
1584 * dimensions will be de-attached and re-attached correctly. */
1585 if (replace_existing_var)
1586 {
1587 NC_DIM_INFO_T *d1;
1588
1589 /* Is there a dim with this var's name? */
1590 if ((d1 = (NC_DIM_INFO_T *)ncindexlookup(grp->dim, var->hdr.name)))
1591 {
1592 nc_bool_t exists;
1593 assert(d1->format_dim_info && d1->hdr.name);
1594
1595 if ((retval = var_exists(hdf5_grp->hdf_grpid, var->hdr.name, &exists)))
1596 return retval;
1597 if (exists)
1598 {
1599 hid_t dsid;
1600
1601 /* Find dataset ID for dimension */
1602 if (d1->coord_var)
1603 dsid = ((NC_HDF5_VAR_INFO_T *)d1->coord_var->format_var_info)->hdf_datasetid;
1604 else
1605 dsid = ((NC_HDF5_DIM_INFO_T *)d1->format_dim_info)->hdf_dimscaleid;
1606 assert(dsid > 0);
1607
1608 /* If we're replacing an existing dimscale dataset, go to
1609 * every var in the file and detach this dimension scale,
1610 * because we have to delete it. */
1611 if ((retval = rec_detach_scales(grp->nc4_info->root_grp,
1612 var->dimids[0], dsid)))
1613 return retval;
1614 }
1615 }
1616 }
1617
1618 /* If this is not a dimension scale, remove any attached scales,
1619 * and delete dimscale attributes from the var. */
1620 if (var->was_coord_var && hdf5_var->dimscale_attached)
1621 {
1622 int d;
1623
1624 /* If the variable already exists in the file, Remove any dimension scale
1625 * attributes from it, if they exist. */
1626 if (var->created)
1627 if ((retval = remove_coord_atts(hdf5_var->hdf_datasetid)))
1628 return retval;
1629
1630 /* If this is a regular var, detach all its dim scales. */
1631 for (d = 0; d < var->ndims; d++)
1632 {
1633 if (hdf5_var->dimscale_attached[d])
1634 {
1635 hid_t dsid; /* Dataset ID for dimension */
1636 assert(var->dim[d] && var->dim[d]->hdr.id == var->dimids[d] &&
1637 var->dim[d]->format_dim_info);
1638
1639 /* Find dataset ID for dimension */
1640 if (var->dim[d]->coord_var)
1641 dsid = ((NC_HDF5_VAR_INFO_T *)var->dim[d]->coord_var->format_var_info)->hdf_datasetid;
1642 else
1643 dsid = ((NC_HDF5_DIM_INFO_T *)var->dim[d]->format_dim_info)->hdf_dimscaleid;
1644 assert(dsid > 0);
1645
1646 /* Detach this dim scale. */
1647 if (H5DSdetach_scale(hdf5_var->hdf_datasetid, dsid, d) < 0)
1648 return NC_EHDFERR;
1649 hdf5_var->dimscale_attached[d] = NC_FALSE;
1650 }
1651 }
1652 }
1653
1654 /* Delete the HDF5 dataset that is to be replaced. */
1655 if (replace_existing_var)
1656 {
1657 /* Free the HDF5 dataset id. */
1658 if (hdf5_var->hdf_datasetid && H5Dclose(hdf5_var->hdf_datasetid) < 0)
1659 return NC_EHDFERR;
1660 hdf5_var->hdf_datasetid = 0;
1661
1662 /* Now delete the variable. */
1663 if (H5Gunlink(hdf5_grp->hdf_grpid, var->hdr.name) < 0)
1664 return NC_EDIMMETA;
1665 }
1666
1667 /* Create the dataset. */
1668 if (var->is_new_var || replace_existing_var)
1669 {
1670 if ((retval = var_create_dataset(grp, var, write_dimid)))
1671 return retval;
1672 }
1673 else
1674 {
1675 if (write_dimid && var->ndims)
1676 if ((retval = write_netcdf4_dimid(hdf5_var->hdf_datasetid,
1677 var->dimids[0])))
1678 return retval;
1679 }
1680
1681 if (replace_existing_var)
1682 {
1683 /* If this is a dimension scale, reattach the scale everywhere it
1684 * is used. (Recall that netCDF dimscales are always 1-D). */
1685 if(hdf5_var->dimscale)
1686 {
1687 if ((retval = rec_reattach_scales(grp->nc4_info->root_grp,
1688 var->dimids[0], hdf5_var->hdf_datasetid)))
1689 return retval;
1690 }
1691 /* If it's not a dimension scale, clear the dimscale attached flags,
1692 * so the dimensions are re-attached. */
1693 else
1694 {
1695 if (hdf5_var->dimscale_attached)
1696 memset(hdf5_var->dimscale_attached, 0, sizeof(nc_bool_t) * var->ndims);
1697 }
1698 }
1699
1700 /* Clear coord. var state transition flags */
1701 var->was_coord_var = NC_FALSE;
1702 var->became_coord_var = NC_FALSE;
1703
1704 /* Now check the attributes for this var. */
1705 if (var->attr_dirty)
1706 {
1707 /* Write attributes for this var. */
1708 if ((retval = write_attlist(var->att, var->hdr.id, grp)))
1709 return retval;
1710 var->attr_dirty = NC_FALSE;
1711 }
1712
1713 return NC_NOERR;
1714}
1715
1729int
1730nc4_create_dim_wo_var(NC_DIM_INFO_T *dim)
1731{
1732 NC_HDF5_DIM_INFO_T *hdf5_dim;
1733 NC_HDF5_GRP_INFO_T *hdf5_grp;
1734 hid_t spaceid = -1, create_propid = -1;
1735 hsize_t dims[1], max_dims[1], chunk_dims[1] = {1};
1736 char dimscale_wo_var[NC_MAX_NAME];
1737 int retval = NC_NOERR;
1738
1739 LOG((4, "%s: creating dim %s", __func__, dim->hdr.name));
1740
1741 /* Sanity check */
1742 assert(!dim->coord_var);
1743
1744 /* Get HDF5-specific dim and group info. */
1745 hdf5_grp = (NC_HDF5_GRP_INFO_T *)dim->container->format_grp_info;
1746 hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info;
1747
1748 /* Create a property list. */
1749 if ((create_propid = H5Pcreate(H5P_DATASET_CREATE)) < 0)
1750 BAIL(NC_EHDFERR);
1751
1752 /* Turn off recording of times associated with this object. */
1753 if (H5Pset_obj_track_times(create_propid, 0) < 0)
1754 BAIL(NC_EHDFERR);
1755
1756 /* Set size of dataset to size of dimension. */
1757 dims[0] = dim->len;
1758 max_dims[0] = dim->len;
1759
1760 /* If this dimension scale is unlimited (i.e. it's an unlimited
1761 * dimension), then set up chunking, with a chunksize of 1. */
1762 if (dim->unlimited)
1763 {
1764 max_dims[0] = H5S_UNLIMITED;
1765 if (H5Pset_chunk(create_propid, 1, chunk_dims) < 0)
1766 BAIL(NC_EHDFERR);
1767 }
1768
1769 /* Set up space. */
1770 if ((spaceid = H5Screate_simple(1, dims, max_dims)) < 0)
1771 BAIL(NC_EHDFERR);
1772
1773 /* Turn on creation-order tracking. */
1774 if (!dim->container->nc4_info->no_attr_create_order) {
1775 if (H5Pset_attr_creation_order(create_propid, H5P_CRT_ORDER_TRACKED|
1776 H5P_CRT_ORDER_INDEXED) < 0)
1777 BAIL(NC_EHDFERR);
1778 }
1779 /* Create the dataset that will be the dimension scale. */
1780 LOG((4, "%s: about to H5Dcreate1 a dimscale dataset %s", __func__,
1781 dim->hdr.name));
1782 if ((hdf5_dim->hdf_dimscaleid = H5Dcreate2(hdf5_grp->hdf_grpid, dim->hdr.name,
1783 H5T_IEEE_F32BE, spaceid,
1784 H5P_DEFAULT, create_propid,
1785 H5P_DEFAULT)) < 0)
1786 BAIL(NC_EHDFERR);
1787
1788 /* Indicate that this is a scale. Also indicate that not
1789 * be shown to the user as a variable. It is hidden. It is
1790 * a DIM WITHOUT A VARIABLE! */
1791 sprintf(dimscale_wo_var, "%s%10d", DIM_WITHOUT_VARIABLE, (int)dim->len);
1792 if (H5DSset_scale(hdf5_dim->hdf_dimscaleid, dimscale_wo_var) < 0)
1793 BAIL(NC_EHDFERR);
1794
1795 /* Since this dimension was created out of order, we cannot rely on
1796 * it getting the correct dimid on file open. We must assign it
1797 * explicitly. */
1798 if ((retval = write_netcdf4_dimid(hdf5_dim->hdf_dimscaleid, dim->hdr.id)))
1799 BAIL(retval);
1800
1801exit:
1802 if (spaceid > 0 && H5Sclose(spaceid) < 0)
1803 BAIL2(NC_EHDFERR);
1804 if (create_propid > 0 && H5Pclose(create_propid) < 0)
1805 BAIL2(NC_EHDFERR);
1806 return retval;
1807}
1808
1821static int
1822write_dim(NC_DIM_INFO_T *dim, NC_GRP_INFO_T *grp, nc_bool_t write_dimid)
1823{
1824 NC_HDF5_DIM_INFO_T *hdf5_dim;
1825 int retval = NC_NOERR;
1826
1827 assert(dim && dim->format_dim_info && grp && grp->format_grp_info);
1828
1829 /* Get HDF5-specific dim and group info. */
1830 hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info;
1831
1832 /* If there's no dimscale dataset for this dim, create one,
1833 * and mark that it should be hidden from netCDF as a
1834 * variable. (That is, it should appear as a dimension
1835 * without an associated variable.) */
1836 if (!hdf5_dim->hdf_dimscaleid)
1837 if ((retval = nc4_create_dim_wo_var(dim)))
1838 BAIL(retval);
1839
1840 /* Did we extend an unlimited dimension? */
1841 if (dim->extended)
1842 {
1843 NC_VAR_INFO_T *v1 = NULL;
1844
1845 assert(dim->unlimited);
1846
1847 /* If this is a dimension with an associated coordinate var,
1848 * then update the length of that coord var. */
1849 v1 = dim->coord_var;
1850 if (v1)
1851 {
1852 NC_HDF5_VAR_INFO_T *hdf5_v1;
1853 hsize_t *new_size;
1854 int d1;
1855
1856 hdf5_v1 = (NC_HDF5_VAR_INFO_T *)v1->format_var_info;
1857
1858 /* Extend the dimension scale dataset to reflect the new
1859 * length of the dimension. */
1860 if (!(new_size = malloc(v1->ndims * sizeof(hsize_t))))
1861 BAIL(NC_ENOMEM);
1862 for (d1 = 0; d1 < v1->ndims; d1++)
1863 {
1864 assert(v1->dim[d1] && v1->dim[d1]->hdr.id == v1->dimids[d1]);
1865 new_size[d1] = v1->dim[d1]->len;
1866 }
1867 if (H5Dset_extent(hdf5_v1->hdf_datasetid, new_size) < 0)
1868 BAIL(NC_EHDFERR);
1869 free(new_size);
1870 }
1871 }
1872
1873 /* If desired, write the secret dimid. This will be used instead of
1874 * the dimid that the dimension would otherwise receive based on
1875 * creation order. This can be necessary when dims and their
1876 * coordinate variables were created in different order. */
1877 if (write_dimid && hdf5_dim->hdf_dimscaleid)
1878 if ((retval = write_netcdf4_dimid(hdf5_dim->hdf_dimscaleid, dim->hdr.id)))
1879 BAIL(retval);
1880
1881exit:
1882
1883 return retval;
1884}
1885
1898int
1899nc4_rec_write_metadata(NC_GRP_INFO_T *grp, nc_bool_t bad_coord_order)
1900{
1901 NC_DIM_INFO_T *dim = NULL;
1902 NC_VAR_INFO_T *var = NULL;
1903 NC_GRP_INFO_T *child_grp = NULL;
1904 int coord_varid = -1;
1905 int var_index = 0;
1906 int dim_index = 0;
1907 int retval;
1908 int i;
1909
1910 assert(grp && grp->hdr.name &&
1911 ((NC_HDF5_GRP_INFO_T *)(grp->format_grp_info))->hdf_grpid);
1912 LOG((3, "%s: grp->hdr.name %s, bad_coord_order %d", __func__, grp->hdr.name,
1913 bad_coord_order));
1914
1915 /* Write global attributes for this group. */
1916 if ((retval = write_attlist(grp->att, NC_GLOBAL, grp)))
1917 return retval;
1918
1919 /* Set the pointers to the beginning of the list of dims & vars in this
1920 * group. */
1921 dim = (NC_DIM_INFO_T *)ncindexith(grp->dim, dim_index);
1922 var = (NC_VAR_INFO_T *)ncindexith(grp->vars, var_index);
1923
1924 /* Because of HDF5 ordering the dims and vars have to be stored in
1925 * this way to ensure that the dims and coordinate vars come out in
1926 * the correct order. */
1927 while (dim || var)
1928 {
1929 nc_bool_t found_coord, wrote_coord;
1930
1931 /* Write non-coord dims in order, stopping at the first one that
1932 * has an associated coord var. */
1933 for (found_coord = NC_FALSE; dim && !found_coord; )
1934 {
1935 if (!dim->coord_var)
1936 {
1937 if ((retval = write_dim(dim, grp, bad_coord_order)))
1938 return retval;
1939 }
1940 else
1941 {
1942 coord_varid = dim->coord_var->hdr.id;
1943 found_coord = NC_TRUE;
1944 }
1945 dim = (NC_DIM_INFO_T *)ncindexith(grp->dim, ++dim_index);
1946 }
1947
1948 /* Write each var. When we get to the coord var we are waiting
1949 * for (if any), then we break after writing it. */
1950 for (wrote_coord = NC_FALSE; var && !wrote_coord; )
1951 {
1952 if ((retval = write_var(var, grp, bad_coord_order)))
1953 return retval;
1954 if (found_coord && var->hdr.id == coord_varid)
1955 wrote_coord = NC_TRUE;
1956 var = (NC_VAR_INFO_T *)ncindexith(grp->vars, ++var_index);
1957 }
1958 } /* end while */
1959
1960 /* Attach dimscales to vars in this group. Unless directed not to. */
1961 if (!grp->nc4_info->no_dimscale_attach) {
1962 if ((retval = attach_dimscales(grp)))
1963 return retval;
1964 }
1965
1966 /* If there are any child groups, write their metadata. */
1967 for (i = 0; i < ncindexsize(grp->children); i++)
1968 {
1969 child_grp = (NC_GRP_INFO_T *)ncindexith(grp->children, i);
1970 assert(child_grp);
1971 if ((retval = nc4_rec_write_metadata(child_grp, bad_coord_order)))
1972 return retval;
1973 }
1974 return NC_NOERR;
1975}
1976
1986int
1987nc4_rec_write_groups_types(NC_GRP_INFO_T *grp)
1988{
1989 NC_GRP_INFO_T *child_grp;
1990 NC_HDF5_GRP_INFO_T *hdf5_grp;
1991 NC_TYPE_INFO_T *type;
1992 int retval;
1993 int i;
1994
1995 assert(grp && grp->hdr.name && grp->format_grp_info);
1996 LOG((3, "%s: grp->hdr.name %s", __func__, grp->hdr.name));
1997
1998 /* Get HDF5-specific group info. */
1999 hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
2000
2001 /* Create the group in the HDF5 file if it doesn't exist. */
2002 if (!hdf5_grp->hdf_grpid)
2003 if ((retval = create_group(grp)))
2004 return retval;
2005
2006 /* If this is the root group of a file with strict NC3 rules, write
2007 * an attribute. But don't leave the attribute open. */
2008 if (!grp->parent && (grp->nc4_info->cmode & NC_CLASSIC_MODEL))
2009 if ((retval = write_nc3_strict_att(hdf5_grp->hdf_grpid)))
2010 return retval;
2011
2012 /* If there are any user-defined types, write them now. */
2013 for(i=0;i<ncindexsize(grp->type);i++) {
2014 type = (NC_TYPE_INFO_T *)ncindexith(grp->type, i);
2015 assert(type);
2016 if ((retval = commit_type(grp, type)))
2017 return retval;
2018 }
2019
2020 /* If there are any child groups, write their groups and types. */
2021 for(i=0;i<ncindexsize(grp->children);i++) {
2022 if((child_grp = (NC_GRP_INFO_T*)ncindexith(grp->children,i)) == NULL) continue;
2023 if ((retval = nc4_rec_write_groups_types(child_grp)))
2024 return retval;
2025 }
2026 return NC_NOERR;
2027}
2028
2042int
2043nc4_rec_match_dimscales(NC_GRP_INFO_T *grp)
2044{
2045 NC_GRP_INFO_T *g;
2046 NC_VAR_INFO_T *var;
2047 NC_DIM_INFO_T *dim;
2048 int retval = NC_NOERR;
2049 int i;
2050
2051 assert(grp && grp->hdr.name);
2052 LOG((4, "%s: grp->hdr.name %s", __func__, grp->hdr.name));
2053
2054 /* Perform var dimscale match for child groups. */
2055 for (i = 0; i < ncindexsize(grp->children); i++)
2056 {
2057 g = (NC_GRP_INFO_T*)ncindexith(grp->children, i);
2058 assert(g);
2059 if ((retval = nc4_rec_match_dimscales(g)))
2060 return retval;
2061 }
2062
2063 /* Check all the vars in this group. If they have dimscale info,
2064 * try and find a dimension for them. */
2065 for (i = 0; i < ncindexsize(grp->vars); i++)
2066 {
2067 NC_HDF5_VAR_INFO_T *hdf5_var;
2068 int ndims;
2069 int d;
2070
2071 /* Get pointer to var and to the HDF5-specific var info. */
2072 var = (NC_VAR_INFO_T*)ncindexith(grp->vars, i);
2073 assert(var && var->format_var_info);
2074 hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
2075
2076 /* Check all vars and see if dim[i] != NULL if dimids[i] valid. */
2077 /* This loop is very odd. Under normal circumstances, var->dimid[d] is zero
2078 (from the initial calloc) which is a legitimate dimid. The code does not
2079 distinquish this case from the dimscale case where the id might actually
2080 be defined.
2081 The original nc4_find_dim searched up the group tree looking for the given
2082 dimid in one of the dim lists associated with each ancestor group.
2083 I changed nc4_fnd_dim to use the dimid directly using h5->alldims.
2084 However, here that is incorrect because it will find the dimid 0 always
2085 (if any dimensions were defined). Except that when dimscale dimids have
2086 been defined, one or more of the values in var->dimids will have a
2087 legitimate value.
2088 The solution I choose is to modify nc4_var_list_add to initialize dimids to
2089 illegal values (-1). This is another example of the problems with dimscales.
2090 */
2091 ndims = var->ndims;
2092 for (d = 0; d < ndims; d++)
2093 {
2094 if (var->dim[d] == NULL) {
2095 nc4_find_dim(grp, var->dimids[d], &var->dim[d], NULL);
2096 }
2097 /* assert(var->dim[d] && var->dim[d]->hdr.id == var->dimids[d]); */
2098 }
2099
2100 /* Skip dimension scale variables */
2101 if (!hdf5_var->dimscale)
2102 {
2103 int d;
2104 int j;
2105
2106 /* Are there dimscales for this variable? */
2107 if (hdf5_var->dimscale_hdf5_objids)
2108 {
2109 for (d = 0; d < var->ndims; d++)
2110 {
2111 nc_bool_t finished = NC_FALSE;
2112 LOG((5, "%s: var %s has dimscale info...", __func__, var->hdr.name));
2113
2114 /* Check this and parent groups. */
2115 for (g = grp; g && !finished; g = g->parent)
2116 {
2117 /* Check all dims in this group. */
2118 for (j = 0; j < ncindexsize(g->dim); j++)
2119 {
2120 /* Get the HDF5 specific dim info. */
2121 NC_HDF5_DIM_INFO_T *hdf5_dim;
2122 dim = (NC_DIM_INFO_T *)ncindexith(g->dim, j);
2123 assert(dim && dim->format_dim_info);
2124 hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info;
2125
2126 /* Check for exact match of fileno/objid arrays
2127 * to find identical objects in HDF5 file. */
2128#if H5_VERSION_GE(1,12,0)
2129 int token_cmp;
2130 if (H5Otoken_cmp(hdf5_var->hdf_datasetid, &hdf5_var->dimscale_hdf5_objids[d].token, &hdf5_dim->hdf5_objid.token, &token_cmp) < 0)
2131 return NC_EHDFERR;
2132
2133 if (hdf5_var->dimscale_hdf5_objids[d].fileno == hdf5_dim->hdf5_objid.fileno &&
2134 token_cmp == 0)
2135#else
2136 if (hdf5_var->dimscale_hdf5_objids[d].fileno[0] == hdf5_dim->hdf5_objid.fileno[0] &&
2137 hdf5_var->dimscale_hdf5_objids[d].objno[0] == hdf5_dim->hdf5_objid.objno[0] &&
2138 hdf5_var->dimscale_hdf5_objids[d].fileno[1] == hdf5_dim->hdf5_objid.fileno[1] &&
2139 hdf5_var->dimscale_hdf5_objids[d].objno[1] == hdf5_dim->hdf5_objid.objno[1])
2140#endif
2141 {
2142 LOG((4, "%s: for dimension %d, found dim %s", __func__,
2143 d, dim->hdr.name));
2144 var->dimids[d] = dim->hdr.id;
2145 var->dim[d] = dim;
2146 finished = NC_TRUE;
2147 break;
2148 }
2149 } /* next dim */
2150 } /* next grp */
2151 LOG((5, "%s: dimid for this dimscale is %d", __func__,
2152 var->type_info->hdr.id));
2153 } /* next var->dim */
2154 }
2155 /* No dimscales for this var! Invent phony dimensions. */
2156 else
2157 {
2158 hid_t spaceid = 0;
2159 hsize_t *h5dimlen = NULL, *h5dimlenmax = NULL;
2160 int dataset_ndims;
2161
2162 /* Find the space information for this dimension. */
2163 if ((spaceid = H5Dget_space(hdf5_var->hdf_datasetid)) < 0)
2164 return NC_EHDFERR;
2165
2166 /* Get the len of each dim in the space. */
2167 if (var->ndims)
2168 {
2169 if (!(h5dimlen = malloc(var->ndims * sizeof(hsize_t))))
2170 return NC_ENOMEM;
2171 if (!(h5dimlenmax = malloc(var->ndims * sizeof(hsize_t))))
2172 {
2173 free(h5dimlen);
2174 return NC_ENOMEM;
2175 }
2176 if ((dataset_ndims = H5Sget_simple_extent_dims(spaceid, h5dimlen,
2177 h5dimlenmax)) < 0) {
2178 free(h5dimlenmax);
2179 free(h5dimlen);
2180 return NC_EHDFERR;
2181 }
2182 if (dataset_ndims != var->ndims) {
2183 free(h5dimlenmax);
2184 free(h5dimlen);
2185 return NC_EHDFERR;
2186 }
2187 }
2188 else
2189 {
2190 /* Make sure it's scalar. */
2191 if (H5Sget_simple_extent_type(spaceid) != H5S_SCALAR)
2192 return NC_EHDFERR;
2193 }
2194
2195 /* Release the space object. */
2196 if (H5Sclose(spaceid) < 0) {
2197 free(h5dimlen);
2198 free(h5dimlenmax);
2199 return NC_EHDFERR;
2200 }
2201
2202 /* Create a phony dimension for each dimension in the
2203 * dataset, unless there already is one the correct
2204 * size. */
2205 for (d = 0; d < var->ndims; d++)
2206 {
2207 int k;
2208 int match;
2209 /* Is there already a phony dimension of the correct size? */
2210 for(match=-1,k=0;k<ncindexsize(grp->dim);k++) {
2211 if((dim = (NC_DIM_INFO_T*)ncindexith(grp->dim,k)) == NULL) continue;
2212 if ((dim->len == h5dimlen[d]) &&
2213 ((h5dimlenmax[d] == H5S_UNLIMITED && dim->unlimited) ||
2214 (h5dimlenmax[d] != H5S_UNLIMITED && !dim->unlimited)))
2215 {match = k; break;}
2216 }
2217
2218 /* Didn't find a phony dim? Then create one. */
2219 if (match < 0)
2220 {
2221 char phony_dim_name[NC_MAX_NAME + 1];
2222 sprintf(phony_dim_name, "phony_dim_%d", grp->nc4_info->next_dimid);
2223 LOG((3, "%s: creating phony dim for var %s", __func__, var->hdr.name));
2224 if ((retval = nc4_dim_list_add(grp, phony_dim_name, h5dimlen[d], -1, &dim)))
2225 {
2226 free(h5dimlenmax);
2227 free(h5dimlen);
2228 return retval;
2229 }
2230 /* Create struct for HDF5-specific dim info. */
2231 if (!(dim->format_dim_info = calloc(1, sizeof(NC_HDF5_DIM_INFO_T))))
2232 return NC_ENOMEM;
2233 if (h5dimlenmax[d] == H5S_UNLIMITED)
2234 dim->unlimited = NC_TRUE;
2235 }
2236
2237 /* The variable must remember the dimid. */
2238 var->dimids[d] = dim->hdr.id;
2239 var->dim[d] = dim;
2240 } /* next dim */
2241
2242 /* Free the memory we malloced. */
2243 free(h5dimlen);
2244 free(h5dimlenmax);
2245 }
2246 }
2247 }
2248
2249 return retval;
2250}
2251
2263void
2264reportobject(int uselog, hid_t id, unsigned int type)
2265{
2266 char name[NC_HDF5_MAX_NAME];
2267 ssize_t len;
2268 const char* typename = NULL;
2269 long long printid = (long long)id;
2270
2271 len = H5Iget_name(id, name, NC_HDF5_MAX_NAME);
2272 if(len < 0) return;
2273 name[len] = '\0';
2274
2275 switch (type) {
2276 case H5F_OBJ_FILE: typename = "File"; break;
2277 case H5F_OBJ_DATASET: typename = "Dataset"; break;
2278 case H5F_OBJ_GROUP: typename = "Group"; break;
2279 case H5F_OBJ_DATATYPE: typename = "Datatype"; break;
2280 case H5F_OBJ_ATTR:
2281 typename = "Attribute";
2282 len = H5Aget_name(id, NC_HDF5_MAX_NAME, name);
2283 if(len < 0) len = 0;
2284 name[len] = '\0';
2285 break;
2286 default: typename = "<unknown>"; break;
2287 }
2288#ifdef LOGGING
2289 if(uselog) {
2290 LOG((0,"Type = %s(%lld) name='%s'",typename,printid,name));
2291 } else
2292#endif
2293 {
2294 fprintf(stderr,"Type = %s(%lld) name='%s'",typename,printid,name);
2295 }
2296
2297}
2298
2309static void
2310reportopenobjectsT(int uselog, hid_t fid, int ntypes, unsigned int* otypes)
2311{
2312 int t,i;
2313 ssize_t ocount;
2314 size_t maxobjs = -1;
2315 hid_t* idlist = NULL;
2316
2317 /* Always report somehow */
2318#ifdef LOGGING
2319 if(uselog)
2320 LOG((0,"\nReport: open objects on %lld",(long long)fid));
2321 else
2322#endif
2323 fprintf(stdout,"\nReport: open objects on %lld\n",(long long)fid);
2324 maxobjs = H5Fget_obj_count(fid,H5F_OBJ_ALL);
2325 if(idlist != NULL) free(idlist);
2326 idlist = (hid_t*)malloc(sizeof(hid_t)*maxobjs);
2327 for(t=0;t<ntypes;t++) {
2328 unsigned int ot = otypes[t];
2329 ocount = H5Fget_obj_ids(fid,ot,maxobjs,idlist);
2330 for(i=0;i<ocount;i++) {
2331 hid_t o = idlist[i];
2332 reportobject(uselog,o,ot);
2333 }
2334 }
2335 if(idlist != NULL) free(idlist);
2336}
2337
2346void
2347reportopenobjects(int uselog, hid_t fid)
2348{
2349 unsigned int OTYPES[5] = {H5F_OBJ_FILE, H5F_OBJ_DATASET, H5F_OBJ_GROUP,
2350 H5F_OBJ_DATATYPE, H5F_OBJ_ATTR};
2351
2352 reportopenobjectsT(uselog, fid ,5, OTYPES);
2353}
2354
2362void
2363showopenobjects5(NC_FILE_INFO_T* h5)
2364{
2365 NC_HDF5_FILE_INFO_T *hdf5_info;
2366
2367 assert(h5 && h5->format_file_info);
2368 hdf5_info = (NC_HDF5_FILE_INFO_T *)h5->format_file_info;
2369
2370 fprintf(stderr,"===== begin showopenobjects =====\n");
2371 reportopenobjects(0,hdf5_info->hdfid);
2372 fprintf(stderr,"===== end showopenobjects =====\n");
2373 fflush(stderr);
2374}
2375
2384void
2385showopenobjects(int ncid)
2386{
2387 NC_FILE_INFO_T* h5 = NULL;
2388
2389 /* Find our metadata for this file. */
2390 if (nc4_find_nc_grp_h5(ncid, NULL, NULL, &h5) != NC_NOERR)
2391 fprintf(stderr,"failed\n");
2392 else
2393 showopenobjects5(h5);
2394 fflush(stderr);
2395}
2396
2408int
2409NC4_hdf5get_libversion(unsigned* major,unsigned* minor,unsigned* release)
2410{
2411 if(H5get_libversion(major,minor,release) < 0)
2412 return NC_EHDFERR;
2413 return NC_NOERR;
2414}
2415
2426int
2427NC4_hdf5get_superblock(struct NC_FILE_INFO* h5, int* idp)
2428{
2429 NC_HDF5_FILE_INFO_T *hdf5_info;
2430 int stat = NC_NOERR;
2431 unsigned super;
2432 hid_t plist = -1;
2433
2434 assert(h5 && h5->format_file_info);
2435 hdf5_info = (NC_HDF5_FILE_INFO_T *)h5->format_file_info;
2436
2437 if((plist = H5Fget_create_plist(hdf5_info->hdfid)) < 0)
2438 {stat = NC_EHDFERR; goto done;}
2439 if(H5Pget_version(plist, &super, NULL, NULL, NULL) < 0)
2440 {stat = NC_EHDFERR; goto done;}
2441 if(idp) *idp = (int)super;
2442done:
2443 if(plist >= 0) H5Pclose(plist);
2444 return stat;
2445}
2446
2447static int NC4_strict_att_exists(NC_FILE_INFO_T*);
2448static int NC4_walk(hid_t, int*);
2449
2475int
2476NC4_isnetcdf4(struct NC_FILE_INFO* h5)
2477{
2478 int stat;
2479 int isnc4 = 0;
2480 int exists;
2481 int count;
2482
2483 /* Look for NC3_STRICT_ATT_NAME */
2484 exists = NC4_strict_att_exists(h5);
2485 if(exists)
2486 goto done;
2487 /* attribute did not exist */
2488 /* => last resort: walk the HDF5 file looking for markers */
2489 count = 0;
2490 stat = NC4_walk(((NC_HDF5_GRP_INFO_T *)(h5->root_grp->format_grp_info))->hdf_grpid,
2491 &count);
2492 if(stat != NC_NOERR)
2493 isnc4 = 0;
2494 else /* Threshold is at least two matches */
2495 isnc4 = (count >= 2);
2496
2497done:
2498 return isnc4;
2499}
2500
2509static int
2510NC4_strict_att_exists(NC_FILE_INFO_T *h5)
2511{
2512 hid_t grpid = -1;
2513 htri_t attr_exists;
2514
2515 /* Get root group ID. */
2516 grpid = ((NC_HDF5_GRP_INFO_T *)(h5->root_grp->format_grp_info))->hdf_grpid;
2517
2518 /* See if the NC3_STRICT_ATT_NAME attribute exists */
2519 if ((attr_exists = H5Aexists(grpid, NC3_STRICT_ATT_NAME)) < 0)
2520 return 1;
2521 return (attr_exists?1:0);
2522}
2523
2533static int
2534NC4_walk(hid_t gid, int* countp)
2535{
2536 int ncstat = NC_NOERR;
2537 int i,j,na;
2538 ssize_t len;
2539 hsize_t nobj;
2540 herr_t err;
2541 int otype;
2542 hid_t grpid, dsid;
2543 char name[NC_HDF5_MAX_NAME];
2544
2545 /* walk group members of interest */
2546 err = H5Gget_num_objs(gid, &nobj);
2547 if(err < 0) return err;
2548
2549 for(i = 0; i < nobj; i++) {
2550 /* Get name & kind of object in the group */
2551 len = H5Gget_objname_by_idx(gid,(hsize_t)i,name,(size_t)NC_HDF5_MAX_NAME);
2552 if(len < 0) return len;
2553
2554 otype = H5Gget_objtype_by_idx(gid,(size_t)i);
2555 switch(otype) {
2556 case H5G_GROUP:
2557 grpid = H5Gopen1(gid,name);
2558 NC4_walk(grpid,countp);
2559 H5Gclose(grpid);
2560 break;
2561 case H5G_DATASET: /* variables */
2562 /* Check for phony_dim */
2563 if(strcmp(name,"phony_dim")==0)
2564 *countp = *countp + 1;
2565 dsid = H5Dopen1(gid,name);
2566 na = H5Aget_num_attrs(dsid);
2567 for(j = 0; j < na; j++) {
2568 hid_t aid = H5Aopen_idx(dsid,(unsigned int) j);
2569 if(aid >= 0) {
2570 const NC_reservedatt* ra;
2571 ssize_t len = H5Aget_name(aid, NC_HDF5_MAX_NAME, name);
2572 if(len < 0) return len;
2573 /* Is this a netcdf-4 marker attribute */
2574 /* Is this a netcdf-4 marker attribute */
2575 ra = NC_findreserved(name);
2576 if(ra != NULL)
2577 *countp = *countp + 1;
2578 }
2579 H5Aclose(aid);
2580 }
2581 H5Dclose(dsid);
2582 break;
2583 default:/* ignore */
2584 break;
2585 }
2586 }
2587 return ncstat;
2588}
2589
EXTERNL int nc_free_vlen(nc_vlen_t *vl)
Free memory in a VLEN object.
Definition: dvlen.c:43
Main header file for the C API.
#define NC_EBADTYPE
Not a netcdf data type.
Definition: netcdf.h:410
#define NC_UINT
unsigned 4-byte int
Definition: netcdf.h:44
#define NC_ENDIAN_NATIVE
In HDF5 files you can set the endianness of variables with nc_def_var_endian().
Definition: netcdf.h:294
#define NC_EFILTER
Filter operation failed.
Definition: netcdf.h:513
#define NC_QUANTIZE_GRANULARBR_ATT_NAME
When quantization is used for a variable, an attribute of the appropriate name is added.
Definition: netcdf.h:344
#define NC_INT
signed 4 byte integer
Definition: netcdf.h:38
#define NC_ENDIAN_BIG
In HDF5 files you can set the endianness of variables with nc_def_var_endian().
Definition: netcdf.h:296
#define NC_MAX_VAR_DIMS
max per variable dimensions
Definition: netcdf.h:282
#define NC_BYTE
signed 1 byte integer
Definition: netcdf.h:35
#define NC_EPERM
Write to read only.
Definition: netcdf.h:379
#define NC_QUANTIZE_BITROUND
Use BitRound quantization.
Definition: netcdf.h:338
#define NC_EDIMSCALE
Problem with HDF5 dimscales.
Definition: netcdf.h:504
#define NC_VLEN
vlen (variable-length) types
Definition: netcdf.h:53
#define NC_NAT
Not A Type.
Definition: netcdf.h:34
#define NC_DOUBLE
double precision floating point number
Definition: netcdf.h:41
#define NC_UBYTE
unsigned 1 byte int
Definition: netcdf.h:42
#define NC_FLOAT
single precision floating point number
Definition: netcdf.h:40
#define NC_ENOMEM
Memory allocation (malloc) failure.
Definition: netcdf.h:448
#define NC_COMPOUND
compound types
Definition: netcdf.h:56
#define NC_CHUNKED
In HDF5 files you can set storage for each variable to be either contiguous or chunked,...
Definition: netcdf.h:304
#define NC_SHORT
signed 2 byte integer
Definition: netcdf.h:37
#define NC_QUANTIZE_GRANULARBR
Use Granular BitRound quantization.
Definition: netcdf.h:337
#define NC_QUANTIZE_BITGROOM
Use BitGroom quantization.
Definition: netcdf.h:336
#define NC_ENUM
enum types
Definition: netcdf.h:55
#define NC_INT64
signed 8-byte int
Definition: netcdf.h:45
#define NC_EATTMETA
Problem with attribute metadata.
Definition: netcdf.h:487
#define NC_EHDFERR
Error at HDF5 layer.
Definition: netcdf.h:481
#define NC_CONTIGUOUS
In HDF5 files you can set storage for each variable to be either contiguous or chunked,...
Definition: netcdf.h:305
#define NC_GLOBAL
Attribute id to put/get a global attribute.
Definition: netcdf.h:254
#define NC_UINT64
unsigned 8-byte int
Definition: netcdf.h:46
#define NC_COMPACT
In HDF5 files you can set storage for each variable to be either contiguous or chunked,...
Definition: netcdf.h:306
#define NC_EFILEMETA
Problem with file metadata.
Definition: netcdf.h:485
#define NC_ENOTVAR
Variable not found.
Definition: netcdf.h:422
#define NC_EINVAL
Invalid Argument.
Definition: netcdf.h:378
#define NC_CLASSIC_MODEL
Enforce classic model on netCDF-4.
Definition: netcdf.h:140
#define NC_ENDIAN_LITTLE
In HDF5 files you can set the endianness of variables with nc_def_var_endian().
Definition: netcdf.h:295
#define NC_MAX_NAME
Maximum for classic library.
Definition: netcdf.h:281
#define NC_NOERR
No Error.
Definition: netcdf.h:368
#define NC_EDIMMETA
Problem with dimension metadata.
Definition: netcdf.h:486
#define NC_USHORT
unsigned 2-byte int
Definition: netcdf.h:43
#define NC_OPAQUE
opaque types
Definition: netcdf.h:54
#define NC_STRING
string
Definition: netcdf.h:47
#define NC_QUANTIZE_BITGROOM_ATT_NAME
When quantization is used for a variable, an attribute of the appropriate name is added.
Definition: netcdf.h:343
#define NC_CHAR
ISO/ASCII character.
Definition: netcdf.h:36
#define NC_EVARMETA
Problem with variable metadata.
Definition: netcdf.h:488
#define NC_QUANTIZE_BITROUND_ATT_NAME
When quantization is used for a variable, an attribute of the appropriate name is added.
Definition: netcdf.h:345
int nc_type
The nc_type type is just an int.
Definition: netcdf.h:25
This is the type of arrays of vlens.
Definition: netcdf.h:746