public void AppendOrCreateDataset(Array dataset) { if (_chunkDims == null) { _chunkDims = new[] { Convert.ToUInt64(dataset.GetLongLength(0)), Convert.ToUInt64(dataset.GetLongLength(1)) }; Rank = dataset.Rank; _currentDims = GetDims(dataset); /* Create the data space with unlimited dimensions. */ _spaceId = H5S.create_simple(Rank, _currentDims, _maxDims); /* Modify dataset creation properties, i.e. enable chunking */ _propId = H5P.create(H5P.DATASET_CREATE); _status = H5P.set_chunk(_propId, Rank, _chunkDims); /* Create a new dataset within the file using chunk creation properties. */ _datasetId = H5D.create(GroupId, Hdf5Utils.NormalizedName(Datasetname), _datatype, _spaceId, H5P.DEFAULT, _propId); /* Write data to dataset */ GCHandle hnd = GCHandle.Alloc(dataset, GCHandleType.Pinned); _status = H5D.write(_datasetId, _datatype, H5S.ALL, H5S.ALL, H5P.DEFAULT, hnd.AddrOfPinnedObject()); hnd.Free(); H5S.close(_spaceId); _spaceId = -1; } else { AppendDataset(dataset); } }
/// <summary> /// Dispose function as suggested in the stackoverflow discussion below /// See: http://stackoverflow.com/questions/538060/proper-use-of-the-idisposable-interface/538238#538238 /// </summary> /// <param name="itIsSafeToAlsoFreeManagedObjects"></param> protected virtual void Dispose(bool itIsSafeToAlsoFreeManagedObjects) { if (!Hdf5Utils.GetRealName(GroupId, Datasetname, string.Empty).valid) { Hdf5Utils.LogInfo?.Invoke("Dataset does not exist."); return; } if (_datasetId >= 0) { H5D.close(_datasetId); } if (_propId >= 0) { H5P.close(_propId); } if (_spaceId >= 0) { H5S.close(_spaceId); } if (itIsSafeToAlsoFreeManagedObjects) { } }
public static (bool success, IEnumerable <string> result) ReadStrings(long groupId, string name, string alternativeName) { long datatype = H5T.create(H5T.class_t.STRING, H5T.VARIABLE); H5T.set_cset(datatype, H5T.cset_t.UTF8); H5T.set_strpad(datatype, H5T.str_t.NULLTERM); //name = ToHdf5Name(name); var datasetId = H5D.open(groupId, Hdf5Utils.NormalizedName(name)); if (datasetId < 0) //does not exist? { datasetId = H5D.open(groupId, Hdf5Utils.NormalizedName(alternativeName)); } if (datasetId <= 0) { Hdf5Utils.LogError?.Invoke($"Error reading {groupId}. Name:{name}. AlternativeName:{alternativeName}"); return(false, Array.Empty <string>()); } long spaceId = H5D.get_space(datasetId); long count = H5S.get_simple_extent_npoints(spaceId); H5S.close(spaceId); var strs = new List <string>(); if (count >= 0) { IntPtr[] rdata = new IntPtr[count]; GCHandle hnd = GCHandle.Alloc(rdata, GCHandleType.Pinned); H5D.read(datasetId, datatype, H5S.ALL, H5S.ALL, H5P.DEFAULT, hnd.AddrOfPinnedObject()); for (int i = 0; i < rdata.Length; ++i) { int len = 0; while (Marshal.ReadByte(rdata[i], len) != 0) { ++len; } byte[] buffer = new byte[len]; Marshal.Copy(rdata[i], buffer, 0, buffer.Length); string s = Encoding.UTF8.GetString(buffer); strs.Add(s); // H5.free_memory(rdata[i]); } hnd.Free(); } H5T.close(datatype); H5D.close(datasetId); return(true, strs); }
public static (int success, long CreatedgroupId) WriteStrings(long groupId, string name, IEnumerable <string> strs) { // create UTF-8 encoded test datasets long datatype = H5T.create(H5T.class_t.STRING, H5T.VARIABLE); H5T.set_cset(datatype, H5T.cset_t.ASCII); H5T.set_strpad(datatype, H5T.str_t.NULLTERM); int strSz = strs.Count(); long spaceId = H5S.create_simple(1, new[] { (ulong)strSz }, null); string normalizedName = Hdf5Utils.NormalizedName(name); var datasetId = Hdf5Utils.GetDatasetId(groupId, normalizedName, datatype, spaceId); if (datasetId == -1L) { return(-1, -1L); } GCHandle[] hnds = new GCHandle[strSz]; IntPtr[] wdata = new IntPtr[strSz]; int cntr = 0; foreach (string str in strs) { hnds[cntr] = GCHandle.Alloc( Encoding.UTF8.GetBytes(str), GCHandleType.Pinned); wdata[cntr] = hnds[cntr].AddrOfPinnedObject(); cntr++; } var hnd = GCHandle.Alloc(wdata, GCHandleType.Pinned); var result = H5D.write(datasetId, datatype, H5S.ALL, H5S.ALL, H5P.DEFAULT, hnd.AddrOfPinnedObject()); hnd.Free(); for (int i = 0; i < strSz; ++i) { hnds[i].Free(); } H5D.close(datasetId); H5S.close(spaceId); H5T.close(datatype); return(result, datasetId); }
public void AppendOrCreateDataset(Array dataset) { if (_chunkDims == null) { if (dataset.Rank < 1) { string msg = "Empty array was passed. Ignoring."; Hdf5Utils.LogError?.Invoke(msg); return; } for (int dimension = 1; dimension <= dataset.Rank; dimension++) { var size = dataset.GetUpperBound(dimension - 1) + 1; if (size == 0) { string msg = $"Empty array was passed for dimension {dimension}. Ignoring."; Hdf5Utils.LogError?.Invoke(msg); return; } } _chunkDims = new[] { Convert.ToUInt64(dataset.GetLongLength(0)), Convert.ToUInt64(dataset.GetLongLength(1)) }; Rank = dataset.Rank; _currentDims = GetDims(dataset); /* Create the data space with unlimited dimensions. */ _spaceId = H5S.create_simple(Rank, _currentDims, _maxDims); /* Modify dataset creation properties, i.e. enable chunking */ _propId = H5P.create(H5P.DATASET_CREATE); _status = H5P.set_chunk(_propId, Rank, _chunkDims); /* Create a new dataset within the file using chunk creation properties. */ _datasetId = H5D.create(GroupId, Hdf5Utils.NormalizedName(Datasetname), _datatype, _spaceId, H5P.DEFAULT, _propId); /* Write data to dataset */ GCHandle hnd = GCHandle.Alloc(dataset, GCHandleType.Pinned); _status = H5D.write(_datasetId, _datatype, H5S.ALL, H5S.ALL, H5P.DEFAULT, hnd.AddrOfPinnedObject()); hnd.Free(); H5S.close(_spaceId); _spaceId = -1; } else { AppendDataset(dataset); } }
// private readonly ReaderWriterLockSlim lock_ = new ReaderWriterLockSlim(); public Hdf5AcquisitionFileWriter(string filename, string groupName = "ROOT") { H5E.set_auto(H5E.DEFAULT, null, IntPtr.Zero); //lock_.EnterWriteLock(); _filename = filename; fileId = Hdf5.CreateFile(filename); _groupName = groupName; _groupId = Hdf5.CreateOrOpenGroup(fileId, Hdf5Utils.NormalizedName(_groupName)); Header = new Hdf5AcquisitionFile(); _nrOfRecords = 0; _sampleCount = 0; //lock_.ExitWriteLock(); }
public static int WriteAsciiString(long groupId, string name, string str) { var spaceNullId = H5S.create(H5S.class_t.NULL); var spaceScalarId = H5S.create(H5S.class_t.SCALAR); // create two datasets of the extended ASCII character set // store as H5T.FORTRAN_S1 -> space padding int strLength = str.Length; ulong[] dims = { (ulong)strLength, 1 }; /* Create the dataset. */ //name = ToHdf5Name(name); var spaceId = H5S.create_simple(1, dims, null); var datasetId = H5D.create(groupId, Hdf5Utils.NormalizedName(name), H5T.FORTRAN_S1, spaceId); H5S.close(spaceId); // we write from C and must provide null-terminated strings byte[] wdata = new byte[strLength * 2]; //for (int i = 0; i < strLength; ++i) //{ // wdata[2 * i] = (byte)i; //} for (int i = 0; i < strLength; ++i) { wdata[2 * i] = Convert.ToByte(str[i]); } var memId = H5T.copy(H5T.C_S1); H5T.set_size(memId, new IntPtr(2)); //H5T.set_strpad(memId, H5T.str_t.NULLTERM); GCHandle hnd = GCHandle.Alloc(wdata, GCHandleType.Pinned); int result = H5D.write(datasetId, memId, H5S.ALL, H5S.ALL, H5P.DEFAULT, hnd.AddrOfPinnedObject()); hnd.Free(); H5T.close(memId); H5D.close(datasetId); return(result); }
public void AppendDataset(Array dataset) { if (!Hdf5Utils.GetRealName(GroupId, Datasetname, string.Empty).valid) { string msg = "call constructor or FirstDataset first before appending."; Hdf5Utils.LogError?.Invoke(msg); throw new Hdf5Exception(msg); } _oldDims = _currentDims; _currentDims = GetDims(dataset); int rank = dataset.Rank; ulong[] zeros = Enumerable.Range(0, rank).Select(z => (ulong)0).ToArray(); /* Extend the dataset. Dataset becomes 10 x 3 */ var size = new[] { _oldDims[0] + _currentDims[0] }.Concat(_oldDims.Skip(1)).ToArray(); _status = H5D.set_extent(_datasetId, size); ulong[] offset = new[] { _oldDims[0] }.Concat(zeros.Skip(1)).ToArray(); /* Select a hyperslab in extended portion of dataset */ var filespaceId = H5D.get_space(_datasetId); _status = H5S.select_hyperslab(filespaceId, H5S.seloper_t.SET, offset, null, _currentDims, null); /* Define memory space */ var memId = H5S.create_simple(Rank, _currentDims, null); /* Write the data to the extended portion of dataset */ GCHandle hnd = GCHandle.Alloc(dataset, GCHandleType.Pinned); _status = H5D.write(_datasetId, _datatype, memId, filespaceId, H5P.DEFAULT, hnd.AddrOfPinnedObject()); hnd.Free(); _currentDims = size; H5S.close(memId); H5S.close(filespaceId); }
public void FirstDataset(Array dataset) { if (GroupId <= 0) { throw new Hdf5Exception("cannot call FirstDataset because group or file couldn't be created"); } if (Hdf5Utils.GetRealName(GroupId, Datasetname, string.Empty).valid) { throw new Hdf5Exception("cannot call FirstDataset because dataset already exists"); } Rank = dataset.Rank; _currentDims = GetDims(dataset); /* Create the data space with unlimited dimensions. */ _spaceId = H5S.create_simple(Rank, _currentDims, _maxDims); /* Modify dataset creation properties, i.e. enable chunking */ _propId = H5P.create(H5P.DATASET_CREATE); _status = H5P.set_chunk(_propId, Rank, _chunkDims); /* Create a new dataset within the file using chunk creation properties. */ _datasetId = H5D.create(GroupId, Hdf5Utils.NormalizedName(Datasetname), _datatype, _spaceId, H5P.DEFAULT, _propId); /* Write data to dataset */ GCHandle hnd = GCHandle.Alloc(dataset, GCHandleType.Pinned); _status = H5D.write(_datasetId, _datatype, H5S.ALL, H5S.ALL, H5P.DEFAULT, hnd.AddrOfPinnedObject()); if (_status < 0) { Hdf5Utils.LogError("Unable to write dataset"); } hnd.Free(); H5S.close(_spaceId); _spaceId = -1; }
public static string ReadAsciiString(long groupId, string name) { var datatype = H5T.FORTRAN_S1; //name = ToHdf5Name(name); var datasetId = H5D.open(groupId, Hdf5Utils.NormalizedName(name)); var spaceId = H5D.get_space(datasetId); int rank = H5S.get_simple_extent_ndims(spaceId); ulong[] maxDims = new ulong[rank]; ulong[] dims = new ulong[rank]; ulong[] chunkDims = new ulong[rank]; var memId_n = H5S.get_simple_extent_dims(spaceId, dims, null); // we write from C and must provide null-terminated strings byte[] wdata = new byte[dims[0] * 2]; var memId = H5T.copy(H5T.C_S1); H5T.set_size(memId, new IntPtr(2)); //H5T.set_strpad(memId, H5T.str_t.NULLTERM); GCHandle hnd = GCHandle.Alloc(wdata, GCHandleType.Pinned); int resultId = H5D.read(datasetId, memId, H5S.ALL, H5S.ALL, H5P.DEFAULT, hnd.AddrOfPinnedObject()); hnd.Free(); wdata = wdata.Where((b, i) => i % 2 == 0). Select(b => (b == 0) ? (byte)32 : b).ToArray(); string result = Encoding.ASCII.GetString(wdata); H5T.close(memId); H5D.close(datasetId); return(result); }
public static T ReadObject <T>(long groupId, T readValue, string groupName) { if (readValue == null) { throw new ArgumentNullException(nameof(readValue)); } Type tyObject = readValue.GetType(); //foreach (Attribute attr in Attribute.GetCustomAttributes(tyObject)) //{ // if (attr is Hdf5GroupName) // groupName = (attr as Hdf5GroupName).Name; // if (attr is Hdf5SaveAttribute) // { // Hdf5SaveAttribute atLeg = attr as Hdf5SaveAttribute; // if (atLeg.SaveKind == Hdf5Save.DoNotSave) // return readValue; // } //} bool isGroupName = !string.IsNullOrWhiteSpace(groupName); if (isGroupName) { groupId = H5G.open(groupId, Hdf5Utils.NormalizedName(groupName)); } ReadFields(tyObject, readValue, groupId); ReadProperties(tyObject, readValue, groupId); if (isGroupName) { CloseGroup(groupId); } return(readValue); }
public static int WriteUnicodeString(long groupId, string name, string str, H5T.str_t strPad = H5T.str_t.SPACEPAD) { byte[] wdata = Encoding.UTF8.GetBytes(str); long spaceId = H5S.create(H5S.class_t.SCALAR); long dtype = H5T.create(H5T.class_t.NO_CLASS, new IntPtr(wdata.Length)); H5T.set_cset(dtype, H5T.cset_t.UTF8); H5T.set_strpad(dtype, strPad); long datasetId = H5D.create(groupId, Hdf5Utils.NormalizedName(name), dtype, spaceId); GCHandle hnd = GCHandle.Alloc(wdata, GCHandleType.Pinned); int result = H5D.write(datasetId, dtype, H5S.ALL, H5S.ALL, H5P.DEFAULT, hnd.AddrOfPinnedObject()); hnd.Free(); H5T.close(dtype); H5D.close(datasetId); H5S.close(spaceId); return(result); }
private static void WriteField(object infoVal, Dictionary <string, List <string> > attributes, long groupId, string name) { Type ty = infoVal.GetType(); TypeCode code = Type.GetTypeCode(ty); foreach (var attribute in Attributes(ty)) { if (!attributes.ContainsKey(attribute.Key)) { attributes.Add(attribute.Key, attribute.Value); } } if (ty.IsArray) { var elType = ty.GetElementType(); TypeCode elCode = Type.GetTypeCode(elType); if (elCode != TypeCode.Object || ty == typeof(TimeSpan[])) { dsetRW.WriteArray(groupId, name, (Array)infoVal, attributes); } else { CallByReflection <(int, long)>(nameof(WriteCompounds), elType, new[] { groupId, name, infoVal, attributes }); } } else if (ty.IsGenericType && ty.GetGenericTypeDefinition() == typeof(List <>)) { var elType = Hdf5Utils.GetEnumerableType(ty); TypeCode elCode = Type.GetTypeCode(elType); if (elCode != TypeCode.Object || ty == typeof(List <TimeSpan>)) { IList infoAsList = (IList)infoVal; Array arr = Array.CreateInstance(elType, infoAsList.Count); for (var i = 0; i < infoAsList.Count; i++) { arr.SetValue(infoAsList[i], i); } dsetRW.WriteArray(groupId, name, arr, attributes); } else { CallByReflection <(int, long)>(nameof(WriteCompounds), elType, new[] { groupId, name, infoVal, attributes }); } } else if (primitiveTypes.Contains(code) || ty == typeof(TimeSpan)) { (int success, long CreatedgroupId) = //WriteOneValue(groupId, name, infoVal); CallByReflection <(int, long)>(nameof(WriteOneValue), ty, new[] { groupId, name, infoVal, attributes }); //todo: fix it //add its attributes if there are: //foreach (Attribute attr in Attribute.GetCustomAttributes(filedInfo)) //{ // if (attr is Hdf5Attribute) // { // var h5at = attr as Hdf5Attribute; // WriteStringAttribute(groupId, name, h5at.Name, name); // } // if (attr is Hdf5Attributes) // { // var h5ats = attr as Hdf5Attributes; // WriteAttributes<string>(groupId, name, h5ats.Names, attr.); // } //} } else { WriteObject(groupId, infoVal, name); } }
private static void ReadProperties(Type tyObject, object readValue, long groupId) { PropertyInfo[] miMembers = tyObject.GetProperties( /*BindingFlags.DeclaredOnly |*/ BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Instance); foreach (PropertyInfo info in miMembers) { bool nextInfo = false; string alternativeName = string.Empty; foreach (Attribute attr in Attribute.GetCustomAttributes(info)) { if (attr is Hdf5SaveAttribute hdf5SaveAttribute) { Hdf5Save kind = hdf5SaveAttribute.SaveKind; nextInfo = (kind == Hdf5Save.DoNotSave); } if (attr is Hdf5EntryNameAttribute hdf5EntryNameAttribute) { alternativeName = hdf5EntryNameAttribute.Name; } } if (nextInfo) { continue; } Type ty = info.PropertyType; TypeCode code = Type.GetTypeCode(ty); string name = info.Name; bool success; Array values; if (ty.IsArray) { var elType = ty.GetElementType(); TypeCode elCode = Type.GetTypeCode(elType); if (elCode != TypeCode.Object || ty == typeof(TimeSpan[])) { (success, values) = dsetRW.ReadArray(elType, groupId, name, alternativeName); if (success) { info.SetValue(readValue, values); } } else { var obj = CallByReflection <IEnumerable>(nameof(ReadCompounds), elType, new object[] { groupId, name, alternativeName }); var objArr = (obj).Cast <object>().ToArray(); values = Array.CreateInstance(elType, objArr.Length); Array.Copy(objArr, values, objArr.Length); info.SetValue(readValue, values); } } else if (ty.IsGenericType && ty.GetGenericTypeDefinition() == typeof(List <>)) { var elType = Hdf5Utils.GetEnumerableType(ty); TypeCode elCode = Type.GetTypeCode(elType); if (elCode != TypeCode.Object) { (success, values) = dsetRW.ReadArray(elType, groupId, name, alternativeName); if (success) { Type genericClass = typeof(List <>); // MakeGenericType is badly named Type constructedClass = genericClass.MakeGenericType(elType); IList created = (IList)Activator.CreateInstance(constructedClass); foreach (var o in values) { created.Add(o); } info.SetValue(readValue, created); } } else { var result = CallByReflection <object>(nameof(ReadCompounds), elType, new object[] { groupId, name, alternativeName }); info.SetValue(readValue, result); } } else if (primitiveTypes.Contains(code) || ty == typeof(TimeSpan)) { (success, values) = dsetRW.ReadArray(ty, groupId, name, alternativeName); if (success && values.Length > 0) { int[] first = new int[values.Rank].Select(f => 0).ToArray(); if (info.CanWrite) { info.SetValue(readValue, values.GetValue(first)); } else { Hdf5Utils.LogMessage($"property {info.Name} is read only. cannot set value", Hdf5LogLevel.Warning); } } } else { object value = info.GetValue(readValue, null); if (value != null) { value = ReadObject(groupId, value, name); info.SetValue(readValue, value); } } } }
public static string ReadUnicodeString(long groupId, string name) { var datasetId = H5D.open(groupId, Hdf5Utils.NormalizedName(name)); var typeId = H5D.get_type(datasetId); if (H5T.is_variable_str(typeId) > 0) { var spaceId = H5D.get_space(datasetId); long count = H5S.get_simple_extent_npoints(spaceId); IntPtr[] rdata = new IntPtr[count]; GCHandle hnd = GCHandle.Alloc(rdata, GCHandleType.Pinned); H5D.read(datasetId, typeId, H5S.ALL, H5S.ALL, H5P.DEFAULT, hnd.AddrOfPinnedObject()); var attrStrings = new List <string>(); for (int i = 0; i < rdata.Length; ++i) { int attrLength = 0; while (Marshal.ReadByte(rdata[i], attrLength) != 0) { ++attrLength; } byte[] buffer = new byte[attrLength]; Marshal.Copy(rdata[i], buffer, 0, buffer.Length); string stringPart = Encoding.UTF8.GetString(buffer); attrStrings.Add(stringPart); H5.free_memory(rdata[i]); } hnd.Free(); H5S.close(spaceId); H5D.close(datasetId); return(attrStrings[0]); } // Must be a non-variable length string. int size = H5T.get_size(typeId).ToInt32(); IntPtr iPtr = Marshal.AllocHGlobal(size); int result = H5D.read(datasetId, typeId, H5S.ALL, H5S.ALL, H5P.DEFAULT, iPtr); if (result < 0) { throw new IOException("Failed to read dataset"); } var strDest = new byte[size]; Marshal.Copy(iPtr, strDest, 0, size); Marshal.FreeHGlobal(iPtr); H5D.close(datasetId); return(Encoding.UTF8.GetString(strDest).TrimEnd((char)0)); }
private static void ReadFields(Type tyObject, object readValue, long groupId) { FieldInfo[] miMembers = tyObject.GetFields(BindingFlags.DeclaredOnly | /*BindingFlags.NonPublic |*/ BindingFlags.Public | BindingFlags.Instance); foreach (FieldInfo info in miMembers) { bool nextInfo = false; string alternativeName = string.Empty; foreach (Attribute attr in Attribute.GetCustomAttributes(info)) { if (attr is Hdf5EntryNameAttribute nameAttribute) { alternativeName = nameAttribute.Name; } if (attr is Hdf5SaveAttribute attribute) { Hdf5Save kind = attribute.SaveKind; nextInfo = (kind == Hdf5Save.DoNotSave); } else { nextInfo = false; } } if (nextInfo) { continue; } Type ty = info.FieldType; TypeCode code = Type.GetTypeCode(ty); string name = info.Name; Hdf5Utils.LogDebug?.Invoke($"groupname: {tyObject.Name}; field name: {name}"); bool success; Array values; if (ty.IsArray) { var elType = ty.GetElementType(); TypeCode elCode = Type.GetTypeCode(elType); if (elCode != TypeCode.Object) { (success, values) = dsetRW.ReadArray(elType, groupId, name, alternativeName); } else { values = CallByReflection <Array>(nameof(ReadCompounds), elType, new object[] { groupId, name, alternativeName }); success = true; } if (success) { info.SetValue(readValue, values); } } else if (ty.IsGenericType && ty.GetGenericTypeDefinition() == typeof(List <>)) { var elType = Hdf5Utils.GetEnumerableType(ty); TypeCode elCode = Type.GetTypeCode(elType); if (elCode != TypeCode.Object) { (success, values) = dsetRW.ReadArray(elType, groupId, name, alternativeName); if (success) { Type genericClass = typeof(List <>); // MakeGenericType is badly named Type constructedClass = genericClass.MakeGenericType(elType); IList created = (IList)Activator.CreateInstance(constructedClass); foreach (var o in values) { created.Add(o); } info.SetValue(readValue, created); } } else { var result = CallByReflection <object>(nameof(ReadCompounds), elType, new object[] { groupId, name, alternativeName }); info.SetValue(readValue, result); } } else if (primitiveTypes.Contains(code) || ty == typeof(TimeSpan)) { (success, values) = dsetRW.ReadArray(ty, groupId, name, alternativeName); // get first value depending on rank of the matrix int[] first = new int[values.Rank].Select(f => 0).ToArray(); if (success) { info.SetValue(readValue, values.GetValue(first)); } } else { object value = info.GetValue(readValue); if (value != null) { ReadObject(groupId, value, name); } } } }
public static bool GroupExists(long groupId, string groupName) => Hdf5Utils.ItemExists(groupId, groupName, Hdf5ElementType.Group);
public static long CreateOrOpenGroup(long groupId, string groupName) { return((Hdf5Utils.ItemExists(groupId, groupName, Hdf5ElementType.Group)) ? H5G.open(groupId, Hdf5Utils.NormalizedName(groupName)) : H5G.create(groupId, Hdf5Utils.NormalizedName(groupName))); }