public void WriteAndReadGroupsWithDataset()
{
string filename = Path.Combine(folder, "testGroups.H5");
try
{
var fileId = Hdf5.CreateFile(filename);
Assert.IsTrue(fileId > 0);
var dset = dsets.First();
var groupId = H5G.create(fileId, Hdf5Utils.NormalizedName("/A")); ///B/C/D/E/F/G/H
Hdf5.WriteDataset(groupId, Hdf5Utils.NormalizedName("test"), dset);
var subGroupId = Hdf5.CreateOrOpenGroup(groupId, Hdf5Utils.NormalizedName("C"));
var subGroupId2 = Hdf5.CreateOrOpenGroup(groupId, Hdf5Utils.NormalizedName("/D")); // will be saved at the root location
dset = dsets.Skip(1).First();
Hdf5.WriteDataset(subGroupId, Hdf5Utils.NormalizedName("test2"), dset);
Hdf5.CloseGroup(subGroupId);
Hdf5.CloseGroup(subGroupId2);
Hdf5.CloseGroup(groupId);
groupId = H5G.create(fileId, Hdf5Utils.NormalizedName("/A/B")); ///B/C/D/E/F/G/H
dset = dsets.Skip(1).First();
Hdf5.WriteDataset(groupId, Hdf5Utils.NormalizedName("test"), dset);
Hdf5.CloseGroup(groupId);
groupId = Hdf5.CreateGroupRecursively(fileId, Hdf5Utils.NormalizedName("A/B/C/D/E/F/I"));
Hdf5.CloseGroup(groupId);
Hdf5.CloseFile(fileId);
fileId = Hdf5.OpenFile(filename);
Assert.IsTrue(fileId > 0);
fileId = Hdf5.OpenFile(filename);
groupId = H5G.open(fileId, Hdf5Utils.NormalizedName("/A/B"));
double[,] dset2 = (double[, ])Hdf5.ReadDataset <double>(groupId, Hdf5Utils.NormalizedName("test")).result;
CompareDatasets(dset, dset2);
Assert.IsTrue(Hdf5.CloseGroup(groupId) >= 0);
groupId = H5G.open(fileId, Hdf5Utils.NormalizedName("/A/C"));
dset2 = (double[, ])Hdf5.ReadDataset <double>(groupId, Hdf5Utils.NormalizedName("test2")).result;
CompareDatasets(dset, dset2);
Assert.IsTrue(Hdf5.CloseGroup(groupId) >= 0);
bool same = dset == dset2;
dset = dsets.First();
dset2 = (double[, ])Hdf5.ReadDataset <double>(fileId, Hdf5Utils.NormalizedName("/A/test")).result;
CompareDatasets(dset, dset2);
Assert.IsTrue(Hdf5Utils.ItemExists(fileId, Hdf5Utils.NormalizedName("A/B/C/D/E/F/I"), DataTypes.Hdf5ElementType.Dataset));
Assert.IsTrue(Hdf5.CloseFile(fileId) == 0);
}
catch (Exception ex)
{
CreateExceptionAssert(ex);
}
}
public static string[] load_attributes_from_hdf5_group(long group, string name)
{
if (Hdf5.AttributeExists(group, name))
{
var(success, attr) = Hdf5.ReadStringAttributes(group, name, "");
if (success)
{
return(attr.ToArray());
}
}
return(null);
}
public KamaAcquisitionReadOnlyFile(string filename)
{
FileName = filename;
ProcedureInformation = new ProcedureInformation();
SystemInformation = new SystemInformation();
PatientInformation = new Patient();
ECG = new ECGData();
EITs = new List <EITEntry>();
Events = new List <SystemEvent>();
Hdf5.Settings.LowerCaseNaming = true;
Hdf5.Settings.DateTimeType = DateTimeType.UnixTimeMilliseconds;
fileId = Hdf5.OpenFile(filename);
}
public static void save_weights_to_hdf5_group(long f, List <ILayer> layers)
{
List <string> layerName = new List <string>();
foreach (var layer in layers)
{
layerName.Add(layer.Name);
}
save_attributes_to_hdf5_group(f, "layer_names", layerName.ToArray());
Hdf5.WriteAttribute(f, "backend", "tensorflow");
Hdf5.WriteAttribute(f, "keras_version", "2.5.0");
long g = 0, crDataGroup = 0;
List <IVariableV1> weights = new List <IVariableV1>();
//List<IVariableV1> weight_values = new List<IVariableV1>();
List <string> weight_names = new List <string>();
foreach (var layer in layers)
{
weight_names = new List <string>();
g = Hdf5.CreateOrOpenGroup(f, Hdf5Utils.NormalizedName(layer.Name));
weights = _legacy_weights(layer);
//weight_values= keras.backend.batch_get_value(weights);
foreach (var weight in weights)
{
weight_names.Add(weight.Name);
}
save_attributes_to_hdf5_group(g, "weight_names", weight_names.ToArray());
Tensor tensor = null;
foreach (var(name, val) in zip(weight_names, weights))
{
tensor = val.AsTensor();
if (name.IndexOf("/") > 1)
{
crDataGroup = Hdf5.CreateOrOpenGroup(g, Hdf5Utils.NormalizedName(name.Split('/')[0]));
WriteDataset(crDataGroup, name.Split('/')[1], tensor);
Hdf5.CloseGroup(crDataGroup);
}
else
{
WriteDataset(crDataGroup, name, tensor);
}
tensor = null;
}
Hdf5.CloseGroup(g);
weight_names = null;
}
weights = null;
// weight_values = null;
}
public void WriteAndReadChunckedDataset2()
{
string filename = Path.Combine(folder, "testChunks2.H5");
string groupName = "/test";
string datasetName = "Data";
try
{
var fileId = Hdf5.CreateFile(filename);
Assert.IsTrue(fileId > 0);
var groupId = Hdf5.CreateGroup(fileId, groupName);
Assert.IsTrue(groupId >= 0);
//var chunkSize = new ulong[] { 5, 5 };
using (var chunkedDset = new ChunkedDataset <double>(datasetName, groupId))
{
foreach (var ds in dsets)
{
chunkedDset.AppendOrCreateDataset(ds);
}
;
}
Hdf5.CloseFile(fileId);
}
catch (Exception ex)
{
CreateExceptionAssert(ex);
}
try
{
var fileId = Hdf5.OpenFile(filename);
//var groupId = H5G.open(fileId, groupName);
//var dset = Hdf5.ReadDatasetToArray<double>(groupId, datasetName);
var dset = Hdf5.ReadDatasetToArray <double>(fileId, string.Concat(groupName, "/", datasetName));
Assert.IsTrue(dset.result.Rank == dsets.First().Rank);
var xSum = dsets.Select(d => d.GetLength(0)).Sum();
Assert.IsTrue(xSum == dset.result.GetLength(0));
var testRange = Enumerable.Range(0, 30).Select(t => (double)t);
// get every 5th element in the matrix
var x0Range = dset.result.Cast <double>().Where((d, i) => i % 5 == 0);
Assert.IsTrue(testRange.SequenceEqual(x0Range));
Hdf5.CloseFile(fileId);
}
catch (Exception ex)
{
CreateExceptionAssert(ex);
}
}
public void WriteAndReadPrimitives()
{
string filename = Path.Combine(folder, "testPrimitives.H5");
int intValue = 2;
double dblValue = 1.1;
string strValue = "test";
bool boolValue = true;
var groupStr = "/test";
string concatFunc(string x) => string.Concat(groupStr, "/", x);
Dictionary <string, List <string> > attributes = new Dictionary <string, List <string> >();
try
{
var fileId = Hdf5.CreateFile(filename);
Assert.IsTrue(fileId > 0);
var groupId = Hdf5.CreateGroup(fileId, groupStr);
Hdf5.WriteOneValue(groupId, concatFunc(nameof(intValue)), intValue, attributes);
Hdf5.WriteOneValue(groupId, concatFunc(nameof(dblValue)), dblValue, attributes);
Hdf5.WriteOneValue(groupId, concatFunc(nameof(strValue)), strValue, attributes);
Hdf5.WriteOneValue(groupId, concatFunc(nameof(boolValue)), boolValue, attributes);
Hdf5.CloseGroup(groupId);
Hdf5.CloseFile(fileId);
}
catch (Exception ex)
{
CreateExceptionAssert(ex);
}
try
{
var fileId = Hdf5.OpenFile(filename);
Assert.IsTrue(fileId > 0);
var groupId = H5G.open(fileId, groupStr);
int readInt = Hdf5.ReadOneValue <int>(groupId, concatFunc(nameof(intValue)));
Assert.IsTrue(intValue == readInt);
double readDbl = Hdf5.ReadOneValue <double>(groupId, concatFunc(nameof(dblValue)));
Assert.IsTrue(dblValue == readDbl);
string readStr = Hdf5.ReadOneValue <string>(groupId, concatFunc(nameof(strValue)));
Assert.IsTrue(strValue == readStr);
bool readBool = Hdf5.ReadOneValue <bool>(groupId, concatFunc(nameof(boolValue)));
Assert.IsTrue(boolValue == readBool);
H5G.close(groupId);
Hdf5.CloseFile(fileId);
}
catch (Exception ex)
{
CreateExceptionAssert(ex);
}
}
public void Dispose()
{
try
{
if (!Disposed)
{
Hdf5.CloseGroup(GroupId);
}
}
catch (Exception)
{
//nothing
}
}
public void Dispose()
{
try
{
if (!Disposed)
{
Hdf5.CloseGroup(GroupId);
}
}
catch (Exception e)
{
Logger.LogError(e, $"Error closing RPosition group: {e.Message}");
}
}
public void WriteAndReadSubsetOfDataset()
{
string filename = Path.Combine(folder, "testSubset.H5");
try
{
var fileId = Hdf5.CreateFile(filename);
Assert.IsTrue(fileId > 0);
var chunkSize = new ulong[] { 5, 5 };
using (var chunkedDset = new ChunkedDataset <double>("/test", fileId, dsets.First()))
{
foreach (var ds in dsets.Skip(1))
{
chunkedDset.AppendDataset(ds);
}
;
}
Hdf5.CloseFile(fileId);
}
catch (Exception ex)
{
CreateExceptionAssert(ex);
}
try
{
var fileId = Hdf5.OpenFile(filename);
ulong begIndex = 8;
ulong endIndex = 21;
var dset = Hdf5.ReadDataset <double>(fileId, "/test", begIndex, endIndex);
Hdf5.CloseFile(fileId);
Assert.IsTrue(dset.Rank == dsets.First().Rank);
int count = Convert.ToInt32(endIndex - begIndex + 1);
Assert.IsTrue(count == dset.GetLength(0));
// Creat a range from number 8 to 21
var testRange = Enumerable.Range((int)begIndex, count).Select(t => (double)t);
// Get the first column from row index number 8 (the 9th row) to row index number 21 (22th row)
var x0Range = dset.Cast <double>().Where((d, i) => i % 5 == 0);
Assert.IsTrue(testRange.SequenceEqual(x0Range));
}
catch (Exception ex)
{
CreateExceptionAssert(ex);
}
}
public void ReadSystemInformation()
{
string groupName = rootName + system_informationName;
if (Hdf5.GroupExists(fileId, groupName))
{
SystemInformation = Hdf5.ReadObject <SystemInformation>(fileId, groupName);
return;
}
groupName = rootNameOld + system_informationName;
if (Hdf5.GroupExists(fileId, groupName))
{
SystemInformation = Hdf5.ReadObject <SystemInformation>(fileId, groupName);
}
}
public void ReadProcedureInformation()
{
string groupName = rootName + procedure_informationName;
if (Hdf5.GroupExists(fileId, groupName))
{
ProcedureInformation = Hdf5.ReadObject <ProcedureInformation>(fileId, groupName);
return;
}
groupName = rootNameOld + procedure_informationName;
if (Hdf5.GroupExists(fileId, groupName))
{
ProcedureInformation = Hdf5.ReadObject <ProcedureInformation>(fileId, groupName);
}
}
public void ReadPatientInformation()
{
string groupName = rootName + patient_informationName;
if (Hdf5.GroupExists(fileId, groupName))
{
PatientInformation = Hdf5.ReadObject <Patient>(fileId, groupName);
return;
}
groupName = rootNameOld + patient_informationName;
if (Hdf5.GroupExists(fileId, groupName))
{
PatientInformation = Hdf5.ReadObject <Patient>(fileId, groupName);
}
}
public void Dispose()
{
try
{
if (!Disposed)
{
Hdf5.CloseGroup(GroupId);
Disposed = true;
}
}
catch (Exception e)
{
Logger?.LogError($"Error closing file: {e}");
}
}
public void ReadECGData()
{
string groupName = rootName + ecgName;
if (Hdf5.GroupExists(fileId, groupName))
{
ECG = Hdf5.ReadObject <ECGData>(fileId, groupName);
return;
}
groupName = rootNameOld + ecgName;
if (Hdf5.GroupExists(fileId, groupName))
{
ECG = Hdf5.ReadObject <ECGData>(fileId, groupName);
}
}
public void WriteAndReadObjectWithPropertiesAndArrayPropertyTest()
{
try
{
var testClass = new TestClassWithArray()
{
TestInteger = 2,
TestDouble = 1.1,
TestBoolean = true,
TestString = "test string",
TestDoubles = new double[] { 1.1, 1.2, -1.1, -1.2 },
TestStrings = new string[] { "one", "two", "three", "four" }
};
testClassWithArrays.TestInteger = 2;
testClassWithArrays.TestDouble = 1.1;
testClassWithArrays.TestBoolean = true;
testClassWithArrays.TestString = "test string";
testClassWithArrays.TestDoubles = new double[] { 1.1, 1.2, -1.1, -1.2 };
testClassWithArrays.TestStrings = new string[] { "one", "two", "three", "four" };
string filename = Path.Combine(folder, "testArrayObjects.H5");
var fileId = Hdf5.CreateFile(filename);
Assert.IsTrue(fileId >= 0);
Hdf5.WriteObject(fileId, testClassWithArrays, "objectWithTwoArrays");
TestClassWithArray readObject = new TestClassWithArray
{
TestStrings = new string[0],
TestDoubles = null,
TestDouble = double.NaN
};
readObject = Hdf5.ReadObject(fileId, readObject, "objectWithTwoArrays");
Assert.IsTrue(testClassWithArrays.Equals(readObject));
readObject = Hdf5.ReadObject <TestClassWithArray>(fileId, "objectWithTwoArrays");
Assert.IsTrue(testClassWithArrays.Equals(readObject));
Assert.IsTrue(Hdf5.CloseFile(fileId) >= 0);
}
catch (Exception ex)
{
CreateExceptionAssert(ex);
}
}
public void ReadStructs()
{
string filename = Path.Combine(folder, "data", "testCompounds_WData2_WData3.H5");
try
{
var fileId = Hdf5.OpenFile(filename);
Assert.IsTrue(fileId > 0);
var cmpList = Hdf5.ReadCompounds <WData3>(fileId, "/test", "").ToArray();
Hdf5.CloseFile(fileId);
CollectionAssert.AreEqual(wData2List, cmpList);
}
catch (Exception ex)
{
CreateExceptionAssert(ex);
}
}
public void WriteAndReadObjectWithHdf5Attributes()
{
string filename = Path.Combine(folder, "testHdf5Attribute.H5");
var attObject = new AttributeClass();
try
{
var fileId = Hdf5.CreateFile(filename);
Assert.IsTrue(fileId > 0);
Hdf5.WriteObject(fileId, attObject, "anObject");
Assert.IsTrue(Hdf5.CloseFile(fileId) == 0);
}
catch (Exception ex)
{
CreateExceptionAssert(ex);
}
}
public void ReadEITData()
{
int index = 1;
string rootGroup = rootName + eitName;
if (!Hdf5.GroupExists(fileId, rootGroup))
{
rootGroup = rootNameOld + eitName;
}
while (Hdf5.GroupExists(fileId, rootGroup + "/d" + index))
{
var entry = Hdf5.ReadObject <EITEntry>(fileId, rootGroup + "/d" + index);
EITs.Add(entry);
index++;
}
}
private long WriteDataset(string filename)
{
long tef2 = Hdf5.CreateFile(filename);
int[] blah = { 1, 2, 4, 5, 0 };
Hdf5.WriteDatasetFromArray <int>(tef2, "blah", blah);
Hdf5.CloseFile(tef2);
var what = "???"; // breakpoint in VS to test h5 file contents independently before next write step
tef2 = Hdf5.OpenFile(filename);
blah[4] = 6;
Hdf5.WriteDatasetFromArray <int>(tef2, "blah", blah); // This command throws several debug errors from PInvoke
var(success, result) = Hdf5.ReadDataset <int>(tef2, "blah");
Assert.IsTrue(success);
Assert.IsTrue(result.Cast <int>().SequenceEqual(blah));
// loading the hdf5 file shows it only has {1, 2, 4, 5, 0} stored.
return(tef2);
}
public Task WaitForDataWritten()
{
if (!userEventsData.Any())
{
Logger.LogWarning("No user event to write to H5 file");
return(Task.CompletedTask);
}
var dataToWrite = userEventsData;
userEventsData = new List <UserEventRecord>();
Hdf5UserEvents eventsToH5 = new Hdf5UserEvents(dataToWrite);
Logger.LogInformation("Start write of User Events");
var status = Hdf5.WriteObject(GroupRoot, eventsToH5, Constants.EventGroupName);
Logger.LogInformation("End write of User Events with status " + status);
return(Task.CompletedTask);
}
public void WriteAndReadObjectWithPropertiesTest()
{
string filename = Path.Combine(folder, "testObjects.H5");
try
{
testClass.TestInteger = 2;
testClass.TestDouble = 1.1;
testClass.TestBoolean = true;
testClass.TestString = "test string";
// 31-Oct-2003, 18:00 is 731885.75 in matlab
testClass.TestTime = new DateTime(2003, 10, 31, 18, 0, 0);
var fileId = Hdf5.CreateFile(filename);
Assert.IsTrue(fileId > 0);
Hdf5.WriteObject(fileId, testClass, "objectWithProperties");
Hdf5.CloseFile(fileId);
}
catch (Exception ex)
{
CreateExceptionAssert(ex);
}
try
{
var fileId = Hdf5.OpenFile(filename);
Assert.IsTrue(fileId > 0);
TestClass readObject = new TestClass();
readObject = Hdf5.ReadObject(fileId, readObject, "objectWithProperties");
Assert.IsTrue(testClass.Equals(readObject));
readObject = Hdf5.ReadObject <TestClass>(fileId, "objectWithProperties");
Assert.IsTrue(testClass.Equals(readObject));
Assert.IsTrue(Hdf5.CloseFile(fileId) >= 0);
}
catch (Exception ex)
{
CreateExceptionAssert(ex);
}
}
//[TestMethod]
public void TestRead()
{
//string filename = @"D:\Data\9_pig.h5";
string filename = @"c:\kalpa\test.h5";
//string filename = @"d:\data\test2400.h5";
var fileId = Hdf5.OpenFile(filename);
Stopwatch st = Stopwatch.StartNew();
var ds = Hdf5.ReadDatasetToArray <float>(fileId, "/eit/d1/voltages.im");
st.Stop();
Console.WriteLine(ds.result.Length);
Console.WriteLine("read time im: " + st.ElapsedMilliseconds);
st.Restart();
ds = Hdf5.ReadDatasetToArray <float>(fileId, "/eit/d1/voltages.re");
st.Stop();
Console.WriteLine(ds.result.Length);
Console.WriteLine("read time re: " + st.ElapsedMilliseconds);
Hdf5.CloseFile(fileId);
}
public void WriteAndReadStringAttribute()
{
string filename = Path.Combine(folder, "testAttributeString.H5");
try
{
var fileId = Hdf5.CreateFile(filename);
Assert.IsTrue(fileId > 0);
var groupId = Hdf5.CreateGroup(fileId, "test");
string attrStr = "this is an attribute";
Hdf5.WriteAttribute(groupId, "time", attrStr);
string readStr = Hdf5.ReadAttribute <string>(groupId, "time");
Assert.IsTrue(readStr == attrStr);
Assert.IsTrue(Hdf5.CloseFile(fileId) == 0);
}
catch (Exception ex)
{
CreateExceptionAssert(ex);
}
}
public void WriteAndReadAttribute()
{
string filename = Path.Combine(folder, "testAttribute.H5");
try
{
var fileId = Hdf5.CreateFile(filename);
Assert.IsTrue(fileId > 0);
var groupId = Hdf5.CreateGroup(fileId, "test");
DateTime nowTime = DateTime.Now;
Hdf5.WriteAttribute(groupId, "time", nowTime);
DateTime readTime = Hdf5.ReadAttribute <DateTime>(groupId, "time");
Assert.IsTrue(readTime == nowTime);
Assert.IsTrue(Hdf5.CloseFile(fileId) == 0);
}
catch (Exception ex)
{
CreateExceptionAssert(ex);
}
}
static void WriteFile(string filePath)
{
var file = Hdf5.CreateFile(filePath);
var group = Hdf5.CreateOrOpenGroup(file, "group");
Hdf5.WriteDataset(group, "dataset",
new int[, ]
{
{ 1, 2, 3, 4 },
{ 5, 6, 7, 8 },
{ 9, 10, 11, 12 }
});
var hello = "早上好！";
Hdf5.WriteStringAttribute(group, "string", hello, "dataset");
Hdf5.CloseGroup(group);
Hdf5.CloseFile(file);
}
public void WriteAndReadAttributeByPath()
{
string filename = Path.Combine(folder, "testAttributeByPath.H5");
string path = "/A/B/C/D/E/F/I";
string attributeValue = "test";
Hdf5.Settings.LowerCaseNaming = false;
var fileId = Hdf5.CreateFile(filename);
Assert.IsTrue(fileId > 0);
var groupId = Hdf5.CreateGroupRecursively(fileId, Hdf5Utils.NormalizedName(path));
var result = Hdf5Utils.WriteAttributeByPath(filename, path, "VALID", attributeValue);
Assert.IsTrue(result);
var write = Hdf5Utils.ReadAttributeByPath(filename, path, "VALID");
Assert.IsTrue(write.success);
Assert.IsTrue(write.value == attributeValue);
Assert.IsTrue(H5G.close(groupId) == 0);
Assert.IsTrue(Hdf5.CloseFile(fileId) == 0);
}
public void WriteAndReadAllPrimitives()
{
string filename = Path.Combine(folder, "testAllPrimitives.H5");
//var groupStr = "/test";
//string concatFunc(string x) => string.Concat(groupStr, "/", x);
try
{
var fileId = Hdf5.CreateFile(filename);
Assert.IsTrue(fileId > 0);
Hdf5.WriteObject(fileId, allTypesObject, "/test");
var readObject = Hdf5.ReadObject <AllTypesClass>(fileId, "/test");
Assert.IsTrue(allTypesObject.PublicInstanceFieldsEqual(readObject));
Assert.IsTrue(Hdf5.CloseFile(fileId) == 0);
}
catch (Exception ex)
{
CreateExceptionAssert(ex);
}
}
public void Dispose()
{
try
{
if (!Disposed)
{
UnFiltered.Dispose();
Filtered.Dispose();
Timestamps.Dispose();
PacketIds?.Dispose();
EcgSamplesData.Dispose();
EcgTaskWriter.Dispose();
Hdf5.CloseGroup(GroupId);
Disposed = true;
}
}
catch (Exception e)
{
Logger.LogError($"Error during dispose of ECG: {e.Message}");
}
}
public void WriteAndReadOneAsciiString()
{
try
{
string test = "This is a test string";
string filename = Path.Combine(folder, "testOneString.H5");
var fileId = Hdf5.CreateFile(filename);
Hdf5.WriteAsciiString(fileId, "/test", test);
Assert.IsTrue(Hdf5.CloseFile(fileId) == 0);
fileId = Hdf5.OpenFile(filename);
string readStr = Hdf5.ReadAsciiString(fileId, "/test");
Assert.IsTrue(test == readStr);
Assert.IsTrue(Hdf5.CloseFile(fileId) == 0);
}
catch (Exception ex)
{
CreateExceptionAssert(ex);
}
}
public void load_weights(string filepath, bool by_name = false, bool skip_mismatch = false, object options = null)
{
long fileId = Hdf5.OpenFile(filepath, true);
bool msuccess = Hdf5.GroupExists(fileId, "model_weights");
bool lsuccess = Hdf5.GroupExists(fileId, "layer_names");
if (!lsuccess && msuccess)
{
fileId = H5G.open(fileId, "model_weights");
}
if (by_name)
{
//fdf5_format.load_weights_from_hdf5_group_by_name();
throw new NotImplementedException("");
}
else
{
hdf5_format.load_weights_from_hdf5_group(fileId, Layers);
}
Hdf5.CloseFile(fileId);
}
