public static unsafe void AddDataWithSharedDataType(long fileId, ContainerType container)
{
long typeId = H5T.copy(H5T.C_S1);
H5T.set_size(typeId, H5T.VARIABLE);
H5T.set_cset(typeId, H5T.cset_t.UTF8);
H5T.commit(fileId, "string_t", typeId);
var data = new string[] { "001", "11", "22", "33", "44", "55", "66", "77", " ", "AA", "ZZ", "!!" };
var dataChar = data
.SelectMany(value => Encoding.ASCII.GetBytes(value + '\0'))
.ToArray();
fixed(byte *dataVarPtr = dataChar)
{
var basePtr = new IntPtr(dataVarPtr);
var addresses = new IntPtr[]
{
IntPtr.Add(basePtr, 0), IntPtr.Add(basePtr, 4), IntPtr.Add(basePtr, 7), IntPtr.Add(basePtr, 10),
IntPtr.Add(basePtr, 13), IntPtr.Add(basePtr, 16), IntPtr.Add(basePtr, 19), IntPtr.Add(basePtr, 22),
IntPtr.Add(basePtr, 25), IntPtr.Add(basePtr, 28), IntPtr.Add(basePtr, 31), IntPtr.Add(basePtr, 34)
};
fixed(void *dataVarAddressesPtr = addresses)
{
TestUtils.Add(container, fileId, "shared_data_type", "shared_data_type", typeId, dataVarAddressesPtr, length: 12);
}
}
if (H5I.is_valid(typeId) > 0)
{
H5T.close(typeId);
}
}
public void H5DwriteTest1()
{
string utf8string
= "Γαζέες καὶ μυρτιὲς δὲν θὰ βρῶ πιὰ στὸ χρυσαφὶ ξέφωτο";
byte[] wdata = Encoding.UTF8.GetBytes(utf8string);
hid_t dtype = H5T.create(H5T.class_t.STRING,
new IntPtr(wdata.Length));
Assert.IsTrue(H5T.set_cset(dtype, H5T.cset_t.UTF8) >= 0);
Assert.IsTrue(H5T.set_strpad(dtype, H5T.str_t.SPACEPAD) >= 0);
hid_t dset_v0 = H5D.create(m_v0_test_file, "dset", dtype,
m_space_scalar);
Assert.IsTrue(dset_v0 >= 0);
hid_t dset_v2 = H5D.create(m_v2_test_file, "dset", dtype,
m_space_scalar);
Assert.IsTrue(dset_v2 >= 0);
GCHandle hnd = GCHandle.Alloc(wdata, GCHandleType.Pinned);
Assert.IsTrue(H5D.write(dset_v0, dtype, H5S.ALL,
H5S.ALL, H5P.DEFAULT, hnd.AddrOfPinnedObject()) >= 0);
Assert.IsTrue(H5D.write(dset_v2, dtype, H5S.ALL,
H5S.ALL, H5P.DEFAULT, hnd.AddrOfPinnedObject()) >= 0);
hnd.Free();
Assert.IsTrue(H5T.close(dtype) >= 0);
Assert.IsTrue(H5D.close(dset_v2) >= 0);
Assert.IsTrue(H5D.close(dset_v0) >= 0);
}
public void H5DreadTest2()
{
hid_t mem_type = H5T.create(H5T.class_t.STRING, H5T.VARIABLE);
Assert.IsTrue(H5T.set_cset(mem_type, H5T.cset_t.UTF8) >= 0);
Assert.IsTrue(H5T.set_strpad(mem_type, H5T.str_t.NULLTERM) >= 0);
hid_t fspace = H5D.get_space(m_v0_utf8_dset);
Assert.IsTrue(fspace >= 0);
hssize_t count = H5S.get_simple_extent_npoints(fspace);
Assert.IsTrue(count > 0);
Assert.IsTrue(H5S.close(fspace) >= 0);
IntPtr[] rdata = new IntPtr[count];
GCHandle hnd = GCHandle.Alloc(rdata, GCHandleType.Pinned);
Assert.IsTrue(H5D.read(m_v0_utf8_dset, mem_type, H5S.ALL, H5S.ALL,
H5P.DEFAULT, hnd.AddrOfPinnedObject()) >= 0);
for (int i = 0; i < rdata.Length; ++i)
{
int len = 0;
while (Marshal.ReadByte(rdata[i], len) != 0)
{
++len;
}
byte[] buffer = new byte[len];
Marshal.Copy(rdata[i], buffer, 0, buffer.Length);
string s = Encoding.UTF8.GetString(buffer);
Assert.IsTrue(s == ((string)m_utf8strings[i]));
Assert.IsTrue(H5.free_memory(rdata[i]) >= 0);
}
Assert.IsTrue(H5D.read(m_v2_utf8_dset, mem_type, H5S.ALL, H5S.ALL,
H5P.DEFAULT, hnd.AddrOfPinnedObject()) >= 0);
for (int i = 0; i < rdata.Length; ++i)
{
int len = 0;
while (Marshal.ReadByte(rdata[i], len) != 0)
{
++len;
}
byte[] buffer = new byte[len];
Marshal.Copy(rdata[i], buffer, 0, buffer.Length);
string s = Encoding.UTF8.GetString(buffer);
Assert.IsTrue(s == ((string)m_utf8strings[i]));
Assert.IsTrue(H5.free_memory(rdata[i]) >= 0);
}
hnd.Free();
}
public static (int success, hid_t CreatedgroupId) WriteStringAttributes(hid_t groupId, string name, IEnumerable <string> strs, string datasetName = null)
{
hid_t tmpId = groupId;
if (!string.IsNullOrWhiteSpace(datasetName))
{
hid_t datasetId = H5D.open(groupId, datasetName);
if (datasetId > 0)
{
groupId = datasetId;
}
}
// create UTF-8 encoded attributes
hid_t datatype = H5T.create(H5T.class_t.STRING, H5T.VARIABLE);
H5T.set_cset(datatype, H5T.cset_t.UTF8);
H5T.set_strpad(datatype, H5T.str_t.SPACEPAD);
int strSz = strs.Count();
hid_t spaceId = H5S.create_simple(1,
new ulong[] { (ulong)strSz }, null);
var attributeId = H5A.create(groupId, name, datatype, spaceId);
GCHandle[] hnds = new GCHandle[strSz];
IntPtr[] wdata = new IntPtr[strSz];
int cntr = 0;
foreach (string str in strs)
{
hnds[cntr] = GCHandle.Alloc(
Encoding.UTF8.GetBytes(str),
GCHandleType.Pinned);
wdata[cntr] = hnds[cntr].AddrOfPinnedObject();
cntr++;
}
var hnd = GCHandle.Alloc(wdata, GCHandleType.Pinned);
var result = H5A.write(attributeId, datatype, hnd.AddrOfPinnedObject());
hnd.Free();
for (int i = 0; i < strSz; ++i)
{
hnds[i].Free();
}
H5A.close(attributeId);
H5S.close(spaceId);
H5T.close(datatype);
if (tmpId != groupId)
{
H5D.close(groupId);
}
return(result, attributeId);
}
/// <summary>
/// Creates the type for UTF8 strings.
/// </summary>
/// <returns></returns>
private static long GetStringType()
{
long dtype = H5T.create(H5T.class_t.STRING, H5T.VARIABLE);
H5T.set_cset(dtype, H5T.cset_t.UTF8);
H5T.set_strpad(dtype, H5T.str_t.SPACEPAD);
return(dtype);
}
public static unsafe void AddString(long fileId, ContainerType container)
{
long res;
var dims = new ulong[] { 2, 2, 3 }; /* "extendible contiguous non-external dataset not allowed" */
// fixed length string attribute (ASCII)
var typeIdFixed = H5T.copy(H5T.C_S1);
res = H5T.set_size(typeIdFixed, new IntPtr(2));
res = H5T.set_cset(typeIdFixed, H5T.cset_t.ASCII);
var dataFixed = new string[] { "00", "11", "22", "33", "44", "55", "66", "77", " ", "AA", "ZZ", "!!" };
var dataFixedChar = dataFixed
.SelectMany(value => Encoding.ASCII.GetBytes(value))
.ToArray();
TestUtils.Add(container, fileId, "string", "fixed", typeIdFixed, dataFixedChar.AsSpan(), dims);
res = H5T.close(typeIdFixed);
// variable length string attribute (ASCII)
var typeIdVar = H5T.copy(H5T.C_S1);
res = H5T.set_size(typeIdVar, H5T.VARIABLE);
res = H5T.set_cset(typeIdVar, H5T.cset_t.ASCII);
var dataVar = new string[] { "00", "11", "22", "33", "44", "55", "66", "77", " ", "AA", "ZZ", "!!" };
var dataVarIntPtr = dataVar.Select(x => Marshal.StringToCoTaskMemUTF8(x)).ToArray();
TestUtils.Add(container, fileId, "string", "variable", typeIdVar, dataVarIntPtr.AsSpan(), dims);
foreach (var ptr in dataVarIntPtr)
{
Marshal.FreeCoTaskMem(ptr);
}
res = H5T.close(typeIdVar);
// variable length string attribute (UTF8)
var typeIdVarUTF8 = H5T.copy(H5T.C_S1);
res = H5T.set_size(typeIdVarUTF8, H5T.VARIABLE);
res = H5T.set_cset(typeIdVarUTF8, H5T.cset_t.UTF8);
var dataVarUTF8 = new string[] { "00", "11", "22", "33", "44", "55", "66", "77", " ", "ÄÄ", "的的", "!!" };
var dataVarUTF8IntPtr = dataVarUTF8.Select(x => Marshal.StringToCoTaskMemUTF8(x)).ToArray();
TestUtils.Add(container, fileId, "string", "variableUTF8", typeIdVarUTF8, dataVarUTF8IntPtr.AsSpan(), dims);
foreach (var ptr in dataVarUTF8IntPtr)
{
Marshal.FreeCoTaskMem(ptr);
}
res = H5T.close(typeIdVarUTF8);
}
public static (bool success, IEnumerable <string> result) ReadStrings(long groupId, string name, string alternativeName)
{
long datatype = H5T.create(H5T.class_t.STRING, H5T.VARIABLE);
H5T.set_cset(datatype, H5T.cset_t.UTF8);
H5T.set_strpad(datatype, H5T.str_t.NULLTERM);
//name = ToHdf5Name(name);
var datasetId = H5D.open(groupId, Hdf5Utils.NormalizedName(name));
if (datasetId < 0) //does not exist?
{
datasetId = H5D.open(groupId, Hdf5Utils.NormalizedName(alternativeName));
}
if (datasetId <= 0)
{
Hdf5Utils.LogError?.Invoke($"Error reading {groupId}. Name:{name}. AlternativeName:{alternativeName}");
return(false, Array.Empty <string>());
}
long spaceId = H5D.get_space(datasetId);
long count = H5S.get_simple_extent_npoints(spaceId);
H5S.close(spaceId);
var strs = new List <string>();
if (count >= 0)
{
IntPtr[] rdata = new IntPtr[count];
GCHandle hnd = GCHandle.Alloc(rdata, GCHandleType.Pinned);
H5D.read(datasetId, datatype, H5S.ALL, H5S.ALL,
H5P.DEFAULT, hnd.AddrOfPinnedObject());
for (int i = 0; i < rdata.Length; ++i)
{
int len = 0;
while (Marshal.ReadByte(rdata[i], len) != 0)
{
++len;
}
byte[] buffer = new byte[len];
Marshal.Copy(rdata[i], buffer, 0, buffer.Length);
string s = Encoding.UTF8.GetString(buffer);
strs.Add(s);
// H5.free_memory(rdata[i]);
}
hnd.Free();
}
H5T.close(datatype);
H5D.close(datasetId);
return(true, strs);
}
public static (int success, long CreatedgroupId) WriteStrings(long groupId, string name, IEnumerable <string> strs)
{
// create UTF-8 encoded test datasets
long datatype = H5T.create(H5T.class_t.STRING, H5T.VARIABLE);
H5T.set_cset(datatype, H5T.cset_t.ASCII);
H5T.set_strpad(datatype, H5T.str_t.NULLTERM);
int strSz = strs.Count();
long spaceId = H5S.create_simple(1, new[] { (ulong)strSz }, null);
string normalizedName = Hdf5Utils.NormalizedName(name);
var datasetId = Hdf5Utils.GetDatasetId(groupId, normalizedName, datatype, spaceId);
if (datasetId == -1L)
{
return(-1, -1L);
}
GCHandle[] hnds = new GCHandle[strSz];
IntPtr[] wdata = new IntPtr[strSz];
int cntr = 0;
foreach (string str in strs)
{
hnds[cntr] = GCHandle.Alloc(
Encoding.UTF8.GetBytes(str),
GCHandleType.Pinned);
wdata[cntr] = hnds[cntr].AddrOfPinnedObject();
cntr++;
}
var hnd = GCHandle.Alloc(wdata, GCHandleType.Pinned);
var result = H5D.write(datasetId, datatype, H5S.ALL, H5S.ALL,
H5P.DEFAULT, hnd.AddrOfPinnedObject());
hnd.Free();
for (int i = 0; i < strSz; ++i)
{
hnds[i].Free();
}
H5D.close(datasetId);
H5S.close(spaceId);
H5T.close(datatype);
return(result, datasetId);
}
public static int WriteStrings(hid_t groupId, string name, IEnumerable <string> strs, string datasetName = null)
{
// create UTF-8 encoded test datasets
hid_t datatype = H5T.create(H5T.class_t.STRING, H5T.VARIABLE);
H5T.set_cset(datatype, H5T.cset_t.UTF8);
H5T.set_strpad(datatype, H5T.str_t.SPACEPAD);
int strSz = strs.Count();
hid_t spaceId = H5S.create_simple(1,
new ulong[] { (ulong)strSz }, null);
var datasetId = H5D.create(groupId, name, datatype, spaceId);
GCHandle[] hnds = new GCHandle[strSz];
IntPtr[] wdata = new IntPtr[strSz];
int cntr = 0;
foreach (string str in strs)
{
hnds[cntr] = GCHandle.Alloc(
Encoding.UTF8.GetBytes(str),
GCHandleType.Pinned);
wdata[cntr] = hnds[cntr].AddrOfPinnedObject();
cntr++;
}
var hnd = GCHandle.Alloc(wdata, GCHandleType.Pinned);
var result = H5D.write(datasetId, datatype, H5S.ALL, H5S.ALL,
H5P.DEFAULT, hnd.AddrOfPinnedObject());
hnd.Free();
for (int i = 0; i < strSz; ++i)
{
hnds[i].Free();
}
H5D.close(datasetId);
H5S.close(spaceId);
H5T.close(datatype);
return(result);
}
public static IEnumerable <string> ReadStrings(hid_t groupId, string name)
{
hid_t datatype = H5T.create(H5T.class_t.STRING, H5T.VARIABLE);
H5T.set_cset(datatype, H5T.cset_t.UTF8);
H5T.set_strpad(datatype, H5T.str_t.NULLTERM);
//name = ToHdf5Name(name);
var datasetId = H5D.open(groupId, name);
hid_t spaceId = H5D.get_space(datasetId);
long count = H5S.get_simple_extent_npoints(spaceId);
H5S.close(spaceId);
var strs = new List <string>();
if (count >= 0)
{
IntPtr[] rdata = new IntPtr[count];
GCHandle hnd = GCHandle.Alloc(rdata, GCHandleType.Pinned);
H5D.read(datasetId, datatype, H5S.ALL, H5S.ALL,
H5P.DEFAULT, hnd.AddrOfPinnedObject());
for (int i = 0; i < rdata.Length; ++i)
{
int len = 0;
while (Marshal.ReadByte(rdata[i], len) != 0)
{
++len;
}
byte[] buffer = new byte[len];
Marshal.Copy(rdata[i], buffer, 0, buffer.Length);
string s = Encoding.UTF8.GetString(buffer);
strs.Add(s);
// H5.free_memory(rdata[i]);
}
hnd.Free();
}
H5T.close(datatype);
H5D.close(datasetId);
return(strs);
}
public void H5DwriteTest2()
{
ArrayList utf8strings = new ArrayList()
{
"Ελληνικά", "日本語", "العربية"
};
hid_t dtype = H5T.create(H5T.class_t.STRING, H5T.VARIABLE);
Assert.IsTrue(H5T.set_cset(dtype, H5T.cset_t.UTF8) >= 0);
Assert.IsTrue(H5T.set_strpad(dtype, H5T.str_t.SPACEPAD) >= 0);
hid_t dspace = H5S.create_simple(1,
new hsize_t[] { (hsize_t)utf8strings.Count }, null);
hid_t dset = H5D.create(m_v0_test_file, "dset", dtype, dspace);
Assert.IsTrue(dset >= 0);
GCHandle[] hnds = new GCHandle[utf8strings.Count];
IntPtr[] wdata = new IntPtr[utf8strings.Count];
for (int i = 0; i < utf8strings.Count; ++i)
{
hnds[i] = GCHandle.Alloc(
Encoding.UTF8.GetBytes((string)utf8strings[i]),
GCHandleType.Pinned);
wdata[i] = hnds[i].AddrOfPinnedObject();
}
GCHandle hnd = GCHandle.Alloc(wdata, GCHandleType.Pinned);
Assert.IsTrue(H5D.write(dset, dtype, H5S.ALL, H5S.ALL, H5P.DEFAULT,
hnd.AddrOfPinnedObject()) >= 0);
hnd.Free();
for (int i = 0; i < utf8strings.Count; ++i)
{
hnds[i].Free();
}
Assert.IsTrue(H5D.close(dset) >= 0);
Assert.IsTrue(H5S.close(dspace) >= 0);
Assert.IsTrue(H5T.close(dtype) >= 0);
}
public static void WriteStrings(int groupId, string name, IEnumerable <string> strs)
{
// create UTF-8 encoded test datasets
int typeId = H5T.create(H5T.class_t.STRING, H5T.VARIABLE);
H5T.set_cset(typeId, H5T.cset_t.UTF8);
H5T.set_strpad(typeId, H5T.str_t.SPACEPAD);
int strSz = strs.Count();
int spaceId = H5S.create_simple(1,
new ulong[] { (ulong)strSz }, null);
name = ToHdf5Name(name);
var datasetId = H5D.create(groupId, name, typeId, spaceId);
GCHandle[] hnds = new GCHandle[strSz];
IntPtr[] wdata1 = new IntPtr[strSz];
foreach (var item in strs.Select((str, i) => new { i, str }))
{
hnds[item.i] = GCHandle.Alloc(
Encoding.UTF8.GetBytes((string)item.str),
GCHandleType.Pinned);
wdata1[item.i] = hnds[item.i].AddrOfPinnedObject();
}
var hnd = GCHandle.Alloc(wdata1, GCHandleType.Pinned);
H5D.write(datasetId, typeId, H5S.ALL, H5S.ALL,
H5P.DEFAULT, hnd.AddrOfPinnedObject());
hnd.Free();
for (int i = 0; i < strSz; ++i)
{
hnds[i].Free();
}
H5S.close(spaceId);
H5T.close(typeId);
}
private static hid_t CreateFixedStringType(byte[] bytes, bool utf8)
{
#if true
var type = H5T.create(H5T.class_t.STRING, new IntPtr(bytes.Length));
if (type < 0)
{
throw new Exception("Failed to create string type");
}
#else
var type = H5T.copy(H5T.C_S1);
if (type < 0)
{
throw new Exception("Failed to create string type");
}
// TODO: bytes.Length or bytes.Length + 1?
if (H5T.set_size(type, new IntPtr(bytes.Length)) < 0)
{
H5T.close(type);
throw new Exception("Failed to set type size");
}
#endif
if (utf8)
{
if (H5T.set_cset(type, H5T.cset_t.UTF8) < 0)
{
H5T.close(type);
throw new Exception("Failed to set cset");
}
}
if (H5T.set_strpad(type, H5T.str_t.NULLTERM) < 0)
{
H5T.close(type);
throw new Exception("Failed to set strpad");
}
return(type);
}
public static string ReadUnicodeString(hid_t groupId, string name)
{
hid_t datatype = H5T.create(H5T.class_t.STRING, H5T.VARIABLE);
H5T.set_cset(datatype, H5T.cset_t.UTF8);
H5T.set_strpad(datatype, H5T.str_t.NULLTERM);
var datasetId = H5D.open(groupId, name);
var typeId = H5D.get_type(datasetId);
var classId = H5T.get_class(typeId);
var order = H5T.get_order(typeId);
IntPtr size = H5T.get_size(typeId);
int strLen = (int)size;
var spaceId = H5D.get_space(datasetId);
hid_t count = H5S.get_simple_extent_npoints(spaceId);
IntPtr[] rdata = new IntPtr[count];
byte[] wdata = new byte[strLen];
GCHandle hnd = GCHandle.Alloc(rdata, GCHandleType.Pinned);
H5D.read(datasetId, datatype, H5S.ALL, H5S.ALL,
H5P.DEFAULT, hnd.AddrOfPinnedObject());
for (int i = 0; i < strLen; i++)
{
Marshal.ReadByte(rdata[0], i);
}
Marshal.Copy(rdata[0], wdata, 0, strLen);
string s = Encoding.UTF8.GetString(wdata);
hnd.Free();
H5S.close(spaceId);
H5T.close(datatype);
H5D.close(datasetId);
return(s);
}
public static string ReadUnicodeString(hid_t groupId, string name)
{
hid_t datatype = H5T.create(H5T.class_t.STRING, H5T.VARIABLE);
H5T.set_cset(datatype, H5T.cset_t.UTF8);
H5T.set_strpad(datatype, H5T.str_t.SPACEPAD);
var datasetId = H5D.open(groupId, name);
var typeId = H5D.get_type(datasetId);
var classId = H5T.get_class(typeId);
var order = H5T.get_order(typeId);
IntPtr size = H5T.get_size(typeId);
int strLen = (int)size;
var spaceId = H5D.get_space(datasetId);
byte[] wdata = new byte[strLen];
//IntPtr ptr = new IntPtr();
GCHandle hnd = GCHandle.Alloc(wdata, GCHandleType.Pinned);
H5D.read(datasetId, datatype, H5S.ALL, H5S.ALL,
H5P.DEFAULT, hnd.AddrOfPinnedObject());
hnd.Free();
//int len = 0;
//while (Marshal.ReadByte(ptr, len) != 0) { ++len; }
//byte[] name_buf = new byte[len];
//Marshal.Copy(ptr, name_buf, 0, len);
string s = Encoding.UTF8.GetString(wdata);
H5S.close(spaceId);
H5T.close(datatype);
H5D.close(datasetId);
return(s);
}
public static int WriteUnicodeString(hid_t groupId, string name, string str, H5T.str_t strPad = H5T.str_t.SPACEPAD)
{
byte[] wdata = Encoding.UTF8.GetBytes(str);
hid_t spaceId = H5S.create(H5S.class_t.SCALAR);
hid_t dtype = H5T.create(H5T.class_t.STRING, new IntPtr(wdata.Length));
H5T.set_cset(dtype, H5T.cset_t.UTF8);
H5T.set_strpad(dtype, strPad);
hid_t datasetId = H5D.create(groupId, name, dtype, spaceId);
GCHandle hnd = GCHandle.Alloc(wdata, GCHandleType.Pinned);
int result = H5D.write(datasetId, dtype, H5S.ALL,
H5S.ALL, H5P.DEFAULT, hnd.AddrOfPinnedObject());
hnd.Free();
H5T.close(dtype);
H5D.close(datasetId);
H5S.close(spaceId);
return(result);
}
public static int WriteUnicodeString(long groupId, string name, string str, H5T.str_t strPad = H5T.str_t.SPACEPAD)
{
byte[] wdata = Hdf5Utils.StringToByte(str);
long spaceId = H5S.create(H5S.class_t.SCALAR);
long dtype = H5T.create(H5T.class_t.STRING, new IntPtr(wdata.Length));
H5T.set_cset(dtype, Hdf5Utils.GetCharacterSet(Settings.CharacterSetType));
H5T.set_strpad(dtype, strPad);
long datasetId = H5D.create(groupId, Hdf5Utils.NormalizedName(name), dtype, spaceId);
GCHandle hnd = GCHandle.Alloc(wdata, GCHandleType.Pinned);
int result = H5D.write(datasetId, dtype, H5S.ALL,
H5S.ALL, H5P.DEFAULT, hnd.AddrOfPinnedObject());
hnd.Free();
H5T.close(dtype);
H5D.close(datasetId);
H5S.close(spaceId);
return(result);
}
public static bool WriteVariableStringAttribute(hid_t hid, string key, IEnumerable <string> values, bool utf8)
{
if (key == null)
{
throw new ArgumentNullException("key");
}
if (values == null)
{
throw new ArgumentNullException("values");
}
var exists = H5A.exists(hid, key);
if (exists < 0)
{
throw new Exception("H5A.exists failed");
}
if (exists == 0) // Attribute doesn't exist
{
#if true
var type = H5T.create(H5T.class_t.STRING, H5T.VARIABLE);
if (type < 0)
{
throw new Exception("Failed to create string type");
}
#else
var type = H5T.copy(H5T.C_S1);
if (type < 0)
{
throw new Exception("Failed to create string type");
}
if (H5T.set_size(type, H5T.VARIABLE) < 0)
{
H5T.close(type);
throw new Exception("Failed to set type size");
}
#endif
if (utf8)
{
if (H5T.set_cset(type, H5T.cset_t.UTF8) < 0)
{
H5T.close(type);
throw new Exception("Failed to set cset");
}
}
if (H5T.set_strpad(type, H5T.str_t.NULLTERM) < 0)
{
H5T.close(type);
throw new Exception("Failed to set strpad");
}
var space = values.Count() == 1 ? H5S.create(H5S.class_t.SCALAR)
: H5S.create_simple(1, new ulong[1] {
(ulong)values.Count()
}, null);
if (space < 0)
{
H5T.close(type);
throw new Exception("Failed to create data space");
}
var attribute = H5A.create(hid, key, type, space);
H5S.close(space);
if (attribute < 0)
{
H5T.close(type);
throw new Exception(string.Format("Failed to create attribute \"{0}\"", key));
}
var pinnedObjects = new PinnedObject[values.Count()];
var data = new IntPtr[values.Count()];
int count = 0;
foreach (string str in values)
{
var bytes = str.ToBytes(utf8);
pinnedObjects[count] = new PinnedObject(bytes);
data[count] = pinnedObjects[count];
count += 1;
}
H5A.write(attribute, type, new PinnedObject(data));
H5T.close(type);
H5A.close(attribute);
}
else
{
// Attribute exists.
var attribute = H5A.open(hid, key);
if (attribute < 0)
{
throw new Exception(string.Format("Failed to open attribute \"{0}\"", key));
}
var type = H5A.get_type(attribute);
if (type < 0)
{
H5A.close(attribute);
throw new Exception("Failed to get data type");
}
var pinnedObjects = new PinnedObject[values.Count()];
var data = new IntPtr[values.Count()];
int count = 0;
foreach (string str in values)
{
var bytes = str.ToBytes(utf8);
pinnedObjects[count] = new PinnedObject(bytes);
data[count] = pinnedObjects[count];
count += 1;
}
H5A.write(attribute, type, new PinnedObject(data));
H5T.close(type);
H5A.close(attribute);
}
return(true);
}
public static long GetHdfTypeIdFromType(long fileId, Type type)
{
Type elementType;
elementType = type.IsArray ? type.GetElementType() : type;
if (elementType == typeof(bool))
{
return(H5T.NATIVE_UINT8);
}
else if (elementType == typeof(Byte))
{
return(H5T.NATIVE_UINT8);
}
else if (elementType == typeof(SByte))
{
return(H5T.NATIVE_INT8);
}
else if (elementType == typeof(UInt16))
{
return(H5T.NATIVE_UINT16);
}
else if (elementType == typeof(Int16))
{
return(H5T.NATIVE_INT16);
}
else if (elementType == typeof(UInt32))
{
return(H5T.NATIVE_UINT32);
}
else if (elementType == typeof(Int32))
{
return(H5T.NATIVE_INT32);
}
else if (elementType == typeof(UInt64))
{
return(H5T.NATIVE_UINT64);
}
else if (elementType == typeof(Int64))
{
return(H5T.NATIVE_INT64);
}
else if (elementType == typeof(Single))
{
return(H5T.NATIVE_FLOAT);
}
else if (elementType == typeof(Double))
{
return(H5T.NATIVE_DOUBLE);
}
else if (elementType == typeof(string) || elementType == typeof(IntPtr))
{
long typeId = 0;
if (H5I.is_valid(fileId) > 0 && H5L.exists(fileId, "string_t") > 0)
{
typeId = H5T.open(fileId, "string_t");
}
else
{
typeId = H5T.copy(H5T.C_S1);
H5T.set_size(typeId, H5T.VARIABLE);
H5T.set_cset(typeId, H5T.cset_t.UTF8);
if (fileId > -1 && H5T.commit(fileId, "string_t", typeId) < 0)
{
throw new Exception(ErrorMessage.TypeConversionHelper_CouldNotCommitDataType);
}
}
return(typeId);
}
else if (elementType.IsValueType && !elementType.IsPrimitive && !elementType.IsEnum)
{
long typeId = 0;
if (H5I.is_valid(fileId) > 0 && H5L.exists(fileId, elementType.Name) > 0)
{
typeId = H5T.open(fileId, elementType.Name);
}
else
{
typeId = H5T.create(H5T.class_t.COMPOUND, new IntPtr(Marshal.SizeOf(elementType)));
foreach (FieldInfo fieldInfo in elementType.GetFields())
{
long fieldType = -1;
fieldType = TypeConversionHelper.GetHdfTypeIdFromType(fileId, fieldInfo.FieldType);
H5T.insert(typeId, fieldInfo.Name, Marshal.OffsetOf(elementType, fieldInfo.Name), fieldType);
if (H5I.is_valid(fieldType) > 0)
{
H5T.close(fieldType);
}
}
if (fileId > -1 && H5T.commit(fileId, elementType.Name, typeId) < 0)
{
throw new Exception(ErrorMessage.TypeConversionHelper_CouldNotCommitDataType);
}
}
return(typeId);
}
else
{
throw new NotSupportedException();
}
}
public static unsafe void AddString(long fileId, ContainerType container)
{
long res;
var dims = new ulong[] { 2, 2, 3 }; /* "extendible contiguous non-external dataset not allowed" */
// fixed length string attribute + null terminate (ASCII)
var typeIdFixed_nullterm = H5T.copy(H5T.C_S1);
res = H5T.set_size(typeIdFixed_nullterm, new IntPtr(4));
res = H5T.set_cset(typeIdFixed_nullterm, H5T.cset_t.ASCII);
res = H5T.set_strpad(typeIdFixed_nullterm, H5T.str_t.NULLTERM);
var dataFixed_nullterm = new string[] { "00\00", "11\0 ", "22\0 ", "3\0 ", "44 \0", "555\0", "66 \0", "77\0 ", " \0 ", "AA \0", "ZZ \0", "!!\0 " };
var dataFixedChar_nullterm = dataFixed_nullterm
.SelectMany(value => Encoding.ASCII.GetBytes(value))
.ToArray();
TestUtils.Add(container, fileId, "string", "fixed+nullterm", typeIdFixed_nullterm, dataFixedChar_nullterm.AsSpan(), dims);
res = H5T.close(typeIdFixed_nullterm);
// fixed length string attribute + null padding (ASCII)
var typeIdFixed_nullpad = H5T.copy(H5T.C_S1);
res = H5T.set_size(typeIdFixed_nullpad, new IntPtr(4));
res = H5T.set_cset(typeIdFixed_nullpad, H5T.cset_t.ASCII);
res = H5T.set_strpad(typeIdFixed_nullpad, H5T.str_t.NULLPAD);
var dataFixed_nullpad = new string[] { "0\00\0", "11\0\0", "22\0\0", "3 \0\0", " 4\0\0", "55 5", "66\0\0", "77\0\0", " \0\0", "AA\0\0", "ZZ\0\0", "!!\0\0" };
var dataFixedChar_nullpad = dataFixed_nullpad
.SelectMany(value => Encoding.ASCII.GetBytes(value))
.ToArray();
TestUtils.Add(container, fileId, "string", "fixed+nullpad", typeIdFixed_nullpad, dataFixedChar_nullpad.AsSpan(), dims);
res = H5T.close(typeIdFixed_nullpad);
// fixed length string attribute + space padding (ASCII)
var typeIdFixed_spacepad = H5T.copy(H5T.C_S1);
res = H5T.set_size(typeIdFixed_spacepad, new IntPtr(4));
res = H5T.set_cset(typeIdFixed_spacepad, H5T.cset_t.ASCII);
res = H5T.set_strpad(typeIdFixed_spacepad, H5T.str_t.SPACEPAD);
var dataFixed_spacepad = new string[] { "00 ", "11 ", "22 ", "3 ", " 4 ", "55 5", "66 ", "77 ", " ", "AA ", "ZZ ", "!! " };
var dataFixedChar_spacepad = dataFixed_spacepad
.SelectMany(value => Encoding.ASCII.GetBytes(value))
.ToArray();
TestUtils.Add(container, fileId, "string", "fixed+spacepad", typeIdFixed_spacepad, dataFixedChar_spacepad.AsSpan(), dims);
res = H5T.close(typeIdFixed_spacepad);
// variable length string attribute (ASCII)
var typeIdVar = H5T.copy(H5T.C_S1);
res = H5T.set_size(typeIdVar, H5T.VARIABLE);
res = H5T.set_cset(typeIdVar, H5T.cset_t.ASCII);
res = H5T.set_strpad(typeIdVar, H5T.str_t.NULLPAD);
var dataVar = new string[] { "001", "11", "22", "33", "44", "55", "66", "77", " ", "AA", "ZZ", "!!" };
var dataVarChar = dataVar
.SelectMany(value => Encoding.ASCII.GetBytes(value + '\0'))
.ToArray();
fixed(byte *dataVarPtr = dataVarChar)
{
var basePtr = new IntPtr(dataVarPtr);
var addresses = new IntPtr[]
{
IntPtr.Add(basePtr, 0), IntPtr.Add(basePtr, 4), IntPtr.Add(basePtr, 7), IntPtr.Add(basePtr, 10),
IntPtr.Add(basePtr, 13), IntPtr.Add(basePtr, 16), IntPtr.Add(basePtr, 19), IntPtr.Add(basePtr, 22),
IntPtr.Add(basePtr, 25), IntPtr.Add(basePtr, 28), IntPtr.Add(basePtr, 31), IntPtr.Add(basePtr, 34)
};
fixed(void *dataVarAddressesPtr = addresses)
{
TestUtils.Add(container, fileId, "string", "variable", typeIdVar, dataVarAddressesPtr, dims);
}
}
res = H5T.close(typeIdVar);
// variable length string attribute + space padding (ASCII)
var typeIdVar_spacepad = H5T.copy(H5T.C_S1);
res = H5T.set_size(typeIdVar_spacepad, H5T.VARIABLE);
res = H5T.set_cset(typeIdVar_spacepad, H5T.cset_t.ASCII);
res = H5T.set_strpad(typeIdVar_spacepad, H5T.str_t.SPACEPAD);
var dataVar_spacepad = new string[] { "001 ", "1 1 ", "22 ", "33", "44", "55", "66", "77", " ", "AA", "ZZ", "!!" };
var dataVarChar_spacepad = dataVar_spacepad
.SelectMany(value => Encoding.ASCII.GetBytes(value + '\0'))
.ToArray();
fixed(byte *dataVarPtr_spacepad = dataVarChar_spacepad)
{
var basePtr = new IntPtr(dataVarPtr_spacepad);
var addresses = new IntPtr[]
{
IntPtr.Add(basePtr, 0), IntPtr.Add(basePtr, 6), IntPtr.Add(basePtr, 11), IntPtr.Add(basePtr, 16),
IntPtr.Add(basePtr, 19), IntPtr.Add(basePtr, 22), IntPtr.Add(basePtr, 25), IntPtr.Add(basePtr, 28),
IntPtr.Add(basePtr, 31), IntPtr.Add(basePtr, 34), IntPtr.Add(basePtr, 37), IntPtr.Add(basePtr, 40)
};
fixed(void *addressesPtr = addresses)
{
TestUtils.Add(container, fileId, "string", "variable+spacepad", typeIdVar_spacepad, addressesPtr, dims);
}
}
res = H5T.close(typeIdVar);
// variable length string attribute (UTF8)
var typeIdVarUTF8 = H5T.copy(H5T.C_S1);
res = H5T.set_size(typeIdVarUTF8, H5T.VARIABLE);
res = H5T.set_cset(typeIdVarUTF8, H5T.cset_t.UTF8);
res = H5T.set_strpad(typeIdVarUTF8, H5T.str_t.NULLPAD);
var dataVarUTF8 = new string[] { "00", "111", "22", "33", "44", "55", "66", "77", " ", "ÄÄ", "的的", "!!" };
var dataVarCharUTF8 = dataVarUTF8
.SelectMany(value => Encoding.UTF8.GetBytes(value + '\0'))
.ToArray();
fixed(byte *dataVarPtrUTF8 = dataVarCharUTF8)
{
var basePtr = new IntPtr(dataVarPtrUTF8);
var addresses = new IntPtr[]
{
IntPtr.Add(basePtr, 0), IntPtr.Add(basePtr, 3), IntPtr.Add(basePtr, 7), IntPtr.Add(basePtr, 10),
IntPtr.Add(basePtr, 13), IntPtr.Add(basePtr, 16), IntPtr.Add(basePtr, 19), IntPtr.Add(basePtr, 22),
IntPtr.Add(basePtr, 25), IntPtr.Add(basePtr, 28), IntPtr.Add(basePtr, 33), IntPtr.Add(basePtr, 40)
};
fixed(void *addressesPtr = addresses)
{
TestUtils.Add(container, fileId, "string", "variableUTF8", typeIdVarUTF8, addressesPtr, dims);
}
}
}
private static long GetHdfTypeIdFromType(Type type)
{
var elementType = type.IsArray ? type.GetElementType() : type;
if (elementType == typeof(bool))
{
return(H5T.NATIVE_UINT8);
}
else if (elementType == typeof(byte))
{
return(H5T.NATIVE_UINT8);
}
else if (elementType == typeof(sbyte))
{
return(H5T.NATIVE_INT8);
}
else if (elementType == typeof(ushort))
{
return(H5T.NATIVE_UINT16);
}
else if (elementType == typeof(short))
{
return(H5T.NATIVE_INT16);
}
else if (elementType == typeof(uint))
{
return(H5T.NATIVE_UINT32);
}
else if (elementType == typeof(int))
{
return(H5T.NATIVE_INT32);
}
else if (elementType == typeof(ulong))
{
return(H5T.NATIVE_UINT64);
}
else if (elementType == typeof(long))
{
return(H5T.NATIVE_INT64);
}
else if (elementType == typeof(float))
{
return(H5T.NATIVE_FLOAT);
}
else if (elementType == typeof(double))
{
return(H5T.NATIVE_DOUBLE);
}
// issues: https://en.wikipedia.org/wiki/Long_double
//else if (elementType == typeof(decimal))
// return H5T.NATIVE_LDOUBLE;
else if (elementType.IsEnum)
{
var baseTypeId = TestUtils.GetHdfTypeIdFromType(Enum.GetUnderlyingType(elementType));
var typeId = H5T.enum_create(baseTypeId);
foreach (var value in Enum.GetValues(type))
{
var value_converted = Convert.ToInt64(value);
var name = Enum.GetName(type, value_converted);
var handle = GCHandle.Alloc(value_converted, GCHandleType.Pinned);
H5T.enum_insert(typeId, name, handle.AddrOfPinnedObject());
}
return(typeId);
}
else if (elementType == typeof(string) || elementType == typeof(IntPtr))
{
var typeId = H5T.copy(H5T.C_S1);
H5T.set_size(typeId, H5T.VARIABLE);
H5T.set_cset(typeId, H5T.cset_t.UTF8);
return(typeId);
}
else if (elementType.IsValueType && !elementType.IsPrimitive)
{
var typeId = H5T.create(H5T.class_t.COMPOUND, new IntPtr(Marshal.SizeOf(elementType)));
foreach (var fieldInfo in elementType.GetFields())
{
var fieldType = TestUtils.GetHdfTypeIdFromType(fieldInfo.FieldType);
var attribute = fieldInfo.GetCustomAttribute <H5NameAttribute>(true);
var hdfFieldName = attribute is not null ? attribute.Name : fieldInfo.Name;
H5T.insert(typeId, hdfFieldName, Marshal.OffsetOf(elementType, fieldInfo.Name), fieldType);
if (H5I.is_valid(fieldType) > 0)
{
H5T.close(fieldType);
}
}
return(typeId);
}
else
{
throw new NotSupportedException();
}
}
public static void ClassInit(TestContext testContext)
{
// create test files which persists across file tests
m_v0_class_file = Utilities.H5TempFile(ref m_v0_class_file_name,
H5F.libver_t.EARLIEST);
Assert.IsTrue(m_v0_class_file >= 0);
m_v2_class_file = Utilities.H5TempFile(ref m_v2_class_file_name);
Assert.IsTrue(m_v2_class_file >= 0);
m_space_null = H5S.create(H5S.class_t.NULL);
Assert.IsTrue(m_space_null >= 0);
m_space_scalar = H5S.create(H5S.class_t.SCALAR);
Assert.IsTrue(m_space_scalar >= 0);
// create two datasets of the extended ASCII character set
// store as H5T.FORTRAN_S1 -> space padding
hsize_t[] dims = { 256 };
hid_t space = H5S.create_simple(1, dims, null);
m_v0_ascii_dset = H5D.create(m_v0_class_file, "ASCII",
H5T.FORTRAN_S1, space);
m_v2_ascii_dset = H5D.create(m_v2_class_file, "ASCII",
H5T.FORTRAN_S1, space);
Assert.IsTrue(H5S.close(space) >= 0);
// we write from C and must provide null-terminated strings
byte[] wdata = new byte[512];
for (int i = 0; i < 256; ++i)
{
wdata[2 * i] = (byte)i;
}
hid_t mem_type = H5T.copy(H5T.C_S1);
Assert.IsTrue(H5T.set_size(mem_type, new IntPtr(2)) >= 0);
GCHandle hnd = GCHandle.Alloc(wdata, GCHandleType.Pinned);
Assert.IsTrue(H5D.write(m_v0_ascii_dset, mem_type, H5S.ALL,
H5S.ALL, H5P.DEFAULT, hnd.AddrOfPinnedObject()) >= 0);
Assert.IsTrue(H5D.write(m_v2_ascii_dset, mem_type, H5S.ALL,
H5S.ALL, H5P.DEFAULT, hnd.AddrOfPinnedObject()) >= 0);
hnd.Free();
Assert.IsTrue(H5T.close(mem_type) >= 0);
// create UTF-8 encoded test datasets
hid_t dtype = H5T.create(H5T.class_t.STRING, H5T.VARIABLE);
Assert.IsTrue(H5T.set_cset(dtype, H5T.cset_t.UTF8) >= 0);
Assert.IsTrue(H5T.set_strpad(dtype, H5T.str_t.SPACEPAD) >= 0);
hid_t dspace = H5S.create_simple(1,
new hsize_t[] { (hsize_t)m_utf8strings.Count }, null);
m_v0_utf8_dset = H5D.create(m_v0_class_file, "UTF-8", dtype, dspace);
Assert.IsTrue(m_v0_utf8_dset >= 0);
m_v2_utf8_dset = H5D.create(m_v2_class_file, "UTF-8", dtype, dspace);
Assert.IsTrue(m_v2_utf8_dset >= 0);
GCHandle[] hnds = new GCHandle[m_utf8strings.Count];
IntPtr[] wdata1 = new IntPtr[m_utf8strings.Count];
for (int i = 0; i < m_utf8strings.Count; ++i)
{
hnds[i] = GCHandle.Alloc(
Encoding.UTF8.GetBytes((string)m_utf8strings[i]),
GCHandleType.Pinned);
wdata1[i] = hnds[i].AddrOfPinnedObject();
}
hnd = GCHandle.Alloc(wdata1, GCHandleType.Pinned);
Assert.IsTrue(H5D.write(m_v0_utf8_dset, dtype, H5S.ALL, H5S.ALL,
H5P.DEFAULT, hnd.AddrOfPinnedObject()) >= 0);
Assert.IsTrue(H5D.write(m_v2_utf8_dset, dtype, H5S.ALL, H5S.ALL,
H5P.DEFAULT, hnd.AddrOfPinnedObject()) >= 0);
hnd.Free();
for (int i = 0; i < m_utf8strings.Count; ++i)
{
hnds[i].Free();
}
Assert.IsTrue(H5S.close(dspace) >= 0);
Assert.IsTrue(H5T.close(dtype) >= 0);
}
private static bool CreateOrOverwriteVariableStringAttribute(hid_t hid, string key, IEnumerable <string> values, bool utf8)
{
if (H5A.exists(hid, key) == 0)
{
// Attribute doesn't exist.
#if true
var type = H5T.create(H5T.class_t.STRING, H5T.VARIABLE);
if (type < 0)
{
return(false);
}
#else
var type = H5T.copy(H5T.C_S1);
if (type < 0)
{
return(false);
}
H5T.set_size(type, H5T.VARIABLE);
#endif
if (utf8)
{
H5T.set_cset(type, H5T.cset_t.UTF8);
}
H5T.set_strpad(type, H5T.str_t.NULLTERM);
var space = values.Count() == 1 ? H5S.create(H5S.class_t.SCALAR)
: H5S.create_simple(1, new ulong[1] {
(ulong)values.Count()
}, null);
if (space < 0)
{
H5T.close(type);
return(false);
}
var attribute = H5A.create(hid, key, type, space);
if (attribute < 0)
{
H5S.close(space);
H5T.close(type);
return(false);
}
H5S.close(space);
var pinnedObjects = new PinnedObject[values.Count()];
var data = new IntPtr[values.Count()];
int count = 0;
foreach (string str in values)
{
var bytes = str.ToBytes(utf8);
pinnedObjects[count] = new PinnedObject(bytes);
data[count] = pinnedObjects[count];
count += 1;
}
H5A.write(attribute, type, new PinnedObject(data));
H5T.close(type);
H5A.close(attribute);
}
else
{
// Attribute exists.
var attribute = H5A.open(hid, key);
if (attribute < 0)
{
return(false);
}
var type = H5A.get_type(attribute);
if (type < 0)
{
H5A.close(attribute);
return(false);
}
var pinnedObjects = new PinnedObject[values.Count()];
var data = new IntPtr[values.Count()];
int count = 0;
foreach (string str in values)
{
var bytes = str.ToBytes(utf8);
pinnedObjects[count] = new PinnedObject(bytes);
data[count] = pinnedObjects[count];
count += 1;
}
H5A.write(attribute, type, new PinnedObject(data));
H5T.close(type);
H5A.close(attribute);
}
return(true);
}
private static bool CreateOrOverwriteFixedStringAttribute(hid_t hid, string key, string value, bool utf8)
{
if (H5A.exists(hid, key) == 0)
{
// Attribute doesn't exist.
var bytes = value.ToBytes(utf8);
#if true
var type = H5T.create(H5T.class_t.STRING, new IntPtr(bytes.Length));
if (type < 0)
{
return(false);
}
#else
var type = H5T.copy(H5T.C_S1);
if (type < 0)
{
return(false);
}
H5T.set_size(type, new IntPtr(bytes.Length));
#endif
if (utf8)
{
H5T.set_cset(type, H5T.cset_t.UTF8);
}
H5T.set_strpad(type, H5T.str_t.NULLTERM);
var space = H5S.create(H5S.class_t.SCALAR);
if (space < 0)
{
H5T.close(type);
return(false);
}
var attribute = H5A.create(hid, key, type, space);
if (attribute < 0)
{
H5S.close(space);
H5T.close(type);
return(false);
}
H5A.write(attribute, type, new PinnedObject(bytes));
H5A.close(attribute);
}
else
{
// Attribute exists.
var attribute = H5A.open(hid, key);
if (attribute < 0)
{
return(false);
}
var type = H5A.get_type(attribute);
H5A.write(attribute, type, new PinnedObject(value.ToBytes(utf8)));
H5T.close(type);
H5A.close(attribute);
}
return(true);
}
