public SlotStream(LocalHeap heap, ExternalFileListSlot slot, long offset, H5DatasetAccess datasetAccess)
{
_heap = heap;
_slot = slot;
this.Offset = offset;
_datasetAccess = datasetAccess;
}
public ExternalFileListStream(ExternalFileListMessage externalFileList, H5DatasetAccess datasetAccess)
{
var offset = 0L;
_slotStreams = externalFileList
.SlotDefinitions
.Select(slot =>
{
var stream = new SlotStream(externalFileList.Heap, slot, offset, datasetAccess);
offset += stream.Length;
return(stream);
})
.ToArray();
if (_slotStreams.Any())
{
this.Length =
_slotStreams.Last().Offset +
_slotStreams.Last().Length;
}
else
{
throw new Exception("There must at least a single file be defined in the external file list.");
}
}
public T[] ReadCompound <T>(
Func <FieldInfo, string>?getName = default,
Selection?fileSelection = default,
Selection?memorySelection = default,
ulong[]?memoryDims = default,
H5DatasetAccess datasetAccess = default) where T : struct
{
var data = this.Read <byte>(
null,
fileSelection,
memorySelection,
memoryDims,
datasetAccess,
skipShuffle: false);
if (data is null)
{
throw new Exception("The buffer is null. This should never happen.");
}
if (getName is null)
{
getName = fieldInfo => fieldInfo.Name;
}
return(H5Utils.ReadCompound <T>(this.InternalDataType, this.InternalDataspace, this.Context.Superblock, data, getName));
}
public H5D_Base(H5Dataset dataset, bool supportsBuffer, bool supportsStream, H5DatasetAccess datasetAccess)
{
this.Dataset = dataset;
this.SupportsBuffer = supportsBuffer;
this.SupportsStream = supportsStream;
this.DatasetAccess = datasetAccess;
}
public void Read <T>(
Memory <T> buffer,
Selection?fileSelection = default,
Selection?memorySelection = default,
ulong[]?memoryDims = default,
H5DatasetAccess datasetAccess = default) where T : unmanaged
{
this.Read(
buffer,
fileSelection,
memorySelection,
memoryDims,
datasetAccess,
skipShuffle: false);
}
private T[] ReadContiguous <T>(H5DatasetAccess datasetAccess) where T : struct
{
ulong address;
if (this.DataLayout is DataLayoutMessage12 layout12)
{
address = layout12.DataAddress;
}
else if (this.DataLayout is DataLayoutMessage3 layout34)
{
var contiguous = (ContiguousStoragePropertyDescription)layout34.Properties;
address = contiguous.Address;
}
else
{
throw new Exception($"Data layout message type '{this.DataLayout.GetType().Name}' is not supported.");
}
// read data
var buffer = this.GetBuffer <T>(out var result);
if (this.Context.Superblock.IsUndefinedAddress(address))
{
if (this.ExternalFileList != null)
{
this.ReadExternalFileList(buffer, this.ExternalFileList, datasetAccess);
}
else if (this.FillValue.IsDefined)
{
buffer.Fill(this.FillValue.Value);
}
}
else
{
this.Context.Reader.Seek((long)address, SeekOrigin.Begin);
this.Context.Reader.Read(buffer);
}
this.EnsureEndianness(buffer);
return(result);
}
public T[] Read <T>(
Selection?fileSelection = default,
Selection?memorySelection = default,
ulong[]?memoryDims = default,
H5DatasetAccess datasetAccess = default) where T : unmanaged
{
var result = this.Read <T>(
null,
fileSelection,
memorySelection,
memoryDims,
datasetAccess,
skipShuffle: false);
if (result is null)
{
throw new Exception("The buffer is null. This should never happen.");
}
return(result);
}
public string[] ReadString(
Selection?fileSelection = default,
Selection?memorySelection = default,
ulong[]?memoryDims = default,
H5DatasetAccess datasetAccess = default)
{
var data = this.Read <byte>(
null,
fileSelection,
memorySelection,
memoryDims,
datasetAccess,
skipShuffle: false,
skipTypeCheck: true);
if (data is null)
{
throw new Exception("The buffer is null. This should never happen.");
}
return(H5Utils.ReadString(this.InternalDataType, data, this.Context.Superblock));
}
public H5D_Chunk4_BTree2(H5Dataset dataset, DataLayoutMessage4 layout, H5DatasetAccess datasetAccess)
: base(dataset, layout, datasetAccess)
{
}
private void ReadExternalFileList(Span <byte> buffer, ExternalFileListMessage externalFileList, H5DatasetAccess datasetAccess)
{
var bufferOffset = 0;
var remainingSize = buffer.Length;
foreach (var slotDefinition in externalFileList.SlotDefinitions)
{
var length = Math.Min(remainingSize, (int)slotDefinition.Size);
var heap = externalFileList.Heap;
var name = heap.GetObjectName(slotDefinition.NameHeapOffset);
var filePath = H5Utils.ConstructExternalFilePath(name, datasetAccess);
if (!File.Exists(filePath))
{
throw new Exception($"External file '{filePath}' does not exist.");
}
try
{
using var fileStream = File.OpenRead(filePath);
fileStream.Seek((long)slotDefinition.Offset, SeekOrigin.Begin);
var actualLength = Math.Min(length, fileStream.Length);
var currentBuffer = buffer.Slice(bufferOffset, (int)actualLength);
fileStream.Read(currentBuffer);
}
catch
{
throw new Exception($"Unable to open external file '{filePath}'.");
}
bufferOffset += length;
remainingSize -= length;
}
}
public H5D_Chunk123_BTree1(H5Dataset dataset, DataLayoutMessage12 layout, H5DatasetAccess datasetAccess) :
base(dataset, datasetAccess)
{
_layout12 = layout;
}
/* Reading large files
* Compact: no problem
* Contiguous: just make sure that the hyperslab is divided into < 2 GB chunks
* Chunked: Chunk size is max 2 GB, but decompressed data will be larger. This means
* that the returned buffer must not be a Span<T> or Memory<T>.
* Virtual: a combination of the solutions above
*/
internal T[] Read <T>(H5DatasetAccess datasetAccess, bool skipTypeCheck = false, bool skipShuffle = false) where T : struct
{
if (!skipTypeCheck)
{
switch (this.Datatype.Class)
{
case DatatypeMessageClass.FixedPoint:
case DatatypeMessageClass.FloatingPoint:
case DatatypeMessageClass.BitField:
case DatatypeMessageClass.Opaque:
case DatatypeMessageClass.Compound:
case DatatypeMessageClass.Reference:
case DatatypeMessageClass.Enumerated:
case DatatypeMessageClass.Array:
break;
default:
throw new Exception($"This method can only be used with one of the following type classes: '{DatatypeMessageClass.FixedPoint}', '{DatatypeMessageClass.FloatingPoint}', '{DatatypeMessageClass.BitField}', '{DatatypeMessageClass.Opaque}', '{DatatypeMessageClass.Compound}', '{DatatypeMessageClass.Reference}', '{DatatypeMessageClass.Enumerated}' and '{DatatypeMessageClass.Array}'.");
}
}
// for testing only
if (skipShuffle && this.FilterPipeline != null)
{
var filtersToRemove = this
.FilterPipeline
.FilterDescriptions
.Where(description => description.Identifier == FilterIdentifier.Shuffle)
.ToList();
foreach (var filter in filtersToRemove)
{
this.FilterPipeline.FilterDescriptions.Remove(filter);
}
}
switch (this.DataLayout.LayoutClass)
{
// Compact: The array is stored in one contiguous block as part of
// this object header message.
case LayoutClass.Compact:
return(this.ReadCompact <T>());
// Contiguous: The array is stored in one contiguous area of the file.
// This layout requires that the size of the array be constant:
// data manipulations such as chunking, compression, checksums,
// or encryption are not permitted. The message stores the total
// storage size of the array. The offset of an element from the
// beginning of the storage area is computed as in a C array.
case LayoutClass.Contiguous:
return(this.ReadContiguous <T>(datasetAccess));
// Chunked: The array domain is regularly decomposed into chunks,
// and each chunk is allocated and stored separately. This layout
// supports arbitrary element traversals, compression, encryption,
// and checksums (these features are described in other messages).
// The message stores the size of a chunk instead of the size of the
// entire array; the storage size of the entire array can be
// calculated by traversing the chunk index that stores the chunk
// addresses.
case LayoutClass.Chunked:
return(this.ReadChunked <T>());
// Virtual: This is only supported for version 4 of the Data Layout
// message. The message stores information that is used to locate
// the global heap collection containing the Virtual Dataset (VDS)
// mapping information. The mapping associates the VDS to the source
// dataset elements that are stored across a collection of HDF5 files.
case LayoutClass.VirtualStorage:
throw new NotImplementedException();
default:
throw new Exception($"The data layout class '{this.DataLayout.LayoutClass}' is not supported.");
}
}
public string[] ReadString(H5DatasetAccess datasetAccess = default)
{
var data = this.Read <byte>(datasetAccess, skipTypeCheck: true);
return(H5Utils.ReadString(this.Datatype, data, this.Context.Superblock));
}
public unsafe T[] ReadCompound <T>(Func <FieldInfo, string> getName, H5DatasetAccess datasetAccess = default) where T : struct
{
var data = this.Read <byte>(datasetAccess);
return(H5Utils.ReadCompound <T>(this.Datatype, this.Dataspace, this.Context.Superblock, data, getName));
}
public T[] ReadCompound <T>(H5DatasetAccess datasetAccess = default) where T : struct
{
return(this.ReadCompound <T>(fieldInfo => fieldInfo.Name, datasetAccess));
}
public T[] Read <T>(H5DatasetAccess datasetAccess = default) where T : struct
{
return(this.Read <T>(datasetAccess, skipShuffle: false));
}
public H5D_Compact(H5Dataset dataset, H5DatasetAccess datasetAccess) :
base(dataset, supportsBuffer: true, supportsStream: false, datasetAccess)
{
//
}
public H5D_Chunk4_FixedArray(H5Dataset dataset, DataLayoutMessage4 layout, H5DatasetAccess datasetAccess) :
base(dataset, layout, datasetAccess)
{
//
}
public H5D_Chunk123_BTree1(H5Dataset dataset, DataLayoutMessage3 layout, H5DatasetAccess datasetAccess) :
base(dataset, datasetAccess)
{
_layout3 = layout;
_chunked3 = (ChunkedStoragePropertyDescription3)_layout3.Properties;
}
internal T[]? Read <T>(
Memory <T> buffer,
Selection?fileSelection = default,
Selection?memorySelection = default,
ulong[]?memoryDims = default,
H5DatasetAccess datasetAccess = default,
bool skipTypeCheck = false,
bool skipShuffle = false) where T : unmanaged
{
// short path for null dataspace
if (this.InternalDataspace.Type == DataspaceType.Null)
{
return(new T[0]);
}
//
if (!skipTypeCheck)
{
switch (this.InternalDataType.Class)
{
case DatatypeMessageClass.FixedPoint:
case DatatypeMessageClass.FloatingPoint:
case DatatypeMessageClass.BitField:
case DatatypeMessageClass.Opaque:
case DatatypeMessageClass.Compound:
case DatatypeMessageClass.Reference:
case DatatypeMessageClass.Enumerated:
case DatatypeMessageClass.Array:
break;
default:
throw new Exception($"This method can only be used with one of the following type classes: '{DatatypeMessageClass.FixedPoint}', '{DatatypeMessageClass.FloatingPoint}', '{DatatypeMessageClass.BitField}', '{DatatypeMessageClass.Opaque}', '{DatatypeMessageClass.Compound}', '{DatatypeMessageClass.Reference}', '{DatatypeMessageClass.Enumerated}' and '{DatatypeMessageClass.Array}'.");
}
}
// for testing only
if (skipShuffle && this.InternalFilterPipeline is not null)
{
var filtersToRemove = this
.InternalFilterPipeline
.FilterDescriptions
.Where(description => description.Identifier == FilterIdentifier.Shuffle)
.ToList();
foreach (var filter in filtersToRemove)
{
this.InternalFilterPipeline.FilterDescriptions.Remove(filter);
}
}
/* buffer provider */
using H5D_Base bufferProvider = this.InternalDataLayout.LayoutClass switch
{
/* Compact: The array is stored in one contiguous block as part of
* this object header message.
*/
LayoutClass.Compact => new H5D_Compact(this, datasetAccess),
/* Contiguous: The array is stored in one contiguous area of the file.
* This layout requires that the size of the array be constant:
* data manipulations such as chunking, compression, checksums,
* or encryption are not permitted. The message stores the total
* storage size of the array. The offset of an element from the
* beginning of the storage area is computed as in a C array.
*/
LayoutClass.Contiguous => new H5D_Contiguous(this, datasetAccess),
/* Chunked: The array domain is regularly decomposed into chunks,
* and each chunk is allocated and stored separately. This layout
* supports arbitrary element traversals, compression, encryption,
* and checksums (these features are described in other messages).
* The message stores the size of a chunk instead of the size of the
* entire array; the storage size of the entire array can be
* calculated by traversing the chunk index that stores the chunk
* addresses.
*/
LayoutClass.Chunked => H5D_Chunk.Create(this, datasetAccess),
/* Virtual: This is only supported for version 4 of the Data Layout
* message. The message stores information that is used to locate
* the global heap collection containing the Virtual Dataset (VDS)
* mapping information. The mapping associates the VDS to the source
* dataset elements that are stored across a collection of HDF5 files.
*/
LayoutClass.VirtualStorage => throw new NotImplementedException(),
/* default */
_ => throw new Exception($"The data layout class '{this.InternalDataLayout.LayoutClass}' is not supported.")
};
bufferProvider.Initialize();
Func <ulong[], Memory <byte> >?getSourceBuffer = bufferProvider.SupportsBuffer
? chunkIndices => bufferProvider.GetBuffer(chunkIndices)
: null;
Func <ulong[], Stream>?getSourceStream = bufferProvider.SupportsStream
? chunkIndices => bufferProvider.GetStream(chunkIndices)
: null;
/* dataset dims */
var datasetDims = bufferProvider.GetDatasetDims();
/* dataset chunk dims */
var datasetChunkDims = bufferProvider.GetChunkDims();
/* file selection */
if (fileSelection is null)
{
fileSelection = bufferProvider.GetSelection();
}
/* result buffer */
var result = default(T[]);
var totalCount = fileSelection.GetTotalCount();
var byteSize = totalCount * this.InternalDataType.Size;
if (buffer.Equals(default))
public H5D_Contiguous(H5Dataset dataset, H5DatasetAccess datasetAccess) :
base(dataset, supportsBuffer: false, supportsStream: true, datasetAccess)
{
//
}
public H5D_Chunk4(H5Dataset dataset, DataLayoutMessage4 layout, H5DatasetAccess datasetAccess) :
base(dataset, datasetAccess)
{
this.Layout = layout;
this.Chunked4 = (ChunkedStoragePropertyDescription4)layout.Properties;
}
