//Load weights from hdf5 file. Weights must be saved as a vector per layer
public static float[] loadH5(string path, string dsname)
{
//Get file id
var h5fid = H5F.open(path, H5F.OpenMode.ACC_RDONLY);
//Get dataset id
var h5did = H5D.open(h5fid, dsname);
//Dataset size
var h5space = H5D.getSpace(h5did);
var h5size = H5S.getSimpleExtentDims(h5space);
//Dataset size to array
var S = h5size.ToArray();
//Empty double array for the data
double[] data = new double[S[0]];
//Read the dataset
var h5array = new H5Array <double>(data);
var h5dtype = H5D.getType(h5did);
H5D.read(h5did, h5dtype, h5array);
//Convert to float
float[] newarray = new float[data.Length];
Parallel.For(0, data.Length, (k) =>
{
newarray[k] = (float)data[k];
});
return(newarray);
}
/// <summary>
/// Writes the entire matrix table
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="matName"></param>
/// <param name="rowIndex"></param>
/// <returns></returns>
public void SetMatrix <T>(string matName, T[,] data)
{
// check that matrix exists
if (tables.ContainsKey(matName))
{
H5DataSetId matId;
tables.TryGetValue(matName, out matId);
H5DataTypeId matDataId = H5D.getType(matId);
H5DataSpaceId spaceId = H5S.create_simple(2, Shape);
long[] start = { 0, 0 };
long[] count = { Shape[0], Shape[1] };
var h5matrix = new H5Array <T>(data);
H5S.selectHyperslab(spaceId, H5S.SelectOperator.SET, start, count);
H5DataSpaceId readSpaceId = H5S.create_simple(2, count);
H5D.write(matId, matDataId, readSpaceId, spaceId, H5P.create(H5P.PropertyListClass.DATASET_XFER), h5matrix);
H5S.close(spaceId);
H5S.close(readSpaceId);
}
else
{
Console.WriteLine("table {0} not found in matrix file", matName);
}
return;
}
// Matrix Specific Methods
// TODO:
// 1. add handling for matrix title
// 2. add specification of NA values
// 3. other attributes: pa-format flag, year int, source string
/// <summary>
/// Returns a row of the matrix
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="matName"></param>
/// <param name="rowIndex"></param>
/// <returns></returns>
public T[] GetMatrixRow <T>(string matName, int rowIndex)
{
var rowData = new T[Shape[1]];
// check that matrix exists
if (tables.ContainsKey(matName))
{
H5DataSetId matId;
tables.TryGetValue(matName, out matId);
H5DataTypeId matDataId = H5D.getType(matId);
H5DataSpaceId spaceId = H5S.create_simple(2, Shape);
var h5matrix = new H5Array <T>(rowData);
long[] start = { rowIndex, 0 };
long[] count = { 1, Shape[1] };
H5S.selectHyperslab(spaceId, H5S.SelectOperator.SET, start, count);
H5DataSpaceId readSpaceId = H5S.create_simple(2, count);
H5D.read(matId, matDataId, readSpaceId, spaceId, H5P.create(H5P.PropertyListClass.DATASET_XFER), h5matrix);
H5S.close(spaceId);
H5S.close(readSpaceId);
}
else
{
Console.WriteLine("table {0} not found in matrix file", matName);
}
return(rowData);
}
/// <summary>
/// 写数据集属性
/// </summary>
public void WriteDatasetAttribute(string datasetName, string attrName, string value)
{
H5DataSetId datasetId = H5D.open(_fileId, datasetName);
H5DataTypeId typeId = H5T.copy(H5T.H5Type.C_S1);
H5DataSpaceId spaceId = H5S.create(H5S.H5SClass.SCALAR);
H5T.setSize(typeId, value.Length);
H5AttributeId attrId = H5A.create(datasetId, attrName, typeId, spaceId);
if (value != "")
{
H5Array <byte> buffer = new H5Array <byte>(Encoding.Default.GetBytes(value));
H5A.write(attrId, typeId, buffer);
}
if (typeId != null)
{
H5T.close(typeId);
}
if (spaceId != null)
{
H5S.close(spaceId);
}
if (attrId != null)
{
H5A.close(attrId);
}
if (datasetId != null)
{
H5D.close(datasetId);
}
}
public static T[,] Read2DArray <T>(this H5FileId fileId, string dataSetName)
{
var dataset = H5D.open(fileId, dataSetName);
var space = H5D.getSpace(dataset);
var dims = H5S.getSimpleExtentDims(space);
var dataType = H5D.getType(dataset);
if (typeof(T) == typeof(string))
{
// this will also need a string hack...
}
T[,] dataArray = new T[dims[0], dims[1]];
var wrapArray = new H5Array <T>(dataArray);
H5D.read(dataset, dataType, wrapArray);
return(dataArray);
}
private static int[] GetInt32DataSet(H5FileId dataFile, string path)
{
if (H5L.Exists(dataFile, path))
{
var dataSet = H5D.open(dataFile, path);
var space = H5D.getSpace(dataSet);
var size2 = H5S.getSimpleExtentDims(space);
long count = size2[0];
var dataArray = new Int32[count];
var wrapArray = new H5Array <Int32>(dataArray);
H5DataTypeId tid1 = H5D.getType(dataSet);
H5D.read(dataSet, tid1, wrapArray);
return(dataArray);
}
return(null);
}
//Reading and Printing Methods
private static double[] GetDoubleDataSet(H5FileId dataFile, string path)
{
if (H5L.Exists(dataFile, path))
{
H5DataSetId dataSet = H5D.open(dataFile, path);
H5DataSpaceId space = H5D.getSpace(dataSet);
long[] size2 = H5S.getSimpleExtentDims(space);
long count = size2[0];
double[] dataArray = new double[count];
H5Array <double> wrapArray = new H5Array <double>(dataArray);
H5DataTypeId tid1 = H5D.getType(dataSet);
H5D.read(dataSet, tid1, wrapArray);
return(dataArray);
}
return(null);
}
public static T[] Read1DArray <T>(H5FileId fileId, string dataSetName)
{
H5DataSetId dataset = null;
H5DataSpaceId space = null;
H5DataTypeId dataType = null;
long[] dims;
try
{
dataset = H5D.open(fileId, dataSetName);
space = H5D.getSpace(dataset);
dims = H5S.getSimpleExtentDims(space);
dataType = H5D.getType(dataset);
if (typeof(T) == typeof(string))
{
int stringLength = H5T.getSize(dataType);
int a = (int)dims[0];
byte[] buffer = new byte[(int)(dims[0]) * stringLength];
H5D.read(dataset, dataType, new H5Array <byte>(buffer));
string stuff = System.Text.ASCIIEncoding.ASCII.GetString(buffer);
return(stuff.SplitInParts(stringLength).Select(ss => (T)(object)ss).ToArray());
}
T[] dataArray = new T[dims[0]];
var wrapArray = new H5Array <T>(dataArray);
H5D.read(dataset, dataType, wrapArray);
return(dataArray);
}
catch {
return(null);
}
finally
{
if (space != null)
{
H5S.close(space);
}
if (dataset != null)
{
H5D.close(dataset);
}
}
}
public static T[] Read1DArray <T>(this H5FileId fileId, string dataSetName)
{
var dataset = H5D.open(fileId, dataSetName);
var space = H5D.getSpace(dataset);
var dims = H5S.getSimpleExtentDims(space);
var dataType = H5D.getType(dataset);
if (typeof(T) == typeof(string))
{
int stringLength = H5T.getSize(dataType);
byte[] buffer = new byte[dims[0] * stringLength];
H5D.read(dataset, dataType, new H5Array <byte>(buffer));
string stuff = System.Text.ASCIIEncoding.ASCII.GetString(buffer);
return(stuff.SplitInParts(stringLength).Select(ss => (T)(object)ss).ToArray());
}
T[] dataArray = new T[dims[0]];
var wrapArray = new H5Array <T>(dataArray);
H5D.read(dataset, dataType, wrapArray);
return(dataArray);
}
private static T[, ,] Read3DArray <T>(H5GroupId groupID, string name)
{
var dataset = H5D.open(groupID, name);
var space = H5D.getSpace(dataset);
var dims = H5S.getSimpleExtentDims(space);
var dataType = H5D.getType(dataset);
if (typeof(T) == typeof(string))
{
// this will also need a string hack...
T[, ,] dataArray = new T[dims[0], 0, 0];
return(dataArray);
}
else
{
T[, ,] dataArray = new T[dims[0], dims[1], dims[2]];
var wrapArray = new H5Array <T>(dataArray);
H5D.read(dataset, dataType, wrapArray);
return(dataArray);
}
}
public string GetFileApi(string source, string filename, string destination)
{
var fileId = H5F.open(source, H5F.OpenMode.ACC_RDONLY);
var datasetId = H5D.open(fileId, filename);
//var datasetTypeId = new H5DataTypeId(H5T.H5Type.NATIVE_OPAQUE);
//var space = H5D.getSpace(datasetId);
//var dims = H5S.getSimpleExtentDims(space);
var dataType = H5D.getType(datasetId);
long size = H5D.getStorageSize(datasetId);
var fileBytes = new byte[size];
var h5Array = new H5Array <byte>(fileBytes);
H5D.read(datasetId, dataType, h5Array);
H5D.close(datasetId);
H5F.close(fileId);
var outPath = Path.Combine(destination, filename);
File.WriteAllBytes(outPath, fileBytes);
return(destination);
}
private static void WriteHdf5Data(int[] values, string className, string p)
{
var fileId = GetFileId();
var setName = GetNamePrefix() + className + p;
var dim1 = GetChunkSize();
var writeSize = values.Length;
var dataType = new H5DataTypeId(H5T.H5Type.NATIVE_INT);
var dataSetId = CreateDatasetIfNoneExists(fileId, setName, dim1, dataType);
var wrapArray = new H5Array <int>(values);
var fileSpaceId = new H5DataSpaceId(H5S.H5SType.ALL);
var all = new H5DataSpaceId(H5S.H5SType.ALL);
var xferProp = H5P.create(H5P.PropertyListClass.DATASET_XFER);
var newSize = new long[] {
writeSize
};
H5D.setExtent(dataSetId, newSize);
H5D.write(dataSetId, dataType, all, fileSpaceId, xferProp, wrapArray);
H5F.close(fileId);
}
/// <summary>
/// Method to read an individual cell or block of data from a matrix
/// rowIndex and colIndex are the zero-based offsets into the matrix
/// rowLength and colLength are the size of the arrays (min 1)
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="matName"></param>
/// <param name="rowIndex"></param>
/// <param name="colIndex"></param>
/// <param name="rowLength"></param>
/// <param name="colLength"></param>
/// <returns></returns>
public T[,] GetMatrixBlock <T>(string matName, int rowIndex, int colIndex, int rowLength, int colLength)
{
// check that block parameters are legit
if ((rowLength < 1 | colLength < 1) | (rowIndex + rowLength > Shape[0] | colIndex + colLength > Shape[1]))
{
Console.WriteLine("invalid block size and/or index, must be non-zero in both dimensions and within matrix size");
return(null);
}
var blockData = new T[rowLength, colLength];
// check that matrix exists
if (tables.ContainsKey(matName))
{
H5DataSetId matId;
tables.TryGetValue(matName, out matId);
H5DataTypeId matDataId = H5D.getType(matId);
H5DataSpaceId spaceId = H5S.create_simple(2, Shape);
var h5matrix = new H5Array <T>(blockData);
long[] start = { rowIndex, colIndex };
long[] count = { rowLength, colLength };
H5S.selectHyperslab(spaceId, H5S.SelectOperator.SET, start, count);
H5DataSpaceId readSpaceId = H5S.create_simple(2, count);
H5D.read(matId, matDataId, readSpaceId, spaceId, H5P.create(H5P.PropertyListClass.DATASET_XFER), h5matrix);
H5S.close(spaceId);
H5S.close(readSpaceId);
}
else
{
Console.WriteLine("table {0} not found in matrix file", matName);
}
return(blockData);
}
static void Main(string[] args)
{
H5FileId file_id;
float[,] data = new float[718, 359];
H5Array<float> h5_data = new H5Array<float>(data);
try
{
file_id = H5F.open("gfs.hdf5", H5F.OpenMode.ACC_RDONLY);
if (file_id.Id >= 0)
{
H5GroupId i_gid = H5G.open(file_id, "/Results/wind modulus");
for (int i = 0; i < 200000; i++)
{
H5DataSetId i_dsid = H5D.open(i_gid, "wind modulus_00001");
H5DataTypeId i_dt = H5D.getType(i_dsid);
H5DataTypeId i_ndt = H5T.getNativeType(i_dt, H5T.Direction.DESCEND);
H5D.read(i_dsid, i_ndt, h5_data);
H5T.close(i_ndt);
H5T.close(i_dt);
H5D.close(i_dsid);
}
H5G.close(i_gid);
}
H5F.close(file_id);
}
catch (Exception ex)
{
Console.WriteLine("An exception happened. The message returned was:");
Console.WriteLine("{0}", ex.Message);
Console.ReadKey();
}
}
private static T[] Read1DArray <T>(H5GroupId fileId, string dataSetName)
{
var dataset = H5D.open(fileId, dataSetName);
var space = H5D.getSpace(dataset);
var dims = H5S.getSimpleExtentDims(space);
var dataType = H5D.getType(dataset);
T[] dataArray = null;
if (typeof(T) == typeof(string))
{
// this will also need a string hack...
dataArray = new T[dims[0]];
H5D.close(dataset);
return(dataArray);
}
else
{
dataArray = new T[dims[0]];
var wrapArray = new H5Array <T>(dataArray);
H5D.read(dataset, dataType, wrapArray);
H5D.close(dataset);
return(dataArray);
}
}
private static void WriteHdf5Data(double[] values, string className, string p)
{
H5FileId fileId = GetFileId();
string setName = GetNamePrefix() + className + p;
int dim1 = GetChunkSize();
int writeSize = values.Length;
H5DataTypeId dataType = new H5DataTypeId(H5T.H5Type.NATIVE_DOUBLE);
H5DataSetId dataSetId = CreateDatasetIfNoneExists(fileId, setName, dim1, dataType);
H5Array <double> wrapArray = new H5Array <double>(values);
H5DataSpaceId fileSpaceId = new H5DataSpaceId(H5S.H5SType.ALL);
H5DataSpaceId all = new H5DataSpaceId(H5S.H5SType.ALL);
H5PropertyListId xferProp = H5P.create(H5P.PropertyListClass.DATASET_XFER);
long[] newSize = new long[] {
writeSize
};
H5D.setExtent(dataSetId, newSize);
H5D.write(dataSetId, dataType, all, fileSpaceId, xferProp, wrapArray);
H5F.close(fileId);
}
public object ReadFile(string filename, Recording recording)
{
RecordingConfig recordingConfig = recording.parameters;
byte[] byteArray;
double[] doubleArray = new double[1];
string result = "";
//This list will contain the positions of the movements to load.
List<int> positionSelection;
try
{
H5FileId fileId = H5F.open(filename, H5F.OpenMode.ACC_RDONLY);
H5GroupId groupId = H5G.open(fileId, "/recSession");
H5ObjectInfo objectInfo = H5O.getInfoByName(fileId, "/recSession");
int nAttributes = (int)objectInfo.nAttributes;
H5AttributeId[] attributeIds = new H5AttributeId[nAttributes];
//Attribute extraction and filling of the RecordingConfig object
for (int i = 0; i < nAttributes; i++)
{
attributeIds[i] = H5A.openByIndex(groupId, "/recSession", H5IndexType.CRT_ORDER, H5IterationOrder.INCREASING, (long)i);
string attributeName = H5A.getName(attributeIds[i]);
H5DataTypeId attributeType = H5A.getType(attributeIds[i]);
H5AttributeInfo attributeInfo = H5A.getInfo(attributeIds[i]);
int dataSize = (int)attributeInfo.dataSize;
int typeSize = H5T.getSize(attributeType);
if ((attributeName == "mov") || (attributeName == "dev") || (attributeName == "cmt"))
{
byteArray = new byte[typeSize];
H5Array<byte> byteBuffer = new H5Array<byte>(byteArray);
H5A.read<byte>(attributeIds[i], attributeType, byteBuffer);
result = System.Text.Encoding.UTF8.GetString(byteArray);
}
else
{
doubleArray = new double[(int)(dataSize / typeSize)];
H5Array<double> doubleBuffer = new H5Array<double>(doubleArray);
H5A.read<double>(attributeIds[i], attributeType, doubleBuffer);
}
switch (attributeName)
{
case "cT": //contraction time
recordingConfig.scheduleItemTime = (float)doubleArray[0];
break;
case "rT": //rest time
_restTime = doubleArray[0];
break;
case "cmt": //comments
_comments = result;
break;
case "date":
_date = doubleArray;
break;
case "dev": //device name
_deviceName = result;
break;
case "mov": //list of movements
//TODO: Extract it to a label sequence. We will use these labels to construct the frame sequence
//Beware: each movement will possibly include rest periods. We can use the value of rT to determine
//if that is the case -- to guess where the rests are and label them as such!
//it seems that all movement recordings in Matlab exports begin with a CONTRACTION
_movements = result.Split(';');
//plan: concatenate all movements on all channels one after another
//This list will be used when writing MPTCE files as a way to store the movements known
//by the program. NOT ALL THE MOVEMENTS WILL NORMALLY APPEAR ON A SINGLE RECORDING.
//For the BioPatRec samples, it seems that this list actually holds the movements
//encoded in the data structure. That would mean that the only way to describe the movement
//would be the string content itself!
break;
case "nCh": //number of channels
recordingConfig.nChannels = (uint)doubleArray[0];
break;
case "nM": //number of movements
//Redundant with list of movements?
_numMovs = doubleArray[0];
break;
case "nR": //number of repetitions
recordingConfig.repetitions = (uint)doubleArray[0];
break;
case "sF": //sampling frequency
recordingConfig.sampleFreq = (uint)doubleArray[0];
break;
case "sT": //sample time -> duration of the recording for each movement, including rests (?)
_partialDuration = doubleArray[0];
break;
// The following are MPTCE-specific attributes not present in BioPatRec files
case "version": //Format version. If defined, this means that some version of MPTCE created it. If not, it
// surely is a Matlab export.
_version = doubleArray[0];
break;
case "mV":
recordingConfig.minVoltage = doubleArray[0];
_minmaxDefined = true;
break;
case "MV":
recordingConfig.maxVoltage = doubleArray[0];
_minmaxDefined = true;
break;
case "movSeq": //Array defining the sequence of and which movements listed in mov were recorded
_movSeq = doubleArray;
break;
default:
break;
}
H5A.close(attributeIds[i]);
}
//Now we do the data reading
H5DataSetId dataSetId = H5D.open(groupId, "tdata");
H5DataTypeId dataSetType = H5D.getType(dataSetId);
int dataTypeSize = (int)H5T.getSize(dataSetType);
H5DataSpaceId dataSpaceId = H5D.getSpace(dataSetId);
int nDims = H5S.getSimpleExtentNDims(dataSpaceId);
long[] dims = H5S.getSimpleExtentDims(dataSpaceId);
//On Matlab-created HDF5 files, the dimensions are as follows:
// 0 -> number of movement
// 1 -> number of channel
// 2-> samples per movement
double[, ,] data = new double[dims[0], dims[1], dims[2]];
H5Array<double> dataBuffer = new H5Array<double>(data);
H5D.read<double>(dataSetId, dataSetType, dataBuffer);
H5D.close(dataSetId);
H5G.close(groupId);
H5F.close(fileId);
//We try to identify the movements stored in the data array with a known movement in MPTCE
List<ScheduleItem> movCodes = GuessMovs(_movements, knownMovements, allowedComplexMovements);
//With this list, we can now select which movements we want to load. If requestedMovements is defined
//and has elements, we satisfy that request and load only those movements. Otherwise we present some kind
//of dialog, so that users can select the movements that they want
if (requestedMovements != null && requestedMovements.Count > 0)
{
//Initialize positionSelection with the position of each of the desired movements
//in the data array
positionSelection = FindMovPositions(requestedMovements, movCodes);
}
else
{
//Prompt for movements
movementSelector.movementNames = knownMovements;
movementSelector.availableMovements = movCodes;
positionSelection = FindMovPositions(_movementSelector.SelectMovements(), movCodes);
}
//We have the information needed to compose our recording data
recordingConfig.scheduleActive = false;
FillRecording(data, recording, positionSelection,movCodes);
}
catch (HDFException e)
{
throw new IFileReadWriterException("Error while reading HDF5 file.", e);
}
return recording;
}
public SkimMatrix Read(string filename, int field, float scale)
{
Console.WriteLine("Reading {0}", filename);
int hdf5NameEnd = filename.IndexOf("/");
// the first part of the name in the roster file is the omx/hdf5 file
string HDFName = filename.Substring(0, hdf5NameEnd);
//rename filename to be only the name of the skim matrix inside of the skim file
//skims are stored in the "data" folder within the omx/hdf5 file
filename = filename.Substring(hdf5NameEnd);
string hdfFile = _path + "\\" + HDFName;
var dataFile = H5F.open(hdfFile, H5F.OpenMode.ACC_RDONLY);
var dataSet = H5D.open(dataFile, filename);
var space = H5D.getSpace(dataSet);
var size2 = H5S.getSimpleExtentDims(space);
long nRows = size2[0];
long nCols = size2[1];
long numZones = _mapping.Count();
// if the count in the hdf5 file is larger than the number of
// tazs in the mapping, ignore the values over the total number
//of tazs in the mapping because these are not valid zones.
_matrix = new ushort[numZones][];
for (var i = 0; i < numZones; i++)
{
_matrix[i] = new ushort[numZones];
}
//OMX is a square matrix of doubles
//In addition to the data folder for matrices, an OMX file has a lookup folder
//with a zone mapping vector. However, this is ignored since DaySim also has one.
//Therefore, it is assumed the OMX matrix does not skip rows/cols and every row/col
//corresponds to an actual zone in the DaySim zone mapping file by index
//Scaling should be set to TRUE since OMX stores doubles (not scaled integers)
var dataArray = new double[nRows, nCols];
var wrapArray = new H5Array <double>(dataArray);
H5DataTypeId tid1 = H5D.getType(dataSet);
H5D.read(dataSet, tid1, wrapArray);
for (var row = 0; row < nRows; row++)
{
if (_mapping.ContainsKey(row + 1))
{
for (var col = 0; col < nCols; col++)
{
if (_mapping.ContainsKey(col + 1))
{
var value = dataArray[row, col] * scale;
if (value > 0)
{
if (value > ushort.MaxValue - 1)
{
value = ushort.MaxValue - 1;
}
_matrix[_mapping[row + 1]][_mapping[col + 1]] = (ushort)value;
}
}
}
}
}
var skimMatrix = new SkimMatrix(_matrix);
return(skimMatrix);
}
public SkimMatrix Read(string filename, int field, float scale)
{
//hdf5 filename contain "filename/group/skim"
//get the index of group
int hdf5GroupEnd = filename.LastIndexOf("/");
//get the index of filename
int hdf5NameEnd = hdf5GroupEnd > 0 ? filename.LastIndexOf("/", hdf5GroupEnd - 1) : -1;
//get the omx/hdf5 filename
string HDFName = filename.Substring(0, hdf5NameEnd);
//rename filename to be only the name of the skim matrix inside of the skim file
filename = filename.Substring(hdf5NameEnd);
string hdfFile = Path.Combine(_path, HDFName);
Console.WriteLine("Loading skim file: {0}.", hdfFile);
FileInfo file = new FileInfo(hdfFile);
if (!file.Exists)
{
throw new FileNotFoundException(string.Format("The skim file {0} could not be found.", file.FullName));
}
H5FileId dataFile = H5F.open(hdfFile, H5F.OpenMode.ACC_RDONLY);
H5DataSetId dataSet = H5D.open(dataFile, filename);
H5DataSpaceId space = H5D.getSpace(dataSet);
long[] size2 = H5S.getSimpleExtentDims(space);
long nRows = size2[0];
long nCols = size2[1];
long numZones = _mapping.Count();
// if the count in the hdf5 file is larger than the number of
// tazs in the mapping, ignore the values over the total number
//of tazs in the mapping because these are not valid zones.
_matrix = new ushort[numZones][];
for (int i = 0; i < numZones; i++)
{
_matrix[i] = new ushort[numZones];
}
//OMX is a square matrix of doubles
//In addition to the data folder for matrices, an OMX file has a lookup folder
//with a zone mapping vector. However, this is ignored since DaySim also has one.
//Therefore, it is assumed the OMX matrix does not skip rows/cols and every row/col
//corresponds to an actual zone in the DaySim zone mapping file by index
//Scaling should be set to TRUE since OMX stores doubles (not scaled integers)
double[,] dataArray = new double[nRows, nCols];
H5Array <double> wrapArray = new H5Array <double>(dataArray);
H5DataTypeId tid1 = H5D.getType(dataSet);
H5D.read(dataSet, tid1, wrapArray);
for (int row = 0; row < nRows; row++)
{
if (_mapping.ContainsKey(row + 1))
{
for (int col = 0; col < nCols; col++)
{
if (_mapping.ContainsKey(col + 1))
{
double value = dataArray[row, col] * scale;
if (value > 0)
{
if (value > ushort.MaxValue - 1)
{
value = ushort.MaxValue - 1;
}
_matrix[_mapping[row + 1]][_mapping[col + 1]] = (ushort)value;
}
}
}
}
}
SkimMatrix skimMatrix = new SkimMatrix(_matrix);
return(skimMatrix);
}
public SkimMatrix Read(string fileNameAndGroupAndDataTable, int field, float scale)
{
//mb merged these changes to local develop before pushing #210
//hdf5 filename contain "filename/group/skim"
//get the index of group
int hdf5GroupEnd = fileNameAndGroupAndDataTable.LastIndexOf("/");
string fileNameAndGroup = hdf5GroupEnd > 0 ? fileNameAndGroupAndDataTable.Substring(0, hdf5GroupEnd) : fileNameAndGroupAndDataTable;
//get the index of filename
int hdf5NameEnd = hdf5GroupEnd > 0 ? fileNameAndGroup.LastIndexOf("/") : -1;
string groupName = fileNameAndGroup.Substring(hdf5NameEnd + 1);
//get the omx/hdf5 filename
string HDFName = fileNameAndGroup.Substring(0, hdf5NameEnd);
string groupAndDataTable = fileNameAndGroupAndDataTable.Substring(hdf5NameEnd);
string hdfFile = Path.Combine(_path, HDFName);
FileInfo file = new FileInfo(hdfFile);
if (!file.Exists)
{
throw new FileNotFoundException(string.Format("The skim file {0} could not be found.", file.FullName));
}
H5FileId dataFile = H5F.open(hdfFile, H5F.OpenMode.ACC_RDONLY);
H5DataSetId dataSet = H5D.open(dataFile, groupAndDataTable);
H5DataSpaceId space = H5D.getSpace(dataSet);
long[] size2 = H5S.getSimpleExtentDims(space);
long nRows = size2[0];
long nCols = size2[1];
Debug.Assert(nRows == nCols);
long numZones = _mapping.Count();
int[] lookupMap = null;
string lookupMapName = null;
string lookupGroupName = "lookup";
H5GroupId luGroup = H5G.open(dataFile, lookupGroupName);
if (H5G.getNumObjects(luGroup) == 1L)
{
lookupMapName = H5G.getObjectNameByIndex(luGroup, 0);
H5DataSetId lookupDataSet = H5D.open(dataFile, string.Concat(lookupGroupName, "/", lookupMapName));
H5DataTypeId lookupMapType = H5D.getType(lookupDataSet);
H5DataSpaceId lookupMapSpace = H5D.getSpace(lookupDataSet);
long lookupMapSize = H5S.getSimpleExtentDims(lookupMapSpace)[0];
lookupMap = new int[lookupMapSize];
H5Array <int> lookupWrapArray = new H5Array <int>(lookupMap);
H5D.read(lookupDataSet, lookupMapType, lookupWrapArray);
}
if (lookupMap != null)
{
if (lookupMap.Length != nRows)
{
Global.PrintFile.WriteLine(string.Format("DATA WARNING: skim file: {0} has a lookup map named {1} but its length ({2}) is different than the matrix size ({3}) in group table {4}", hdfFile, lookupMapName, lookupMap.Length, nRows, groupAndDataTable));
lookupMap = null;
}
}
// if the count in the hdf5 file is larger than the number of
// tazs in the mapping, ignore the values over the total number
//of tazs in the mapping because these are not valid zones.
_matrix = new ushort[numZones][];
for (int i = 0; i < numZones; i++)
{
_matrix[i] = new ushort[numZones];
}
//OMX is a square matrix of doubles
//In addition to the data folder for matrices, an OMX file has a lookup folder
//with a zone mapping vector. However, this is ignored since DaySim also has one.
//Therefore, it is assumed the OMX matrix does not skip rows/cols and every row/col
//corresponds to an actual zone in the DaySim zone mapping file by index
double[,] dataArray = new double[nRows, nCols];
H5Array <double> wrapArray = new H5Array <double>(dataArray);
H5DataTypeId tid1 = H5D.getType(dataSet);
H5D.read(dataSet, tid1, wrapArray);
long numSuccessfuLookups = 0;
for (int row = 0; row < nRows; row++)
{
int mappingKey = (lookupMap == null) ? (row + 1) : lookupMap[row];
if (_mapping.TryGetValue(mappingKey, out int mappedRow))
{
for (int col = 0; col < nCols; col++)
{
mappingKey = (lookupMap == null) ? (col + 1) : lookupMap[col];
if (_mapping.TryGetValue(mappingKey, out int mappedCol))
{
double value = dataArray[row, col] * scale;
++numSuccessfuLookups;
if (value > 0)
{
if (value > ushort.MaxValue - 1)
{
value = ushort.MaxValue - 1;
}
_matrix[mappedRow][mappedCol] = (ushort)value; //bug #208 deferred but this will eventually changed be Convert.ToUInt16 to avoid 0.57*100=56 bug
}
}
} //for col
}
} //for row
Global.PrintFile.WriteLine(string.Format("Loaded skim file: {0} dataset: {1}, scale={6} with input matrix size={2} and DaySim matrix size={3} zones. Percentage successful mappings {4}. {5}", hdfFile, groupAndDataTable, nRows, numZones, (numSuccessfuLookups / (nRows * nCols)) * 100.0, lookupMap != null ? string.Format("Using lookupMap {0}.", lookupMapName) : "No lookupMap.", scale), true);
SkimMatrix skimMatrix = new SkimMatrix(_matrix);
return(skimMatrix);
}
public SkimMatrix Read(string filename, int field, float scale)
{
Console.WriteLine("Reading {0}", filename);
int hdf5NameEnd = filename.IndexOf("/");
// the first part of the name in the roster file is the hdf5 file:
string HDFName = filename.Substring(0, hdf5NameEnd);
//rename filename to be only the name of the skim inside of the time period file
filename = filename.Substring(hdf5NameEnd);
string hdfFile = _path + "\\" + HDFName;
var dataFile = H5F.open(hdfFile, H5F.OpenMode.ACC_RDONLY);
var dataSet = H5D.open(dataFile, filename);
var space = H5D.getSpace(dataSet);
var size2 = H5S.getSimpleExtentDims(space);
long nRows = size2[0];
long nCols = size2[1];
long numZones = _mapping.Count();
var dataArray = new double[nRows, nCols];
var wrapArray = new H5Array <double>(dataArray);
H5DataTypeId tid1 = H5D.getType(dataSet);
H5D.read(dataSet, tid1, wrapArray);
// if the count in the hdf5 file is larger than the number of
// tazs in the mapping, ignore the values over the total number
//of tazs in the mapping because these are not valid zones.
_matrix = new ushort[numZones][];
for (var i = 0; i < numZones; i++)
{
_matrix[i] = new ushort[numZones];
}
//leave as is for PSRC. Values are already scaled integers and matrices already condensed
if (Global.Configuration.PSRC)
{
for (var i = 0; i < numZones; i++)
{
for (var j = 0; j < numZones; j++)
{
_matrix[i][j] = (ushort)dataArray[i, j];
}
}
}
else
{
for (var row = 0; row < nRows; row++)
{
if (_mapping.ContainsKey(row + 1))
{
for (var col = 0; col < nCols; col++)
{
if (_mapping.ContainsKey(col + 1))
{
var value = dataArray[row, col] * scale;
if (value > 0)
{
if (value > short.MaxValue)
{
value = short.MaxValue;
}
_matrix[_mapping[row + 1]][_mapping[col + 1]] = (ushort)value;
}
}
}
}
}
}
var skimMatrix = new SkimMatrix(_matrix);
return(skimMatrix);
}
/// <summary>
/// Creates a new dataset from an HDF5 data file.
/// </summary>
/// <param name="blob">The input blob is reshaped to the dataset item shape.</param>
/// <param name="strDatasetName">Specifies the new dataset name.</param>
/// <param name="bReshape">Specifies whether to reshape the 'blob' parameter.</param>
/// <param name="nMinDim">Specifies the minimum dimension.</param>
/// <param name="nMaxDim">Specifies the maximum dimension.</param>
/// <param name="id">Optional, group ID to use instead of internal file (default = null).</param>
/// <param name="bAllowSingleItems">When true single item values are allowed and used to copy across entire blob.</param>
public void load_nd_dataset(Blob <T> blob, string strDatasetName, bool bReshape = false, int nMinDim = 1, int nMaxDim = int.MaxValue, H5GroupId id = null, bool bAllowSingleItems = false)
{
H5DataSetId ds = null;
int nSingleItemSize = 0;
try
{
Tuple <H5DataSetId, int> ds1 = load_nd_datasetEx(blob, strDatasetName, bReshape, nMinDim, nMaxDim, id, bAllowSingleItems);
ds = ds1.Item1;
nSingleItemSize = ds1.Item2;
H5DataTypeId dsType = H5D.getType(ds);
int nSize = H5T.getSize(dsType);
if (nSize == sizeof(double))
{
double[] rgBuffer = new double[blob.count()];
H5Array <double> rgData = new H5Array <double>(rgBuffer);
H5D.read <double>(ds, dsType, rgData);
if (!bAllowSingleItems || nSingleItemSize == 0)
{
blob.mutable_cpu_data = Utility.ConvertVec <T>(rgBuffer);
}
else
{
blob.SetData(rgBuffer[0]);
}
}
else if (nSize == sizeof(float))
{
float[] rgBuffer = new float[blob.count()];
H5Array <float> rgData = new H5Array <float>(rgBuffer);
H5D.read <float>(ds, dsType, rgData);
if (!bAllowSingleItems || nSingleItemSize == 0)
{
blob.mutable_cpu_data = Utility.ConvertVec <T>(rgBuffer);
}
else
{
blob.SetData(rgBuffer[0]);
}
}
else if (nSize == sizeof(byte))
{
byte[] rgBuffer = new byte[blob.count()];
H5Array <byte> rgData = new H5Array <byte>(rgBuffer);
H5D.read <byte>(ds, dsType, rgData);
float[] rgf = rgBuffer.Select(p1 => (float)p1).ToArray();
blob.mutable_cpu_data = Utility.ConvertVec <T>(rgf);
}
else
{
m_log.FAIL("The dataset size of '" + nSize.ToString() + "' is not supported!");
}
}
catch (Exception excpt)
{
m_log.FAIL(excpt.Message);
}
finally
{
if (ds != null)
{
H5D.close(ds);
}
}
}
private void button1_Click(object sender, EventArgs e)
{
fileNameTextBox.Text = "";
string filename = "";
if (openFileDialog1.ShowDialog() == DialogResult.OK)
{
if ((openFileDialog1.OpenFile()) != null)
{
filename = openFileDialog1.FileName;
fileNameTextBox.Text = openFileDialog1.FileName;
Debug.WriteLine(filename);
}
H5.Open();
var h5 = H5F.open(filename, H5F.OpenMode.ACC_RDONLY);
var dataset = H5D.open(h5, "/radarData/data");
var Space = H5D.getSpace(dataset);
var size = H5S.getSimpleExtentDims(Space);
float[, , , ,] dataarray = new float[size[0], size[1], size[2], size[3], size[4] * 2];
var wrapArray = new H5Array <float>(dataarray);
NOS = (int)size[2]; //number_of_scan (X)
NOC = (int)size[3]; //Number of channel (antenna) (Y)
NOF = (int)size[4]; // Number of frequency (Z)
if (NOS < NOF)
{
m = NOS;
last = n = 50;
}
else
{
m = NOS;
last = n = 50;
}
x = new float[m][];
xb = new float[m][];
c1 = new float[m][];
y1 = new float[m][];
for (int i = 0; i < m; i++)
{
x[i] = new float[n];
xb[i] = new float[n];
c1[i] = new float[n];
y1[i] = new float[n];
for (int j = 0; j < n; j++)
{
y1[i][j] = 0;
x[i][j] = 0;
}
}
textBox1.Text = size[2].ToString();
textBox2.Text = size[4].ToString();
textBox3.Text = size[3].ToString();
var dataType = H5D.getType(dataset);
H5D.read <float>(dataset, dataType, wrapArray);
data = new float[size[2], size[3], size[4] * 2];
var xd = data.Length;
Debug.WriteLine(xd);
for (int k = 0; k < size[2]; k++)
{
for (int i = 0; i < size[3]; i++)
{
for (int j = 0; j < size[4] * 2; j++)
{
data[k, i, j] = dataarray[0, 0, k, i, j];
}
}
}
// res = 10; //10mm
res = 1; //100mm
n_o_s = NOS; //640;// 510;//number of files
n_o_c = NOC * res; //100;// NOC* res; //* res; //for outdoor =NOC * res for indoor=100; 150 for outdoor as after 15th channel readings were not proper
n_o_f = NOF;
grid = new float[NOS, NOC *res, NOF];
for (int k = 0; k < NOS; k++) //100mm
{
for (int i = 0; i < NOC; i++)
{
for (int j = 0; j < NOF; j++)
{
grid[k, i, j] = data[k, i, j * 2];
}
}
}
H5.Close();
}
hscn = 0; depth = 0; dtscn = 0; dtdep = 0; chnl = 0; dtchnl = 0;
imagescanner(0, 0, 0);
}
