public static (long AttributeId, bool IsNew) OpenOrCreateAttribute(long locationId, string name, long attributeTypeId, ulong length, ulong[] dimensionLimitSet) { return(IOHelper.OpenOrCreateAttribute(locationId, name, attributeTypeId, () => { long dataspaceId = -1; long attributeId = -1; try { dataspaceId = H5S.create_simple(1, new ulong[] { length }, dimensionLimitSet); attributeId = H5A.create(locationId, name, attributeTypeId, dataspaceId); if (H5I.is_valid(attributeId) <= 0) { throw new Exception(ErrorMessage.IOHelper_CouldNotOpenOrCreateAttribute); } } finally { if (H5I.is_valid(dataspaceId) > 0) { H5S.close(dataspaceId); } } return attributeId; })); }
public static void UpdateCampaignInfoSet() { long vdsFileId = -1; long vdsMetaFileId = -1; long groupId = -1; lock (_lock) { try { if (File.Exists(_options.VdsFilePath)) { vdsFileId = H5F.open(_options.VdsFilePath, H5F.ACC_RDONLY); vdsMetaFileId = H5F.open(_options.VdsMetaFilePath, H5F.ACC_RDONLY); Program.CampaignInfoSet = GeneralHelper.GetCampaignInfoSet(vdsFileId, false); } else { Program.CampaignInfoSet = new List <CampaignInfo>(); } Program.CampaignDescriptionSet = Program.CampaignInfoSet.ToDictionary(campaignInfo => campaignInfo.Name, campaignInfo => { if (IOHelper.CheckLinkExists(vdsMetaFileId, campaignInfo.Name)) { try { groupId = H5G.open(vdsMetaFileId, campaignInfo.Name); if (H5A.exists(groupId, "description") > 0) { return(IOHelper.ReadAttribute <string>(groupId, "description").First()); } } finally { if (H5I.is_valid(groupId) > 0) { H5G.close(groupId); } } } return("no description available"); }); } finally { if (H5I.is_valid(vdsFileId) > 0) { H5F.close(vdsFileId); } if (H5I.is_valid(vdsMetaFileId) > 0) { H5F.close(vdsMetaFileId); } } } }
/// <summary> /// Reads the data from the specified attribute. /// </summary> /// <typeparam name="T">The data type.</typeparam> /// <param name="locationId">The location ID.</param> /// <param name="attributeName">The name of the attribute.</param> /// <returns>Returns a set of type T which represents the read data.</returns> public static T[] ReadAttribute <T>(long locationId, string attributeName) { long attributeId = -1; T[] result; try { attributeId = H5A.open(locationId, attributeName); if (H5I.is_valid(attributeId) <= 0) { throw new Exception(ErrorMessage.IOHelper_CouldNotOpenAttribute); } result = IOHelper.Read <T>(attributeId, DataContainerType.Attribute); } finally { if (H5I.is_valid(attributeId) > 0) { H5A.close(attributeId); } } return(result); }
protected override void CloseId(long id) { if (H5I.is_valid(id) > 0) { H5D.close(id); } }
public static unsafe void AddDataWithSharedDataType(long fileId, ContainerType container) { long typeId = H5T.copy(H5T.C_S1); H5T.set_size(typeId, H5T.VARIABLE); H5T.set_cset(typeId, H5T.cset_t.UTF8); H5T.commit(fileId, "string_t", typeId); var data = new string[] { "001", "11", "22", "33", "44", "55", "66", "77", " ", "AA", "ZZ", "!!" }; var dataChar = data .SelectMany(value => Encoding.ASCII.GetBytes(value + '\0')) .ToArray(); fixed(byte *dataVarPtr = dataChar) { var basePtr = new IntPtr(dataVarPtr); var addresses = new IntPtr[] { IntPtr.Add(basePtr, 0), IntPtr.Add(basePtr, 4), IntPtr.Add(basePtr, 7), IntPtr.Add(basePtr, 10), IntPtr.Add(basePtr, 13), IntPtr.Add(basePtr, 16), IntPtr.Add(basePtr, 19), IntPtr.Add(basePtr, 22), IntPtr.Add(basePtr, 25), IntPtr.Add(basePtr, 28), IntPtr.Add(basePtr, 31), IntPtr.Add(basePtr, 34) }; fixed(void *dataVarAddressesPtr = addresses) { TestUtils.Add(container, fileId, "shared_data_type", "shared_data_type", typeId, dataVarAddressesPtr, length: 12); } } if (H5I.is_valid(typeId) > 0) { H5T.close(typeId); } }
protected override void FreeUnmanagedResources() { base.FreeUnmanagedResources(); if (H5I.is_valid(_fileId) > 0) { this.CloseHdfFile(_fileId); } }
private void Menu_2() { long vdsFileId = -1; long vdsMetaFileId = -1; long fcPropertyId = -1; string vdsFilePath; string vdsMetaFilePath; List <CampaignInfo> campaignInfoSet; IList <HdfElementBase> currentList; // vdsFilePath = Path.Combine(Program.BaseDirectoryPath, "VDS.h5"); vdsMetaFilePath = Path.Combine(Program.BaseDirectoryPath, "VDS_META.h5"); try { if (File.Exists(vdsFilePath)) { vdsFileId = H5F.open(vdsFilePath, H5F.ACC_RDONLY); } else { return; } if (File.Exists(vdsMetaFilePath)) { vdsMetaFileId = H5F.open(vdsMetaFilePath, H5F.ACC_RDWR); } if (vdsMetaFileId == -1) { fcPropertyId = H5P.create(H5P.FILE_CREATE); H5P.set_file_space(fcPropertyId, H5F.file_space_type_t.ALL_PERSIST); vdsMetaFileId = H5F.create(vdsMetaFilePath, H5F.ACC_TRUNC, fcPropertyId); } campaignInfoSet = GeneralHelper.GetCampaignInfoSet(vdsFileId, true); currentList = campaignInfoSet.Cast <HdfElementBase>().ToList(); new VdsMetaNavigator(vdsFileId, vdsMetaFileId, "/", currentList); } finally { if (H5I.is_valid(vdsFileId) > 0) { H5F.close(vdsFileId); } if (H5I.is_valid(vdsMetaFileId) > 0) { H5F.close(vdsMetaFileId); } } }
private void UpdateVariableInfo(FileContext fileContext, long variableGroupId) { ulong idx; idx = 0; _variableGroupSet = IOHelper.UpdateAttributeList(variableGroupId, "group_set", _variableGroupSet.ToArray()).ToList(); if (fileContext.FormatVersion != 1) { _unitSet = IOHelper.UpdateAttributeList(variableGroupId, "unit_set", _unitSet.ToArray()).ToList(); _transferFunctionSet = IOHelper.UpdateAttributeList(variableGroupId, "transfer_function_set", _transferFunctionSet.ToArray()).ToList(); } H5L.iterate(variableGroupId, H5.index_t.NAME, H5.iter_order_t.INC, ref idx, Callback, IntPtr.Zero); int Callback(long variableGroupId2, IntPtr intPtrName, ref H5L.info_t info, IntPtr userDataPtr) { long datasetId = -1; long typeId_do_not_close = -1; string name; DatasetInfo currentDatasetInfo; try { name = Marshal.PtrToStringAnsi(intPtrName); if (H5L.exists(variableGroupId2, name) > 0) { datasetId = H5D.open(variableGroupId2, name); currentDatasetInfo = _datasetInfoSet.FirstOrDefault(datasetInfo => datasetInfo.Name == name); if (currentDatasetInfo == null) { typeId_do_not_close = H5D.get_type(datasetId); currentDatasetInfo = new DatasetInfo(name, typeId_do_not_close, this, this.IsLazyLoading); _datasetInfoSet.Add(currentDatasetInfo); } currentDatasetInfo.Update(fileContext); } } finally { if (H5I.is_valid(datasetId) > 0) { H5D.close(datasetId); } } return(0); } }
public static (long DatasetId, bool IsNew) OpenOrCreateDataset(long locationId, string datasetPath, long datasetTypeId, ulong chunkLength, ulong chunkCount, IntPtr fillValue = default) { return(IOHelper.OpenOrCreateDataset(locationId, datasetPath, datasetTypeId, () => { long dcPropertyId = -1; long lcPropertyId = -1; long dataspaceId = -1; long datasetId = -1; try { dcPropertyId = H5P.create(H5P.DATASET_CREATE); if (fillValue != IntPtr.Zero) { H5P.set_fill_value(dcPropertyId, datasetTypeId, fillValue); } H5P.set_shuffle(dcPropertyId); H5P.set_deflate(dcPropertyId, 7); H5P.set_chunk(dcPropertyId, 1, new ulong[] { chunkLength }); lcPropertyId = H5P.create(H5P.LINK_CREATE); H5P.set_create_intermediate_group(lcPropertyId, 1); dataspaceId = H5S.create_simple(1, new ulong[] { chunkLength *chunkCount }, null); datasetId = H5D.create(locationId, datasetPath, datasetTypeId, dataspaceId, lcPropertyId, dcPropertyId); if (H5I.is_valid(datasetId) <= 0) { throw new Exception($"{ ErrorMessage.IOHelper_CouldNotOpenOrCreateDataset } Dataset: '{ datasetPath }'."); } } finally { if (H5I.is_valid(dcPropertyId) > 0) { H5P.close(dcPropertyId); } if (H5I.is_valid(lcPropertyId) > 0) { H5P.close(lcPropertyId); } if (H5I.is_valid(dataspaceId) > 0) { H5S.close(dataspaceId); } } return datasetId; })); }
private ISimpleDataStorage LoadDataset(long sourceFileId, string datasetPath, ulong start, ulong stride, ulong block, ulong count) { long datasetId = -1; long typeId = -1; Array dataset; Array dataset_status; Type genericType; ExtendedDataStorageBase extendedDataStorage; ISimpleDataStorage simpleDataStorage; dataset = IOHelper.ReadDataset(sourceFileId, datasetPath, start, stride, block, count); // apply status (only if native dataset) if (H5L.exists(sourceFileId, datasetPath + "_status") > 0) { try { datasetId = H5D.open(sourceFileId, datasetPath); typeId = H5D.get_type(datasetId); dataset_status = IOHelper.ReadDataset(sourceFileId, datasetPath + "_status", start, stride, block, count).Cast <byte>().ToArray(); genericType = typeof(ExtendedDataStorage <>).MakeGenericType(TypeConversionHelper.GetTypeFromHdfTypeId(typeId)); extendedDataStorage = (ExtendedDataStorageBase)Activator.CreateInstance(genericType, dataset, dataset_status); dataset_status = null; } finally { if (H5I.is_valid(datasetId) > 0) { H5D.close(datasetId); } if (H5I.is_valid(typeId) > 0) { H5T.close(typeId); } } simpleDataStorage = extendedDataStorage.ToSimpleDataStorage(); extendedDataStorage.Dispose(); return(simpleDataStorage); } else { return(new SimpleDataStorage(dataset.Cast <double>().ToArray())); } }
private void UpdateCampaignInfo(FileContext fileContext, long campaignGroupId) { ulong idx; idx = 0; _chunkDatasetInfo.Update(fileContext); GeneralHelper.SuppressErrors(() => H5L.iterate(campaignGroupId, H5.index_t.NAME, H5.iter_order_t.INC, ref idx, Callback, IntPtr.Zero)); int Callback(long campaignGroupId2, IntPtr intPtrName, ref H5L.info_t info, IntPtr userDataPtr) { long groupId = -1; string name; VariableInfo currentVariableInfo; try { name = Marshal.PtrToStringAnsi(intPtrName); // this is necessary, since H5Oget_info_by_name is slow because it wants verbose object header data // and H5G_loc_info is not directly accessible // only chance is to modify source code (H5Oget_info_by_name) groupId = H5G.open(campaignGroupId2, name); if (groupId > -1) { currentVariableInfo = _variableInfoSet.FirstOrDefault(variableInfo => variableInfo.Name == name); if (currentVariableInfo == null) { currentVariableInfo = new VariableInfo(name, this, this.IsLazyLoading); _variableInfoSet.Add(currentVariableInfo); } currentVariableInfo.Update(fileContext); } } finally { if (H5I.is_valid(groupId) > 0) { H5G.close(groupId); } } return(0); } }
public void H5Iis_validTest1() { hid_t gid = H5G.create(m_v0_test_file, "A"); Assert.IsTrue(gid > 0); Assert.IsTrue(H5I.is_valid(gid) > 0); Assert.IsTrue(H5G.close(gid) >= 0); Assert.IsTrue(H5I.is_valid(gid) == 0); gid = H5G.create(m_v2_test_file, "A"); Assert.IsTrue(gid > 0); Assert.IsTrue(H5I.is_valid(gid) > 0); Assert.IsTrue(H5G.close(gid) >= 0); Assert.IsTrue(H5I.is_valid(gid) == 0); }
public static ulong[] PrepareAttributeValueSet <T>(long attributeId, ref T[] valueSet, bool isReference) { long dataspaceId = -1; ulong[] dimensionSet; ulong[] dimensionLimitSet; dimensionSet = new ulong[] { 0 }; dimensionLimitSet = new ulong[] { 0 }; try { dataspaceId = H5A.get_space(attributeId); H5S.get_simple_extent_dims(dataspaceId, null, dimensionLimitSet); // merge data if (dimensionLimitSet[0] == H5S.UNLIMITED) { T[] valueSet_File = IOHelper.Read <T>(attributeId, DataContainerType.Attribute); if (isReference) { if (valueSet_File.Count() == 0 || !Enumerable.SequenceEqual(valueSet_File, valueSet.Skip(Math.Max(0, valueSet.Count() - valueSet_File.Count())))) { valueSet = valueSet.Concat(valueSet_File).ToArray(); } } else { if (valueSet.Count() == 0 || !Enumerable.SequenceEqual(valueSet, valueSet_File.Skip(Math.Max(0, valueSet_File.Count() - valueSet.Count())))) { valueSet = valueSet_File.Concat(valueSet).ToArray(); } } } } finally { if (H5I.is_valid(dataspaceId) > 0) { H5S.close(dataspaceId); } } return(dimensionLimitSet); }
public static T[] ReadDataset <T>(long locationId, string datasetPath, ulong start = 0, ulong stride = 0, ulong block = 0, ulong count = 0) { long datasetId = -1; long dataspaceId = -1; T[] result; try { datasetId = H5D.open(locationId, datasetPath); if (H5I.is_valid(datasetId) <= 0) { throw new Exception(ErrorMessage.IOHelper_CouldNotOpenDataset); } if (start == 0 && stride == 0 && block == 0 && count == 0) { result = IOHelper.Read <T>(datasetId, DataContainerType.Dataset); } else { dataspaceId = H5D.get_space(datasetId); if (H5S.select_hyperslab(dataspaceId, H5S.seloper_t.SET, new ulong[] { start }, new ulong[] { stride }, new ulong[] { block }, new ulong[] { count }) < 0) { throw new Exception(ErrorMessage.IOHelper_CouldNotSelectHyperslab); } result = IOHelper.Read <T>(datasetId, DataContainerType.Dataset, dataspaceId); } } finally { if (H5I.is_valid(dataspaceId) > 0) { H5S.close(dataspaceId); } if (H5I.is_valid(datasetId) > 0) { H5D.close(datasetId); } } return(result); }
public static (long DatasetId, bool IsNew) OpenOrCreateDataset(long locationId, string datasetPath, long datasetTypeId, Func <long> createDatasetCallback) { Contract.Requires(createDatasetCallback != null); long datasetId = -1; long datasetTypeId_actual = -1; bool isNew; try { if (IOHelper.CheckLinkExists(locationId, datasetPath)) { datasetId = H5D.open(locationId, datasetPath); datasetTypeId_actual = H5D.get_type(datasetId); if (H5T.equal(datasetTypeId_actual, datasetTypeId) <= 0) { throw new Exception($"{ ErrorMessage.IOHelper_DataTypeMismatch } Dataset: '{ datasetPath }'."); } isNew = false; } else { datasetId = createDatasetCallback.Invoke(); isNew = true; } if (H5I.is_valid(datasetId) <= 0) { throw new Exception($"{ ErrorMessage.IOHelper_CouldNotOpenOrCreateDataset } Dataset: '{ datasetPath }'."); } } finally { if (H5I.is_valid(datasetTypeId_actual) > 0) { H5T.close(datasetTypeId_actual); } } return(datasetId, isNew); }
/// <summary> /// Writes data to the specified attribute. /// </summary> /// <typeparam name="T">The data type.</typeparam> /// <param name="locationId">The location ID.</param> /// <param name="attributeName">The name of the attribute.</param> /// <param name="valueSet">The data to be written.</param> public static void WriteAttribute <T>(long locationId, string attributeName, T[] valueSet) { Contract.Requires(valueSet != null, nameof(valueSet)); long attributeId = -1; try { attributeId = H5A.open(locationId, attributeName); IOHelper.Write <T>(attributeId, valueSet, DataContainerType.Attribute); } finally { if (H5I.is_valid(attributeId) > 0) { H5A.close(attributeId); } } }
public static void WriteDataset <T>(long locationId, string datasetName, T[] valueSet) { Contract.Requires(valueSet != null, nameof(valueSet)); long datasetId = -1; try { datasetId = H5D.open(locationId, datasetName); IOHelper.Write <T>(datasetId, valueSet, DataContainerType.Dataset); } finally { if (H5I.is_valid(datasetId) > 0) { H5D.close(datasetId); } } }
public static (long AttributeId, bool IsNew) OpenOrCreateAttribute(long locationId, string name, long attributeTypeId, Func <long> createAttributeCallback) { long attributeId = -1; long attributeTypeId_actual = -1; bool isNew; try { if (H5A.exists(locationId, name) > 0) { attributeId = H5A.open(locationId, name); attributeTypeId_actual = H5A.get_type(attributeId); if (H5T.equal(attributeTypeId_actual, attributeTypeId) <= 0) { throw new Exception($"{ ErrorMessage.IOHelper_DataTypeMismatch } Attribute: '{ name }'."); } isNew = false; } else { attributeId = createAttributeCallback.Invoke(); isNew = true; } if (H5I.is_valid(attributeId) <= 0) { throw new Exception($"{ ErrorMessage.IOHelper_CouldNotOpenOrCreateAttribute } Attribute: '{ name }'."); } } finally { if (H5I.is_valid(attributeTypeId_actual) > 0) { H5T.close(attributeTypeId_actual); } } return(attributeId, isNew); }
public static T[] UpdateAttributeList <T>(long locationId, string attributeName, T[] referenceValueSet) { long attributeId = -1; try { attributeId = H5A.open(locationId, attributeName); IOHelper.PrepareAttributeValueSet(attributeId, ref referenceValueSet, true); } finally { if (H5I.is_valid(attributeId) > 0) { H5A.close(attributeId); } } return(referenceValueSet); }
protected override void OnWrite(ChannelContextGroup contextGroup, ulong fileOffset, ulong bufferOffset, ulong length) { long groupId = -1; try { var firstChunk = (long)this.ToChunkIndex(fileOffset, contextGroup.SampleRate); var lastChunk = (long)this.ToChunkIndex(fileOffset + length, contextGroup.SampleRate) - 1; groupId = H5G.open(_fileId, $"/info"); _lastCompletedChunk = IOHelper.ReadDataset <double>(groupId, "last_completed_chunk").FirstOrDefault(); // this does not work in conjunction with multiple context groups if (firstChunk <= _lastCompletedChunk) { // throw new Exception(ErrorMessage.Mat73Writer_ChunkAlreadyWritten); } // write data for (int i = 0; i < contextGroup.ChannelContextSet.Count(); i++) { this.WriteData(fileOffset, bufferOffset, length, contextGroup.ChannelContextSet[i]); } // write last_completed_chunk IOHelper.WriteDataset(groupId, "last_completed_chunk", new double[] { lastChunk }); } finally { if (H5I.is_valid(groupId) > 0) { H5G.close(groupId); } H5F.flush(_fileId, H5F.scope_t.GLOBAL); } }
public static object ReadAttribute(long locationId, string attributeName) { long attributeId = -1; long typeId = -1; object result; try { attributeId = H5A.open(locationId, attributeName); if (H5I.is_valid(attributeId) <= 0) { throw new Exception(ErrorMessage.IOHelper_CouldNotOpenAttribute); } typeId = H5A.get_type(attributeId); // invoke HdfHelper.Read result = GeneralHelper.InvokeGenericMethod(typeof(IOHelper), null, nameof(IOHelper.Read), BindingFlags.Public | BindingFlags.Static, TypeConversionHelper.GetTypeFromHdfTypeId(typeId), new object[] { attributeId, DataContainerType.Attribute, Type.Missing }); } finally { if (H5I.is_valid(typeId) > 0) { H5T.close(typeId); } if (H5I.is_valid(attributeId) > 0) { H5A.close(attributeId); } } return(result); }
private void UpdateDatasetInfo(FileContext fileContext, long datasetId) { long dataspaceId = -1; ulong[] actualDimenionSet = new ulong[1]; ulong[] maximumDimensionSet = new ulong[1]; try { dataspaceId = H5D.get_space(datasetId); H5S.get_simple_extent_dims(dataspaceId, actualDimenionSet, maximumDimensionSet); _sourceFileInfoSet.Add(new SourceFileInfo(fileContext.FilePath, actualDimenionSet.First(), fileContext.DateTime)); } finally { if (H5I.is_valid(dataspaceId) > 0) { H5S.close(dataspaceId); } } }
public static (long GroupId, bool IsNew) OpenOrCreateGroup(long locationId, string groupPath) { long groupId = -1; long propertyId = -1; bool isNew; try { if (IOHelper.CheckLinkExists(locationId, groupPath)) { groupId = H5G.open(locationId, groupPath); isNew = false; } else { propertyId = H5P.create(H5P.LINK_CREATE); H5P.set_create_intermediate_group(propertyId, 1); groupId = H5G.create(locationId, groupPath, propertyId); isNew = true; } if (H5I.is_valid(groupId) <= 0) { throw new Exception($"{ ErrorMessage.IOHelper_CouldNotOpenOrCreateGroup } Group name: '{ groupPath }'."); } } finally { if (H5I.is_valid(propertyId) > 0) { H5P.close(propertyId); } } return(groupId, isNew); }
protected override bool OnWaitForUserInput() { if (this.SelectedIndex < 0) { this.SelectedIndex = _vdsMetaAggregateFunctionSet.Count() > 0 ? 0 : -1; } else if (this.SelectedIndex >= _vdsMetaAggregateFunctionSet.Count()) { this.SelectedIndex = _vdsMetaAggregateFunctionSet.Count() - 1; } if (this.LastIndex >= 0) { Console.SetCursorPosition(1, this.LastIndex); Console.Write(" "); } if (this.SelectedIndex >= 0) { Console.SetCursorPosition(1, this.SelectedIndex + 2); Console.Write("x"); } _consoleKeyInfo = Console.ReadKey(true); switch (_consoleKeyInfo.Key) { case ConsoleKey.UpArrow: if (this.SelectedIndex > 0) { this.LastIndex = Console.CursorTop; this.SelectedIndex -= 1; } break; case ConsoleKey.DownArrow: if (this.SelectedIndex < _vdsMetaAggregateFunctionSet.Count() - 1) { this.LastIndex = Console.CursorTop; this.SelectedIndex += 1; } break; case ConsoleKey.Enter: case ConsoleKey.RightArrow: if (this.SelectedIndex >= 0) { _vdsMetaAggregateFunctionSet[this.SelectedIndex] = Program.PromptAggregateFunctionData(_vdsMetaAggregateFunctionSet[this.SelectedIndex]); _updateAttribute = true; } break; case ConsoleKey.N: _vdsMetaAggregateFunctionSet.Add(Program.PromptAggregateFunctionData(new hdf_aggregate_function_t())); this.SelectedIndex = _vdsMetaAggregateFunctionSet.Count(); _updateAttribute = true; break; case ConsoleKey.C: if (this.SelectedIndex >= 0) { _vdsMetaAggregateFunctionSet.Add(_vdsMetaAggregateFunctionSet[this.SelectedIndex]); this.SelectedIndex = _vdsMetaAggregateFunctionSet.Count(); this.OnRedraw(); } break; case ConsoleKey.D: _vdsMetaAggregateFunctionSet.Add(new hdf_aggregate_function_t("mean", "none")); _vdsMetaAggregateFunctionSet.Add(new hdf_aggregate_function_t("min", "none")); _vdsMetaAggregateFunctionSet.Add(new hdf_aggregate_function_t("max", "none")); _vdsMetaAggregateFunctionSet.Add(new hdf_aggregate_function_t("std", "none")); _updateAttribute = true; this.OnRedraw(); break; case ConsoleKey.Delete: if (this.SelectedIndex >= 0) { Console.Clear(); Console.Write("The selected item will be deleted. Proceed (Y/N)? "); if (Console.ReadKey().Key == ConsoleKey.Y) { _vdsMetaAggregateFunctionSet.RemoveAt(this.SelectedIndex); _updateAttribute = true; } this.OnRedraw(); } break; case ConsoleKey.PageUp: if (this.SelectedIndex >= 1 && this.SelectedIndex < _vdsMetaAggregateFunctionSet.Count()) { hdf_aggregate_function_t hdf_aggregate_function = _vdsMetaAggregateFunctionSet[this.SelectedIndex]; _vdsMetaAggregateFunctionSet.RemoveAt(this.SelectedIndex); _vdsMetaAggregateFunctionSet.Insert(this.SelectedIndex - 1, hdf_aggregate_function); this.SelectedIndex -= 1; _updateAttribute = true; } break; case ConsoleKey.PageDown: if (this.SelectedIndex >= 0 && this.SelectedIndex < _vdsMetaAggregateFunctionSet.Count() - 1) { hdf_aggregate_function_t hdf_aggregate_function = _vdsMetaAggregateFunctionSet[this.SelectedIndex]; _vdsMetaAggregateFunctionSet.RemoveAt(this.SelectedIndex); _vdsMetaAggregateFunctionSet.Insert(this.SelectedIndex + 1, hdf_aggregate_function); this.SelectedIndex += 1; _updateAttribute = true; } break; case ConsoleKey.Escape: case ConsoleKey.LeftArrow: return(true); } // update attribute if (_updateAttribute) { _groupId = IOHelper.OpenOrCreateGroup(_vdsMetaFileId, _currentPath).GroupId; IOHelper.PrepareAttribute(_groupId, "aggregate_function_set", _vdsMetaAggregateFunctionSet.ToArray(), new ulong[] { (ulong)_vdsMetaAggregateFunctionSet.Count() }, true); _updateAttribute = false; this.OnRedraw(); } // clean up if (H5I.is_valid(_groupId) > 0) { H5F.flush(_groupId, H5F.scope_t.GLOBAL); H5G.close(_groupId); } return(false); }
public static Type GetTypeFromHdfTypeId(long typeId) { long typeId_reference = -1; Type type = null; type = null; if (H5T.equal(typeId, H5T.NATIVE_UINT8) > 0) { type = typeof(Byte); } else if (H5T.equal(typeId, H5T.NATIVE_INT8) > 0) { type = typeof(SByte); } else if (H5T.equal(typeId, H5T.NATIVE_UINT16) > 0) { type = typeof(UInt16); } else if (H5T.equal(typeId, H5T.NATIVE_INT16) > 0) { type = typeof(Int16); } else if (H5T.equal(typeId, H5T.NATIVE_UINT32) > 0) { type = typeof(UInt32); } else if (H5T.equal(typeId, H5T.NATIVE_INT32) > 0) { type = typeof(Int32); } else if (H5T.equal(typeId, H5T.NATIVE_UINT64) > 0) { type = typeof(UInt64); } else if (H5T.equal(typeId, H5T.NATIVE_INT64) > 0) { type = typeof(Int64); } else if (H5T.equal(typeId, H5T.NATIVE_FLOAT) > 0) { type = typeof(Single); } else if (H5T.equal(typeId, H5T.NATIVE_DOUBLE) > 0) { type = typeof(Double); } else if (H5T.equal(typeId, typeId_reference = TypeConversionHelper.GetHdfTypeIdFromType(typeof(string))) > 0) { type = typeof(string); } else if (H5T.equal(typeId, typeId_reference = TypeConversionHelper.GetHdfTypeIdFromType(typeof(hdf_aggregate_function_t))) > 0) { type = typeof(hdf_aggregate_function_t); } else if (H5T.equal(typeId, typeId_reference = TypeConversionHelper.GetHdfTypeIdFromType(typeof(hdf_tag_t))) > 0) { type = typeof(hdf_tag_t); } else if (H5T.equal(typeId, typeId_reference = TypeConversionHelper.GetHdfTypeIdFromType(typeof(hdf_transfer_function_t))) > 0) { type = typeof(hdf_transfer_function_t); } if (H5I.is_valid(typeId_reference) > 0) { H5T.close(typeId_reference); } if (type == null) { throw new NotSupportedException(); } return(type); }
public static long GetHdfTypeIdFromType(long fileId, Type type) { Type elementType; elementType = type.IsArray ? type.GetElementType() : type; if (elementType == typeof(bool)) { return(H5T.NATIVE_UINT8); } else if (elementType == typeof(Byte)) { return(H5T.NATIVE_UINT8); } else if (elementType == typeof(SByte)) { return(H5T.NATIVE_INT8); } else if (elementType == typeof(UInt16)) { return(H5T.NATIVE_UINT16); } else if (elementType == typeof(Int16)) { return(H5T.NATIVE_INT16); } else if (elementType == typeof(UInt32)) { return(H5T.NATIVE_UINT32); } else if (elementType == typeof(Int32)) { return(H5T.NATIVE_INT32); } else if (elementType == typeof(UInt64)) { return(H5T.NATIVE_UINT64); } else if (elementType == typeof(Int64)) { return(H5T.NATIVE_INT64); } else if (elementType == typeof(Single)) { return(H5T.NATIVE_FLOAT); } else if (elementType == typeof(Double)) { return(H5T.NATIVE_DOUBLE); } else if (elementType == typeof(string) || elementType == typeof(IntPtr)) { long typeId = 0; if (H5I.is_valid(fileId) > 0 && H5L.exists(fileId, "string_t") > 0) { typeId = H5T.open(fileId, "string_t"); } else { typeId = H5T.copy(H5T.C_S1); H5T.set_size(typeId, H5T.VARIABLE); H5T.set_cset(typeId, H5T.cset_t.UTF8); if (fileId > -1 && H5T.commit(fileId, "string_t", typeId) < 0) { throw new Exception(ErrorMessage.TypeConversionHelper_CouldNotCommitDataType); } } return(typeId); } else if (elementType.IsValueType && !elementType.IsPrimitive && !elementType.IsEnum) { long typeId = 0; if (H5I.is_valid(fileId) > 0 && H5L.exists(fileId, elementType.Name) > 0) { typeId = H5T.open(fileId, elementType.Name); } else { typeId = H5T.create(H5T.class_t.COMPOUND, new IntPtr(Marshal.SizeOf(elementType))); foreach (FieldInfo fieldInfo in elementType.GetFields()) { long fieldType = -1; fieldType = TypeConversionHelper.GetHdfTypeIdFromType(fileId, fieldInfo.FieldType); H5T.insert(typeId, fieldInfo.Name, Marshal.OffsetOf(elementType, fieldInfo.Name), fieldType); if (H5I.is_valid(fieldType) > 0) { H5T.close(fieldType); } } if (fileId > -1 && H5T.commit(fileId, elementType.Name, typeId) < 0) { throw new Exception(ErrorMessage.TypeConversionHelper_CouldNotCommitDataType); } } return(typeId); } else { throw new NotSupportedException(); } }
public Task <string> GetCampaignDocumentation(string campaigInfoName) { long vdsMetaFileId = -1; string csvFileName; CampaignInfo campaignInfo; List <string> groupNameSet; campaignInfo = Program.CampaignInfoSet.First(campaign => campaign.Name == campaigInfoName); return(Task.Run(() => { this.CheckState(); try { if (File.Exists(_options.VdsMetaFilePath)) { vdsMetaFileId = H5F.open(_options.VdsMetaFilePath, H5F.ACC_RDONLY); } csvFileName = Path.Combine(_options.SupportDirectoryPath, "EXPORT", $"OneDAS_{ campaignInfo.Name.ToLower().Replace("/", "_").TrimStart('_') }_{ Guid.NewGuid().ToString() }.csv"); groupNameSet = campaignInfo.VariableInfoSet.SelectMany(variableInfo => variableInfo.VariableGroupSet.Last().Split('\n')).Distinct().ToList(); using (StreamWriter streamWriter = new StreamWriter(new FileStream(csvFileName, FileMode.Create, FileAccess.Write, FileShare.Read), Encoding.UTF8)) { // campaign header streamWriter.WriteLine($"# { campaignInfo.Name.TrimStart('/').Replace("/", " / ") }"); // campaign description streamWriter.WriteLine($"# { Program.CampaignDescriptionSet[campaigInfoName] }"); // header streamWriter.WriteLine("Group;Name;Unit;Transfer function;Aggregate function;Guid"); // groups foreach (string groupName in groupNameSet) { List <VariableInfo> groupedVariableInfoSet; groupedVariableInfoSet = campaignInfo.VariableInfoSet.Where(variableInfo => variableInfo.VariableGroupSet.Last().Split('\n').Contains(groupName)).OrderBy(variableInfo => variableInfo.VariableNameSet.Last()).ToList(); // variables groupedVariableInfoSet.ForEach(variableInfo => { long variable_groupId = -1; string transferFunction; string aggregateFunction; string groupPath; string name; string guid; string unit; List <hdf_transfer_function_t> transferFunctionSet; List <hdf_aggregate_function_t> aggregateFunctionSet; name = variableInfo.VariableNameSet.Last(); guid = variableInfo.Name; unit = string.Empty; transferFunctionSet = new List <hdf_transfer_function_t>(); aggregateFunctionSet = new List <hdf_aggregate_function_t>(); try { groupPath = GeneralHelper.CombinePath(campaignInfo.Name, variableInfo.Name); if (H5I.is_valid(vdsMetaFileId) > 0 && IOHelper.CheckLinkExists(vdsMetaFileId, groupPath)) { variable_groupId = H5G.open(vdsMetaFileId, groupPath); if (H5A.exists(variable_groupId, "unit") > 0) { unit = IOHelper.ReadAttribute <string>(variable_groupId, "unit").FirstOrDefault(); } if (H5A.exists(variable_groupId, "transfer_function_set") > 0) { transferFunctionSet = IOHelper.ReadAttribute <hdf_transfer_function_t>(variable_groupId, "transfer_function_set").ToList(); } if (H5A.exists(variable_groupId, "aggregate_function_set") > 0) { aggregateFunctionSet = IOHelper.ReadAttribute <hdf_aggregate_function_t>(variable_groupId, "aggregate_function_set").ToList(); } } } finally { if (H5I.is_valid(variable_groupId) > 0) { H5G.close(variable_groupId); } } // transfer function transferFunction = string.Empty; transferFunctionSet.ForEach(tf => { if (!string.IsNullOrWhiteSpace(transferFunction)) { transferFunction += " | "; } transferFunction += $"{ tf.date_time }, { tf.type }, { tf.option }, { tf.argument }"; }); transferFunction = $"\"{ transferFunction }\""; // aggregate function aggregateFunction = string.Empty; aggregateFunctionSet.ForEach(af => { if (!string.IsNullOrWhiteSpace(aggregateFunction)) { aggregateFunction += " | "; } aggregateFunction += $"{ af.type }, { af.argument }"; }); aggregateFunction = $"\"{ aggregateFunction }\""; // write row streamWriter.WriteLine($"{ groupName };{ name };{ unit };{ transferFunction };{ aggregateFunction };{ guid }"); }); } } return $"download/{ Path.GetFileName(csvFileName) }"; } finally { if (H5I.is_valid(vdsMetaFileId) > 0) { H5F.close(vdsMetaFileId); } } })); }
public Task <string> GetData(DateTime dateTimeBegin, DateTime dateTimeEnd, string sampleRateDescription, FileFormat fileFormat, FileGranularity fileGranularity, Dictionary <string, Dictionary <string, List <string> > > campaignInfoSet) { long fileId = -1; long datasetId = -1; ulong start; ulong stride; ulong block; ulong count; ulong segmentLength; ulong segmentSize; ulong bytesPerRow; double sampleRate; DateTime epochStart; DateTime epochEnd; string zipFilePath; // task return(Task.Run(() => { this.CheckState(); if (!campaignInfoSet.Any()) { return string.Empty; } // zip file zipFilePath = Path.Combine(_options.SupportDirectoryPath, "EXPORT", $"OneDAS_{ dateTimeBegin.ToString("yyyy-MM-ddTHH-mm") }_{ sampleRateDescription }_{ Guid.NewGuid().ToString() }.zip"); // sampleRate sampleRate = sampleRateDescription.ToSampleRate(); // epoch & hyperslab epochStart = new DateTime(2000, 01, 01); epochEnd = new DateTime(2030, 01, 01); if (!(epochStart <= dateTimeBegin && dateTimeBegin <= dateTimeEnd && dateTimeEnd <= epochEnd)) { throw new Exception("requirement >> epochStart <= dateTimeBegin && dateTimeBegin <= dateTimeEnd && dateTimeBegin <= epochEnd << is not matched"); } start = (ulong)(Math.Floor((dateTimeBegin - epochStart).TotalSeconds * sampleRate)); stride = 1; block = (ulong)(Math.Ceiling((dateTimeEnd - dateTimeBegin).TotalSeconds * sampleRate)); count = 1; try { // open file fileId = H5F.open(_options.VdsFilePath, H5F.ACC_RDONLY); // byte count bytesPerRow = 0; foreach (var campaignInfo in campaignInfoSet) { foreach (var variableInfo in campaignInfo.Value) { foreach (string datasetInfo in variableInfo.Value) { try { datasetId = H5D.open(fileId, $"{ campaignInfo.Key }/{ variableInfo.Key }/{ datasetInfo }"); bytesPerRow += (ulong)OneDasUtilities.SizeOf(TypeConversionHelper.GetTypeFromHdfTypeId(H5D.get_type(datasetId))); } finally { if (H5I.is_valid(datasetId) > 0) { H5D.close(datasetId); } } } } } this.GetClient().SendByteCount(bytesPerRow * block); segmentSize = (50 * 1024 * 1024) / bytesPerRow * bytesPerRow; segmentLength = segmentSize / bytesPerRow; // ensure that dataset length is multiple of 1 minute if ((segmentLength / sampleRate) % 60 != 0) { segmentLength = (ulong)((ulong)(segmentLength / sampleRate / 60) * 60 * sampleRate); } // start _stateManager.SetState(this.Context.ConnectionId, HdfExplorerState.Loading); using (ZipArchive zipArchive = ZipFile.Open(zipFilePath, ZipArchiveMode.Create)) { foreach (var campaignInfo in campaignInfoSet) { HdfDataLoader hdfDataLoader; hdfDataLoader = new HdfDataLoader(_stateManager.GetToken(this.Context.ConnectionId)); hdfDataLoader.ProgressUpdated += this.OnProgressUpdated; if (!hdfDataLoader.WriteZipFileCampaignEntry(zipArchive, fileGranularity, fileFormat, new ZipSettings(dateTimeBegin, campaignInfo, fileId, sampleRate, start, stride, block, count, segmentLength))) { return string.Empty; } } } } catch (Exception ex) { this.WriteLogEntry(ex.Message, true); throw; } finally { _stateManager.SetState(this.Context.ConnectionId, HdfExplorerState.Idle); if (H5I.is_valid(fileId) > 0) { H5F.close(fileId); } } this.WriteLogEntry($"{ this.Context.GetHttpContext().Connection.RemoteIpAddress } requested data: { dateTimeBegin.ToString("yyyy-MM-dd HH:mm:ss") } to { dateTimeEnd.ToString("yyyy-MM-dd HH:mm:ss") }", false); return $"download/{ Path.GetFileName(zipFilePath) }"; }, _stateManager.GetToken(this.Context.ConnectionId))); }
public void WriteCampaignDocumentation(string directoryPath, CampaignInfo campaignInfo) { long campaign_groupId = -1; string description; string filePath; RestructuredTextWriter restructuredTextWriter; List <string> groupNameSet; Console.Clear(); groupNameSet = campaignInfo.VariableInfoSet.Select(variableInfo => variableInfo.VariableGroupSet.Last()).Distinct().ToList(); filePath = Path.Combine(directoryPath, $"{ campaignInfo.Name.ToLower().Replace("/", "_").TrimStart('_') }.rst"); try { using (StreamWriter streamWriter = new StreamWriter(new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.Read))) { restructuredTextWriter = new RestructuredTextWriter(streamWriter); // campaign header restructuredTextWriter.WriteHeading(campaignInfo.Name.TrimStart('/').Replace("/", " / "), SectionHeader.Section); // campaign description restructuredTextWriter.WriteLine(); try { if (IOHelper.CheckLinkExists(_vdsMetaFileId, campaignInfo.Name)) { campaign_groupId = H5G.open(_vdsMetaFileId, campaignInfo.Name); if (H5A.exists(campaign_groupId, "description") > 0) { description = IOHelper.ReadAttribute <string>(campaign_groupId, "description").First(); } else { description = "no description available"; } } else { description = "no description available"; } restructuredTextWriter.WriteNote(description); } finally { if (H5I.is_valid(campaign_groupId) > 0) { H5G.close(campaign_groupId); } } // groups foreach (string groupName in groupNameSet) { RestructuredTextTable restructuredTextTable; List <VariableInfo> groupedVariableInfoSet; restructuredTextWriter.WriteLine(); restructuredTextWriter.WriteHeading(groupName, SectionHeader.SubSection); restructuredTextWriter.WriteLine(); restructuredTextTable = new RestructuredTextTable(new List <string>() { "Name", "Unit", "Guid" }); groupedVariableInfoSet = campaignInfo.VariableInfoSet.Where(variableInfo => variableInfo.VariableGroupSet.Last() == groupName).OrderBy(variableInfo => variableInfo.VariableNameSet.Last()).ToList(); // variables groupedVariableInfoSet.ForEach(variableInfo => { long variable_groupId = -1; string groupPath; string name; string guid; string unit; List <hdf_transfer_function_t> transferFunctionSet; List <hdf_aggregate_function_t> aggregateFunctionSet; // name name = variableInfo.VariableNameSet.Last(); if (name.Count() > 43) { name = $"{ name.Substring(0, 40) }..."; } // guid guid = $"{ variableInfo.Name.Substring(0, 8) }..."; // unit, transferFunctionSet, aggregateFunctionSet unit = string.Empty; try { groupPath = variableInfo.GetPath(); if (IOHelper.CheckLinkExists(_vdsMetaFileId, groupPath)) { variable_groupId = H5G.open(_vdsMetaFileId, groupPath); if (H5A.exists(variable_groupId, "unit") > 0) { unit = IOHelper.ReadAttribute <string>(variable_groupId, "unit").FirstOrDefault(); } if (H5A.exists(variable_groupId, "transfer_function_set") > 0) { transferFunctionSet = IOHelper.ReadAttribute <hdf_transfer_function_t>(variable_groupId, "transfer_function_set").ToList(); } if (H5A.exists(variable_groupId, "aggregate_function_set") > 0) { aggregateFunctionSet = IOHelper.ReadAttribute <hdf_aggregate_function_t>(variable_groupId, "aggregate_function_set").ToList(); } } } finally { if (H5I.is_valid(variable_groupId) > 0) { H5G.close(variable_groupId); } } // restructuredTextTable.AddRow(new List <string> { name, unit, guid }); //transferFunctionSet.ForEach(x => streamWriter.Write($"{ x.date_time }, { x.type }, { x.option }, { x.argument } | ")); //aggregateFunctionSet.ForEach(x => streamWriter.Write($"{ x.type }, { x.argument } | ")); }); restructuredTextWriter.WriteTable(restructuredTextTable); restructuredTextWriter.WriteLine(); } } Console.WriteLine($"The file has been successfully written to:\n{ filePath }"); } catch (Exception ex) { Console.WriteLine(ex.ToString()); } Console.WriteLine("\nPress any key to continue ..."); Console.ReadKey(); }
public static List <CampaignInfo> InternalUpdateCampaignInfoSet(long fileId, bool isLazyLoading, List <CampaignInfo> campaignInfoSet = null, string campaignGroupPath = "") { ulong idx; idx = 0; if (campaignInfoSet == null) { campaignInfoSet = new List <CampaignInfo>(); } GeneralHelper.SuppressErrors(() => H5L.iterate(fileId, H5.index_t.NAME, H5.iter_order_t.INC, ref idx, Callback, Marshal.StringToHGlobalAnsi("/"))); return(campaignInfoSet); int Callback(long campaignGroupId, IntPtr intPtrName, ref H5L.info_t info, IntPtr intPtrUserData) { ulong idx2 = 0; long groupId = -1; long datasetId = -1; int level; int formatVersion; string name; string fullName; string userData; H5O.type_t objectType; CampaignInfo currentCampaignInfo; StringBuilder filePath; DateTime dateTime; // if (H5A.exists(fileId, "format_version") > 0) // raw file { formatVersion = IOHelper.ReadAttribute <int>(fileId, "format_version").First(); } else // virtual file { formatVersion = -1; } filePath = new StringBuilder(260); H5F.get_name(fileId, filePath, new IntPtr(260)); dateTime = DateTime.ParseExact(IOHelper.ReadAttribute <string>(fileId, "date_time").First(), "yyyy-MM-ddTHH-mm-ssZ", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal); name = Marshal.PtrToStringAnsi(intPtrName); userData = Marshal.PtrToStringAnsi(intPtrUserData); fullName = GeneralHelper.CombinePath(userData, name); level = userData.Split("/".ToArray()).Count(); // this is necessary, since H5Oget_info_by_name is slow because it wants verbose object header data // and H5G_loc_info is not directly accessible // only chance is to modify source code (H5Oget_info_by_name) datasetId = H5D.open(campaignGroupId, name); if (H5I.is_valid(datasetId) > 0) { objectType = H5O.type_t.DATASET; } else { groupId = H5G.open(campaignGroupId, name); if (H5I.is_valid(groupId) > 0) { objectType = H5O.type_t.GROUP; } else { objectType = H5O.type_t.UNKNOWN; } } switch (level) { case 1: case 2: break; case 3: if (objectType == H5O.type_t.GROUP) { if (!string.IsNullOrWhiteSpace(campaignGroupPath) && fullName != campaignGroupPath) { return(0); } currentCampaignInfo = campaignInfoSet.FirstOrDefault(campaignInfo => campaignInfo.Name == fullName); if (currentCampaignInfo == null) { currentCampaignInfo = new CampaignInfo(fullName, null, isLazyLoading); campaignInfoSet.Add(currentCampaignInfo); } currentCampaignInfo.Update(new FileContext(fileId, formatVersion, dateTime, filePath.ToString())); } break; } if (objectType == H5O.type_t.GROUP && level < 3) { H5L.iterate(groupId, H5.index_t.NAME, H5.iter_order_t.INC, ref idx2, Callback, Marshal.StringToHGlobalAnsi(fullName)); } // clean up if (H5I.is_valid(groupId) > 0) { H5G.close(groupId); } if (H5I.is_valid(datasetId) > 0) { H5D.close(datasetId); } return(0); } }