private ChannelHubBase CreateChannelHub(ChannelHubSettings channelHubSettings)
{
Type type;
type = typeof(ChannelHub <>).MakeGenericType(new Type[] { OneDasUtilities.GetTypeFromOneDasDataType(channelHubSettings.DataType) });
return((ChannelHubBase)Activator.CreateInstance(type, channelHubSettings));
}
public unsafe override object GetValue()
{
int elementSize;
byte * sourcePtr;
DataPort dataPort;
dataPort = this.AssociatedDataInput;
if (this.AssociatedDataInput != null)
{
sourcePtr = (byte *)dataPort.DataPtr.ToPointer();
elementSize = OneDasUtilities.SizeOf(typeof(T));
byte *targetPtr = stackalloc byte[elementSize];
if (dataPort.DataType == OneDasDataType.BOOLEAN && dataPort.BitOffset > -1) // special handling for boolean
{
// from bit to byte
bool value;
value = (*sourcePtr & (1 << dataPort.BitOffset)) > 0;
targetPtr[0] = *(byte *)&value;
}
else
{
switch (dataPort.Endianness)
{
case Endianness.LittleEndian:
for (int i = 0; i < elementSize; i++)
{
targetPtr[i] = sourcePtr[i];
}
break;
case Endianness.BigEndian:
for (int i = 0; i < elementSize; i++)
{
targetPtr[i] = sourcePtr[elementSize - i - 1];
}
break;
default:
throw new ArgumentException();
}
}
return(new Span <T>(targetPtr, 1)[0]);
}
else
{
return(null);
}
}
public unsafe DataStorageBase(Type type, int elementCount)
{
this.ElementSize = OneDasUtilities.SizeOf(type);
_byteCount = this.ElementSize * elementCount;
this.DataBufferPtr = Marshal.AllocHGlobal(_byteCount);
this.DataBuffer.Clear();
}
public void Validate()
{
Contract.Requires(this.ProductVersion != null);
string errorMessage;
if (!OneDasUtilities.CheckNamingConvention(this.Id, out errorMessage))
{
throw new Exception($"The ID is invalid: { errorMessage }");
}
}
public void Validate()
{
IEnumerable <Guid> guidSet;
string errorDescription;
// -> naming convention
if (!OneDasUtilities.CheckNamingConvention(this.Description.PrimaryGroupName, out errorDescription))
{
throw new Exception(ErrorMessage.OneDasProject_PrimaryGroupNameInvalid);
}
if (!OneDasUtilities.CheckNamingConvention(this.Description.SecondaryGroupName, out errorDescription))
{
throw new Exception(ErrorMessage.OneDasProject_SecondaryGroupNameInvalid);
}
if (!OneDasUtilities.CheckNamingConvention(this.Description.CampaignName, out errorDescription))
{
throw new Exception(ErrorMessage.OneDasProject_CampaignNameInvalid);
}
if (!this.ChannelHubSettingsSet.ToList().TrueForAll(x => OneDasUtilities.CheckNamingConvention(x.Name, out errorDescription)))
{
throw new Exception(ErrorMessage.OneDasProject_ChannelHubNameInvalid);
}
// -> ChannelHub
guidSet = this.ChannelHubSettingsSet.Select(x => x.Guid).ToList();
if (guidSet.Count() > guidSet.Distinct().Count())
{
throw new Exception(ErrorMessage.OneDasProject_ChannelHubNotUnqiue);
}
// -> data gateway settings
if (this.DataGatewaySettingsSet.Select(x => x.Description.InstanceId).Count() > this.DataGatewaySettingsSet.Select(x => x.Description.InstanceId).Distinct().Count())
{
throw new Exception(ErrorMessage.OneDasProject_DataGatewaySettingsIdNotUnique);
}
this.DataGatewaySettingsSet.ToList().ForEach(dataGatewaySettings => dataGatewaySettings.Validate());
// -> data writer settings
if (this.DataWriterSettingsSet.Select(x => x.Description.InstanceId).Count() > this.DataWriterSettingsSet.Select(x => x.Description.InstanceId).Distinct().Count())
{
throw new Exception(ErrorMessage.OneDasProject_DataWriterSettingsIdNotUnique);
}
this.DataWriterSettingsSet.ToList().ForEach(dataWriterSettings => dataWriterSettings.Validate());
}
protected virtual int SetDataPortBufferOffset(KeyValuePair <OneDasModule, List <DataPort> > moduleEntry, int bufferOffsetBase, DataDirection dataDirection)
{
int dataPortOffset;
dataPortOffset = 0;
moduleEntry.Value.ForEach(dataPort =>
{
dataPort.DataPtr = new IntPtr(bufferOffsetBase + dataPortOffset);
dataPortOffset += OneDasUtilities.SizeOf(dataPort.DataType);
});
return(moduleEntry.Key.GetByteCount());
}
public DataGatewaySampleGateway(DataGatewaySampleSettings settings) : base(settings)
{
Random random;
random = new Random();
_settings = settings;
_numberGeneratorSet = this.DataPortSet.Where(dataPort => dataPort.DataDirection == DataDirection.Input).ToList().Select(dataPort =>
{
Type type;
type = typeof(DataGatewaySampleNumberGenerator <>).MakeGenericType(OneDasUtilities.GetTypeFromOneDasDataType(dataPort.DataType));
return((DataGatewaySampleNumberGeneratorBase)Activator.CreateInstance(type, dataPort, random));
}).ToList();
}
public SlaveVariable(SlavePdo parent, string name, ushort index, byte subIndex, DataDirection dataDirection, OneDasDataType dataType, byte bitLength = 0) : base(name, dataType, dataDirection, Endianness.LittleEndian)
{
Contract.Requires(parent != null);
this.Parent = parent;
this.Index = index;
this.SubIndex = subIndex;
if (bitLength == 0)
{
this.BitLength = OneDasUtilities.GetBitLength(dataType, false);
}
else
{
this.BitLength = bitLength;
}
}
public DataWriterContext(string systemName, string dataDirectoryPath, OneDasCampaignDescription campaignDescription, IList <CustomMetadataEntry> customMetadataEntrySet)
{
Contract.Requires(customMetadataEntrySet != null);
customMetadataEntrySet.ToList().ForEach(customMetaDataEntry =>
{
if (!OneDasUtilities.CheckNamingConvention(customMetaDataEntry.Key, out errorDescription))
{
throw new ArgumentException($"Argument '{ nameof(customMetadataEntrySet) }', value '{ customMetaDataEntry.Key }': { errorDescription }");
}
});
this.SystemName = systemName;
this.DataDirectoryPath = dataDirectoryPath;
this.CampaignDescription = campaignDescription;
this.CustomMetadataEntrySet = customMetadataEntrySet;
}
public void Validate()
{
string errorMessage;
if (!OneDasUtilities.CheckNamingConvention(this.ExtensionId, out errorMessage))
{
throw new Exception($"The extension ID is invalid: { errorMessage }");
}
if (this.InstanceId < 0)
{
throw new Exception(ErrorMessage.ActionRequest_InstanceIdInvalid);
}
if (!OneDasUtilities.CheckNamingConvention(this.MethodName, out errorMessage))
{
throw new Exception($"The method name is invalid: { errorMessage }");
}
}
public void Validate()
{
string errorMessage;
if (this.Version < 0)
{
throw new Exception(ErrorMessage.OneDasCampaignDescription_InvalidVersion);
}
if (!OneDasUtilities.CheckNamingConvention(this.PrimaryGroupName, out errorMessage))
{
throw new Exception($"The PrimaryGroupName is invalid: { errorMessage }");
}
if (!OneDasUtilities.CheckNamingConvention(this.SecondaryGroupName, out errorMessage))
{
throw new Exception($"The SecondaryGroupName is invalid: { errorMessage }");
}
if (!OneDasUtilities.CheckNamingConvention(this.CampaignName, out errorMessage))
{
throw new Exception($"The CampaignName is invalid: { errorMessage }");
}
}
public Task <DataAvailabilityStatistics> GetDataAvailabilityStatistics(string campaignName, DateTime dateTimeBegin, DateTime dateTimeEnd)
{
long fileId = -1;
ulong lengthPerDay;
ulong offset;
ulong start;
ulong stride;
ulong block;
ulong count;
double totalDays;
int[] data;
int[] aggregatedData;
DateTime epochStart;
DateTime epochEnd;
DataAvailabilityGranularity granularity;
return(Task.Run(() =>
{
this.CheckState();
// open file
fileId = H5F.open(_options.VdsFilePath, H5F.ACC_RDONLY);
// epoch & hyperslab
epochStart = new DateTime(2000, 01, 01);
epochEnd = new DateTime(2030, 01, 01);
if (!(epochStart <= dateTimeBegin && dateTimeBegin <= dateTimeEnd && dateTimeEnd <= epochEnd))
{
throw new Exception("requirement >> epochStart <= dateTimeBegin && dateTimeBegin <= dateTimeEnd && dateTimeBegin <= epochEnd << is not matched");
}
lengthPerDay = OneDasUtilities.GetSamplesPerDayFromString("is_chunk_completed_set");
start = (ulong)(Math.Floor((dateTimeBegin - epochStart).TotalDays * lengthPerDay));
stride = 1;
block = (ulong)(Math.Ceiling((dateTimeEnd - dateTimeBegin).TotalDays * lengthPerDay));
count = 1;
// get data
totalDays = (dateTimeEnd - dateTimeBegin).TotalDays;
data = IOHelper.ReadDataset <byte>(fileId, $"{ campaignName }/is_chunk_completed_set", start, stride, block, count).Select(value => (int)value).ToArray();
if (totalDays <= 7)
{
granularity = DataAvailabilityGranularity.ChunkLevel;
aggregatedData = data;
}
else if (totalDays <= 365)
{
granularity = DataAvailabilityGranularity.DayLevel;
offset = (ulong)dateTimeBegin.TimeOfDay.TotalMinutes;
aggregatedData = new int[(int)Math.Ceiling(totalDays)];
Parallel.For(0, (int)Math.Ceiling(totalDays), day =>
{
ulong startIndex; // inclusive
ulong endIndex; // exclusive
startIndex = (ulong)day * lengthPerDay;
endIndex = startIndex + lengthPerDay;
if ((int)startIndex - (int)offset < 0)
{
startIndex = 0;
}
else
{
startIndex = startIndex - offset;
}
if (endIndex - offset >= (ulong)data.Length)
{
endIndex = (ulong)data.Length;
}
else
{
endIndex = endIndex - offset;
}
aggregatedData[day] = (int)((double)data.Skip((int)startIndex).Take((int)(endIndex - startIndex)).Sum() / (endIndex - startIndex) * 100);
});
}
else
{
int totalMonths;
DateTime totalDateTimeBegin;
totalMonths = (dateTimeEnd.Month - dateTimeBegin.Month) + 1 + 12 * (dateTimeEnd.Year - dateTimeBegin.Year);
totalDateTimeBegin = new DateTime(dateTimeBegin.Year, dateTimeBegin.Month, 1);
granularity = DataAvailabilityGranularity.MonthLevel;
offset = (ulong)(dateTimeBegin - totalDateTimeBegin).TotalMinutes;
aggregatedData = new int[totalMonths];
Parallel.For(0, totalMonths, month =>
{
ulong startIndex; // inclusive
ulong endIndex; // exclusive
DateTime currentDateTimeBegin;
DateTime currentDateTimeEnd;
currentDateTimeBegin = totalDateTimeBegin.AddMonths(month);
currentDateTimeEnd = currentDateTimeBegin.AddMonths(1);
if ((currentDateTimeBegin - totalDateTimeBegin).TotalMinutes - offset < 0)
{
startIndex = 0;
}
else
{
startIndex = (ulong)(currentDateTimeBegin - totalDateTimeBegin).TotalMinutes - offset;
}
if ((currentDateTimeEnd - totalDateTimeBegin).TotalMinutes - offset >= data.Length)
{
endIndex = (ulong)data.Length;
}
else
{
endIndex = (ulong)(currentDateTimeEnd - totalDateTimeBegin).TotalMinutes - offset;
}
aggregatedData[month] = (int)((double)data.Skip((int)startIndex).Take((int)(endIndex - startIndex)).Sum() / (endIndex - startIndex) * 100);
});
}
// clean up
H5F.close(fileId);
return new DataAvailabilityStatistics(granularity, aggregatedData);
}));
}
public Task <string> GetData(DateTime dateTimeBegin, DateTime dateTimeEnd, string sampleRateDescription, FileFormat fileFormat, FileGranularity fileGranularity, Dictionary <string, Dictionary <string, List <string> > > campaignInfoSet)
{
long fileId = -1;
long datasetId = -1;
ulong start;
ulong stride;
ulong block;
ulong count;
ulong segmentLength;
ulong segmentSize;
ulong bytesPerRow;
double sampleRate;
DateTime epochStart;
DateTime epochEnd;
string zipFilePath;
// task
return(Task.Run(() =>
{
this.CheckState();
if (!campaignInfoSet.Any())
{
return string.Empty;
}
// zip file
zipFilePath = Path.Combine(_options.SupportDirectoryPath, "EXPORT", $"OneDAS_{ dateTimeBegin.ToString("yyyy-MM-ddTHH-mm") }_{ sampleRateDescription }_{ Guid.NewGuid().ToString() }.zip");
// sampleRate
sampleRate = sampleRateDescription.ToSampleRate();
// epoch & hyperslab
epochStart = new DateTime(2000, 01, 01);
epochEnd = new DateTime(2030, 01, 01);
if (!(epochStart <= dateTimeBegin && dateTimeBegin <= dateTimeEnd && dateTimeEnd <= epochEnd))
{
throw new Exception("requirement >> epochStart <= dateTimeBegin && dateTimeBegin <= dateTimeEnd && dateTimeBegin <= epochEnd << is not matched");
}
start = (ulong)(Math.Floor((dateTimeBegin - epochStart).TotalSeconds * sampleRate));
stride = 1;
block = (ulong)(Math.Ceiling((dateTimeEnd - dateTimeBegin).TotalSeconds * sampleRate));
count = 1;
try
{
// open file
fileId = H5F.open(_options.VdsFilePath, H5F.ACC_RDONLY);
// byte count
bytesPerRow = 0;
foreach (var campaignInfo in campaignInfoSet)
{
foreach (var variableInfo in campaignInfo.Value)
{
foreach (string datasetInfo in variableInfo.Value)
{
try
{
datasetId = H5D.open(fileId, $"{ campaignInfo.Key }/{ variableInfo.Key }/{ datasetInfo }");
bytesPerRow += (ulong)OneDasUtilities.SizeOf(TypeConversionHelper.GetTypeFromHdfTypeId(H5D.get_type(datasetId)));
}
finally
{
if (H5I.is_valid(datasetId) > 0)
{
H5D.close(datasetId);
}
}
}
}
}
this.GetClient().SendByteCount(bytesPerRow * block);
segmentSize = (50 * 1024 * 1024) / bytesPerRow * bytesPerRow;
segmentLength = segmentSize / bytesPerRow;
// ensure that dataset length is multiple of 1 minute
if ((segmentLength / sampleRate) % 60 != 0)
{
segmentLength = (ulong)((ulong)(segmentLength / sampleRate / 60) * 60 * sampleRate);
}
// start
_stateManager.SetState(this.Context.ConnectionId, HdfExplorerState.Loading);
using (ZipArchive zipArchive = ZipFile.Open(zipFilePath, ZipArchiveMode.Create))
{
foreach (var campaignInfo in campaignInfoSet)
{
HdfDataLoader hdfDataLoader;
hdfDataLoader = new HdfDataLoader(_stateManager.GetToken(this.Context.ConnectionId));
hdfDataLoader.ProgressUpdated += this.OnProgressUpdated;
if (!hdfDataLoader.WriteZipFileCampaignEntry(zipArchive, fileGranularity, fileFormat, new ZipSettings(dateTimeBegin, campaignInfo, fileId, sampleRate, start, stride, block, count, segmentLength)))
{
return string.Empty;
}
}
}
}
catch (Exception ex)
{
this.WriteLogEntry(ex.Message, true);
throw;
}
finally
{
_stateManager.SetState(this.Context.ConnectionId, HdfExplorerState.Idle);
if (H5I.is_valid(fileId) > 0)
{
H5F.close(fileId);
}
}
this.WriteLogEntry($"{ this.Context.GetHttpContext().Connection.RemoteIpAddress } requested data: { dateTimeBegin.ToString("yyyy-MM-dd HH:mm:ss") } to { dateTimeEnd.ToString("yyyy-MM-dd HH:mm:ss") }", false);
return $"download/{ Path.GetFileName(zipFilePath) }";
}, _stateManager.GetToken(this.Context.ConnectionId)));
}
public bool WriteZipFileCampaignEntry(ZipArchive zipArchive, FileGranularity fileGranularity, FileFormat fileFormat, ZipSettings zipSettings)
{
IList <VariableDescription> variableDescriptionSet;
IList <CustomMetadataEntry> customMetadataEntrySet;
ZipArchiveEntry zipArchiveEntry;
DataWriterExtensionLogicBase dataWriter;
DataWriterExtensionSettingsBase settings;
DataWriterContext dataWriterContext;
string directoryPath;
string[] campaignName_splitted;
string[] filePathSet;
int currentFile;
int fileCount;
// build variable descriptions
variableDescriptionSet = new List <VariableDescription>();
zipSettings.CampaignInfo.Value.ToList().ForEach(variableInfo =>
{
variableInfo.Value.ForEach(datasetName =>
{
long groupId = -1;
long typeId = -1;
long datasetId = -1;
string displayName;
string groupName;
string unit;
ulong samplesPerDay;
OneDasDataType oneDasDataType;
hdf_transfer_function_t[] hdf_transfer_function_t_set;
List <TransferFunction> transferFunctionSet;
try
{
groupId = H5G.open(zipSettings.SourceFileId, $"{ zipSettings.CampaignInfo.Key }/{ variableInfo.Key }");
datasetId = H5D.open(groupId, datasetName);
typeId = H5D.get_type(datasetId);
displayName = IOHelper.ReadAttribute <string>(groupId, "name_set").Last();
groupName = IOHelper.ReadAttribute <string>(groupId, "group_set").Last();
unit = IOHelper.ReadAttribute <string>(groupId, "unit_set").LastOrDefault();
hdf_transfer_function_t_set = IOHelper.ReadAttribute <hdf_transfer_function_t>(groupId, "transfer_function_set");
transferFunctionSet = hdf_transfer_function_t_set.Select(tf => new TransferFunction(DateTime.ParseExact(tf.date_time, "yyyy-MM-ddTHH-mm-ssZ", CultureInfo.InvariantCulture), tf.type, tf.option, tf.argument)).ToList();
oneDasDataType = OneDasUtilities.GetOneDasDataTypeFromType(TypeConversionHelper.GetTypeFromHdfTypeId(typeId));
samplesPerDay = OneDasUtilities.GetSamplesPerDayFromString(datasetName);
variableDescriptionSet.Add(new VariableDescription(new Guid(variableInfo.Key), displayName, datasetName, groupName, oneDasDataType, samplesPerDay, unit, transferFunctionSet, typeof(ISimpleDataStorage)));
}
finally
{
if (H5I.is_valid(datasetId) > 0)
{
H5D.close(datasetId);
}
if (H5I.is_valid(groupId) > 0)
{
H5G.close(groupId);
}
if (H5I.is_valid(typeId) > 0)
{
H5T.close(typeId);
}
}
});
});
dataWriter = null; // REMOVE ONE IMPLEMENTED PROPERLY
switch (fileFormat)
{
case FileFormat.CSV:
//settings = new CsvSettings() { FileGranularity = fileGranularity };
//dataWriter = new CsvWriter((CsvSettings)settings, new LoggerFactory());
break;
case FileFormat.GAM:
//settings = new GamSettings() { FileGranularity = fileGranularity };
//dataWriter = new GamWriter((GamSettings)settings, new LoggerFactory());
break;
case FileFormat.MAT73:
//settings = new Mat73Settings() { FileGranularity = fileGranularity };
//dataWriter = new Mat73Writer((Mat73Settings)settings, new LoggerFactory());
break;
default:
throw new NotImplementedException();
}
// create temp directory
directoryPath = Path.Combine(Path.GetTempPath(), "OneDas.Hdf.Explorer", Guid.NewGuid().ToString());
Directory.CreateDirectory(directoryPath);
// create custom meta data
customMetadataEntrySet = new List <CustomMetadataEntry>();
//customMetadataEntrySet.Add(new CustomMetadataEntry("system_name", "HDF Explorer", CustomMetadataEntryLevel.File));
// initialize data writer
campaignName_splitted = zipSettings.CampaignInfo.Key.Split('/');
dataWriterContext = new DataWriterContext("HDF Explorer", directoryPath, new OneDasCampaignDescription(Guid.Empty, 0, campaignName_splitted[1], campaignName_splitted[2], campaignName_splitted[3]), customMetadataEntrySet);
dataWriter.Configure(dataWriterContext, variableDescriptionSet);
// create temp files
try
{
if (!this.CreateFiles(dataWriter, zipSettings))
{
this.CleanUp(directoryPath);
return(false);
}
}
finally
{
dataWriter.Dispose();
}
// write zip archive entries
filePathSet = Directory.GetFiles(directoryPath, "*", SearchOption.AllDirectories);
currentFile = 0;
fileCount = filePathSet.Count();
foreach (string filePath in filePathSet)
{
zipArchiveEntry = zipArchive.CreateEntry(Path.GetFileName(filePath), CompressionLevel.Optimal);
this.OnProgressUpdated(new ProgressUpdatedEventArgs(currentFile / (double)fileCount * 100, $"Writing file { currentFile + 1 } / { fileCount } to ZIP archive ..."));
using (FileStream fileStream = File.Open(filePath, FileMode.Open, FileAccess.Read))
{
using (Stream zipArchiveEntryStream = zipArchiveEntry.Open())
{
fileStream.CopyTo(zipArchiveEntryStream);
}
}
currentFile++;
}
this.CleanUp(directoryPath);
return(true);
}
private bool IsAssociationAllowed(DataPort dataPort, ChannelHubSettings channelHub)
{
return(OneDasUtilities.GetBitLength(dataPort.DataType, true) == OneDasUtilities.GetBitLength(channelHub.DataType, true));
}
