Exemple #1
0
        private ChannelHubBase CreateChannelHub(ChannelHubSettings channelHubSettings)
        {
            Type type;

            type = typeof(ChannelHub <>).MakeGenericType(new Type[] { OneDasUtilities.GetTypeFromOneDasDataType(channelHubSettings.DataType) });

            return((ChannelHubBase)Activator.CreateInstance(type, channelHubSettings));
        }
Exemple #2
0
        public unsafe override object GetValue()
        {
            int      elementSize;
            byte *   sourcePtr;
            DataPort dataPort;

            dataPort = this.AssociatedDataInput;

            if (this.AssociatedDataInput != null)
            {
                sourcePtr   = (byte *)dataPort.DataPtr.ToPointer();
                elementSize = OneDasUtilities.SizeOf(typeof(T));

                byte *targetPtr = stackalloc byte[elementSize];

                if (dataPort.DataType == OneDasDataType.BOOLEAN && dataPort.BitOffset > -1) // special handling for boolean
                {
                    // from bit to byte
                    bool value;

                    value        = (*sourcePtr & (1 << dataPort.BitOffset)) > 0;
                    targetPtr[0] = *(byte *)&value;
                }
                else
                {
                    switch (dataPort.Endianness)
                    {
                    case Endianness.LittleEndian:

                        for (int i = 0; i < elementSize; i++)
                        {
                            targetPtr[i] = sourcePtr[i];
                        }

                        break;

                    case Endianness.BigEndian:

                        for (int i = 0; i < elementSize; i++)
                        {
                            targetPtr[i] = sourcePtr[elementSize - i - 1];
                        }

                        break;

                    default:

                        throw new ArgumentException();
                    }
                }

                return(new Span <T>(targetPtr, 1)[0]);
            }
            else
            {
                return(null);
            }
        }
Exemple #3
0
        public unsafe DataStorageBase(Type type, int elementCount)
        {
            this.ElementSize = OneDasUtilities.SizeOf(type);

            _byteCount = this.ElementSize * elementCount;

            this.DataBufferPtr = Marshal.AllocHGlobal(_byteCount);
            this.DataBuffer.Clear();
        }
        public void Validate()
        {
            Contract.Requires(this.ProductVersion != null);

            string errorMessage;

            if (!OneDasUtilities.CheckNamingConvention(this.Id, out errorMessage))
            {
                throw new Exception($"The ID is invalid: { errorMessage }");
            }
        }
        public void Validate()
        {
            IEnumerable <Guid> guidSet;

            string errorDescription;

            // -> naming convention
            if (!OneDasUtilities.CheckNamingConvention(this.Description.PrimaryGroupName, out errorDescription))
            {
                throw new Exception(ErrorMessage.OneDasProject_PrimaryGroupNameInvalid);
            }

            if (!OneDasUtilities.CheckNamingConvention(this.Description.SecondaryGroupName, out errorDescription))
            {
                throw new Exception(ErrorMessage.OneDasProject_SecondaryGroupNameInvalid);
            }

            if (!OneDasUtilities.CheckNamingConvention(this.Description.CampaignName, out errorDescription))
            {
                throw new Exception(ErrorMessage.OneDasProject_CampaignNameInvalid);
            }

            if (!this.ChannelHubSettingsSet.ToList().TrueForAll(x => OneDasUtilities.CheckNamingConvention(x.Name, out errorDescription)))
            {
                throw new Exception(ErrorMessage.OneDasProject_ChannelHubNameInvalid);
            }

            // -> ChannelHub
            guidSet = this.ChannelHubSettingsSet.Select(x => x.Guid).ToList();

            if (guidSet.Count() > guidSet.Distinct().Count())
            {
                throw new Exception(ErrorMessage.OneDasProject_ChannelHubNotUnqiue);
            }

            // -> data gateway settings
            if (this.DataGatewaySettingsSet.Select(x => x.Description.InstanceId).Count() > this.DataGatewaySettingsSet.Select(x => x.Description.InstanceId).Distinct().Count())
            {
                throw new Exception(ErrorMessage.OneDasProject_DataGatewaySettingsIdNotUnique);
            }

            this.DataGatewaySettingsSet.ToList().ForEach(dataGatewaySettings => dataGatewaySettings.Validate());

            // -> data writer settings
            if (this.DataWriterSettingsSet.Select(x => x.Description.InstanceId).Count() > this.DataWriterSettingsSet.Select(x => x.Description.InstanceId).Distinct().Count())
            {
                throw new Exception(ErrorMessage.OneDasProject_DataWriterSettingsIdNotUnique);
            }

            this.DataWriterSettingsSet.ToList().ForEach(dataWriterSettings => dataWriterSettings.Validate());
        }
        protected virtual int SetDataPortBufferOffset(KeyValuePair <OneDasModule, List <DataPort> > moduleEntry, int bufferOffsetBase, DataDirection dataDirection)
        {
            int dataPortOffset;

            dataPortOffset = 0;

            moduleEntry.Value.ForEach(dataPort =>
            {
                dataPort.DataPtr = new IntPtr(bufferOffsetBase + dataPortOffset);
                dataPortOffset  += OneDasUtilities.SizeOf(dataPort.DataType);
            });

            return(moduleEntry.Key.GetByteCount());
        }
        public DataGatewaySampleGateway(DataGatewaySampleSettings settings) : base(settings)
        {
            Random random;

            random    = new Random();
            _settings = settings;

            _numberGeneratorSet = this.DataPortSet.Where(dataPort => dataPort.DataDirection == DataDirection.Input).ToList().Select(dataPort =>
            {
                Type type;

                type = typeof(DataGatewaySampleNumberGenerator <>).MakeGenericType(OneDasUtilities.GetTypeFromOneDasDataType(dataPort.DataType));

                return((DataGatewaySampleNumberGeneratorBase)Activator.CreateInstance(type, dataPort, random));
            }).ToList();
        }
Exemple #8
0
        public SlaveVariable(SlavePdo parent, string name, ushort index, byte subIndex, DataDirection dataDirection, OneDasDataType dataType, byte bitLength = 0) : base(name, dataType, dataDirection, Endianness.LittleEndian)
        {
            Contract.Requires(parent != null);

            this.Parent   = parent;
            this.Index    = index;
            this.SubIndex = subIndex;

            if (bitLength == 0)
            {
                this.BitLength = OneDasUtilities.GetBitLength(dataType, false);
            }
            else
            {
                this.BitLength = bitLength;
            }
        }
        public DataWriterContext(string systemName, string dataDirectoryPath, OneDasCampaignDescription campaignDescription, IList <CustomMetadataEntry> customMetadataEntrySet)
        {
            Contract.Requires(customMetadataEntrySet != null);

            customMetadataEntrySet.ToList().ForEach(customMetaDataEntry =>
            {
                if (!OneDasUtilities.CheckNamingConvention(customMetaDataEntry.Key, out errorDescription))
                {
                    throw new ArgumentException($"Argument '{ nameof(customMetadataEntrySet) }', value '{ customMetaDataEntry.Key }': { errorDescription }");
                }
            });

            this.SystemName             = systemName;
            this.DataDirectoryPath      = dataDirectoryPath;
            this.CampaignDescription    = campaignDescription;
            this.CustomMetadataEntrySet = customMetadataEntrySet;
        }
Exemple #10
0
        public void Validate()
        {
            string errorMessage;

            if (!OneDasUtilities.CheckNamingConvention(this.ExtensionId, out errorMessage))
            {
                throw new Exception($"The extension ID is invalid: { errorMessage }");
            }

            if (this.InstanceId < 0)
            {
                throw new Exception(ErrorMessage.ActionRequest_InstanceIdInvalid);
            }

            if (!OneDasUtilities.CheckNamingConvention(this.MethodName, out errorMessage))
            {
                throw new Exception($"The method name is invalid: { errorMessage }");
            }
        }
        public void Validate()
        {
            string errorMessage;

            if (this.Version < 0)
            {
                throw new Exception(ErrorMessage.OneDasCampaignDescription_InvalidVersion);
            }

            if (!OneDasUtilities.CheckNamingConvention(this.PrimaryGroupName, out errorMessage))
            {
                throw new Exception($"The PrimaryGroupName is invalid: { errorMessage }");
            }

            if (!OneDasUtilities.CheckNamingConvention(this.SecondaryGroupName, out errorMessage))
            {
                throw new Exception($"The SecondaryGroupName is invalid: { errorMessage }");
            }

            if (!OneDasUtilities.CheckNamingConvention(this.CampaignName, out errorMessage))
            {
                throw new Exception($"The CampaignName is invalid: { errorMessage }");
            }
        }
Exemple #12
0
        public Task <DataAvailabilityStatistics> GetDataAvailabilityStatistics(string campaignName, DateTime dateTimeBegin, DateTime dateTimeEnd)
        {
            long fileId = -1;

            ulong lengthPerDay;
            ulong offset;

            ulong start;
            ulong stride;
            ulong block;
            ulong count;

            double totalDays;

            int[] data;
            int[] aggregatedData;

            DateTime epochStart;
            DateTime epochEnd;

            DataAvailabilityGranularity granularity;

            return(Task.Run(() =>
            {
                this.CheckState();

                // open file
                fileId = H5F.open(_options.VdsFilePath, H5F.ACC_RDONLY);

                // epoch & hyperslab
                epochStart = new DateTime(2000, 01, 01);
                epochEnd = new DateTime(2030, 01, 01);

                if (!(epochStart <= dateTimeBegin && dateTimeBegin <= dateTimeEnd && dateTimeEnd <= epochEnd))
                {
                    throw new Exception("requirement >> epochStart <= dateTimeBegin && dateTimeBegin <= dateTimeEnd && dateTimeBegin <= epochEnd << is not matched");
                }

                lengthPerDay = OneDasUtilities.GetSamplesPerDayFromString("is_chunk_completed_set");

                start = (ulong)(Math.Floor((dateTimeBegin - epochStart).TotalDays * lengthPerDay));
                stride = 1;
                block = (ulong)(Math.Ceiling((dateTimeEnd - dateTimeBegin).TotalDays * lengthPerDay));
                count = 1;

                // get data
                totalDays = (dateTimeEnd - dateTimeBegin).TotalDays;
                data = IOHelper.ReadDataset <byte>(fileId, $"{ campaignName }/is_chunk_completed_set", start, stride, block, count).Select(value => (int)value).ToArray();

                if (totalDays <= 7)
                {
                    granularity = DataAvailabilityGranularity.ChunkLevel;
                    aggregatedData = data;
                }
                else if (totalDays <= 365)
                {
                    granularity = DataAvailabilityGranularity.DayLevel;
                    offset = (ulong)dateTimeBegin.TimeOfDay.TotalMinutes;
                    aggregatedData = new int[(int)Math.Ceiling(totalDays)];

                    Parallel.For(0, (int)Math.Ceiling(totalDays), day =>
                    {
                        ulong startIndex; // inclusive
                        ulong endIndex;   // exclusive

                        startIndex = (ulong)day * lengthPerDay;
                        endIndex = startIndex + lengthPerDay;

                        if ((int)startIndex - (int)offset < 0)
                        {
                            startIndex = 0;
                        }
                        else
                        {
                            startIndex = startIndex - offset;
                        }

                        if (endIndex - offset >= (ulong)data.Length)
                        {
                            endIndex = (ulong)data.Length;
                        }
                        else
                        {
                            endIndex = endIndex - offset;
                        }

                        aggregatedData[day] = (int)((double)data.Skip((int)startIndex).Take((int)(endIndex - startIndex)).Sum() / (endIndex - startIndex) * 100);
                    });
                }
                else
                {
                    int totalMonths;
                    DateTime totalDateTimeBegin;

                    totalMonths = (dateTimeEnd.Month - dateTimeBegin.Month) + 1 + 12 * (dateTimeEnd.Year - dateTimeBegin.Year);
                    totalDateTimeBegin = new DateTime(dateTimeBegin.Year, dateTimeBegin.Month, 1);
                    granularity = DataAvailabilityGranularity.MonthLevel;
                    offset = (ulong)(dateTimeBegin - totalDateTimeBegin).TotalMinutes;
                    aggregatedData = new int[totalMonths];

                    Parallel.For(0, totalMonths, month =>
                    {
                        ulong startIndex; // inclusive
                        ulong endIndex;   // exclusive

                        DateTime currentDateTimeBegin;
                        DateTime currentDateTimeEnd;

                        currentDateTimeBegin = totalDateTimeBegin.AddMonths(month);
                        currentDateTimeEnd = currentDateTimeBegin.AddMonths(1);

                        if ((currentDateTimeBegin - totalDateTimeBegin).TotalMinutes - offset < 0)
                        {
                            startIndex = 0;
                        }
                        else
                        {
                            startIndex = (ulong)(currentDateTimeBegin - totalDateTimeBegin).TotalMinutes - offset;
                        }

                        if ((currentDateTimeEnd - totalDateTimeBegin).TotalMinutes - offset >= data.Length)
                        {
                            endIndex = (ulong)data.Length;
                        }
                        else
                        {
                            endIndex = (ulong)(currentDateTimeEnd - totalDateTimeBegin).TotalMinutes - offset;
                        }

                        aggregatedData[month] = (int)((double)data.Skip((int)startIndex).Take((int)(endIndex - startIndex)).Sum() / (endIndex - startIndex) * 100);
                    });
                }

                // clean up
                H5F.close(fileId);

                return new DataAvailabilityStatistics(granularity, aggregatedData);
            }));
        }
Exemple #13
0
        public Task <string> GetData(DateTime dateTimeBegin, DateTime dateTimeEnd, string sampleRateDescription, FileFormat fileFormat, FileGranularity fileGranularity, Dictionary <string, Dictionary <string, List <string> > > campaignInfoSet)
        {
            long fileId    = -1;
            long datasetId = -1;

            ulong start;
            ulong stride;
            ulong block;
            ulong count;

            ulong segmentLength;
            ulong segmentSize;
            ulong bytesPerRow;

            double sampleRate;

            DateTime epochStart;
            DateTime epochEnd;

            string zipFilePath;

            // task
            return(Task.Run(() =>
            {
                this.CheckState();

                if (!campaignInfoSet.Any())
                {
                    return string.Empty;
                }

                // zip file
                zipFilePath = Path.Combine(_options.SupportDirectoryPath, "EXPORT", $"OneDAS_{ dateTimeBegin.ToString("yyyy-MM-ddTHH-mm") }_{ sampleRateDescription }_{ Guid.NewGuid().ToString() }.zip");

                // sampleRate
                sampleRate = sampleRateDescription.ToSampleRate();

                // epoch & hyperslab
                epochStart = new DateTime(2000, 01, 01);
                epochEnd = new DateTime(2030, 01, 01);

                if (!(epochStart <= dateTimeBegin && dateTimeBegin <= dateTimeEnd && dateTimeEnd <= epochEnd))
                {
                    throw new Exception("requirement >> epochStart <= dateTimeBegin && dateTimeBegin <= dateTimeEnd && dateTimeBegin <= epochEnd << is not matched");
                }

                start = (ulong)(Math.Floor((dateTimeBegin - epochStart).TotalSeconds * sampleRate));
                stride = 1;
                block = (ulong)(Math.Ceiling((dateTimeEnd - dateTimeBegin).TotalSeconds * sampleRate));
                count = 1;

                try
                {
                    // open file
                    fileId = H5F.open(_options.VdsFilePath, H5F.ACC_RDONLY);

                    // byte count
                    bytesPerRow = 0;

                    foreach (var campaignInfo in campaignInfoSet)
                    {
                        foreach (var variableInfo in campaignInfo.Value)
                        {
                            foreach (string datasetInfo in variableInfo.Value)
                            {
                                try
                                {
                                    datasetId = H5D.open(fileId, $"{ campaignInfo.Key }/{ variableInfo.Key }/{ datasetInfo }");
                                    bytesPerRow += (ulong)OneDasUtilities.SizeOf(TypeConversionHelper.GetTypeFromHdfTypeId(H5D.get_type(datasetId)));
                                }
                                finally
                                {
                                    if (H5I.is_valid(datasetId) > 0)
                                    {
                                        H5D.close(datasetId);
                                    }
                                }
                            }
                        }
                    }

                    this.GetClient().SendByteCount(bytesPerRow * block);

                    segmentSize = (50 * 1024 * 1024) / bytesPerRow * bytesPerRow;
                    segmentLength = segmentSize / bytesPerRow;

                    // ensure that dataset length is multiple of 1 minute
                    if ((segmentLength / sampleRate) % 60 != 0)
                    {
                        segmentLength = (ulong)((ulong)(segmentLength / sampleRate / 60) * 60 * sampleRate);
                    }

                    // start
                    _stateManager.SetState(this.Context.ConnectionId, HdfExplorerState.Loading);

                    using (ZipArchive zipArchive = ZipFile.Open(zipFilePath, ZipArchiveMode.Create))
                    {
                        foreach (var campaignInfo in campaignInfoSet)
                        {
                            HdfDataLoader hdfDataLoader;

                            hdfDataLoader = new HdfDataLoader(_stateManager.GetToken(this.Context.ConnectionId));
                            hdfDataLoader.ProgressUpdated += this.OnProgressUpdated;

                            if (!hdfDataLoader.WriteZipFileCampaignEntry(zipArchive, fileGranularity, fileFormat, new ZipSettings(dateTimeBegin, campaignInfo, fileId, sampleRate, start, stride, block, count, segmentLength)))
                            {
                                return string.Empty;
                            }
                        }
                    }
                }
                catch (Exception ex)
                {
                    this.WriteLogEntry(ex.Message, true);
                    throw;
                }
                finally
                {
                    _stateManager.SetState(this.Context.ConnectionId, HdfExplorerState.Idle);

                    if (H5I.is_valid(fileId) > 0)
                    {
                        H5F.close(fileId);
                    }
                }

                this.WriteLogEntry($"{ this.Context.GetHttpContext().Connection.RemoteIpAddress } requested data: { dateTimeBegin.ToString("yyyy-MM-dd HH:mm:ss") } to { dateTimeEnd.ToString("yyyy-MM-dd HH:mm:ss") }", false);

                return $"download/{ Path.GetFileName(zipFilePath) }";
            }, _stateManager.GetToken(this.Context.ConnectionId)));
        }
Exemple #14
0
        public bool WriteZipFileCampaignEntry(ZipArchive zipArchive, FileGranularity fileGranularity, FileFormat fileFormat, ZipSettings zipSettings)
        {
            IList <VariableDescription> variableDescriptionSet;
            IList <CustomMetadataEntry> customMetadataEntrySet;

            ZipArchiveEntry zipArchiveEntry;
            DataWriterExtensionLogicBase    dataWriter;
            DataWriterExtensionSettingsBase settings;
            DataWriterContext dataWriterContext;

            string directoryPath;

            string[] campaignName_splitted;
            string[] filePathSet;

            int currentFile;
            int fileCount;

            // build variable descriptions
            variableDescriptionSet = new List <VariableDescription>();

            zipSettings.CampaignInfo.Value.ToList().ForEach(variableInfo =>
            {
                variableInfo.Value.ForEach(datasetName =>
                {
                    long groupId   = -1;
                    long typeId    = -1;
                    long datasetId = -1;

                    string displayName;
                    string groupName;
                    string unit;

                    ulong samplesPerDay;

                    OneDasDataType oneDasDataType;

                    hdf_transfer_function_t[] hdf_transfer_function_t_set;
                    List <TransferFunction> transferFunctionSet;

                    try
                    {
                        groupId   = H5G.open(zipSettings.SourceFileId, $"{ zipSettings.CampaignInfo.Key }/{ variableInfo.Key }");
                        datasetId = H5D.open(groupId, datasetName);
                        typeId    = H5D.get_type(datasetId);

                        displayName = IOHelper.ReadAttribute <string>(groupId, "name_set").Last();
                        groupName   = IOHelper.ReadAttribute <string>(groupId, "group_set").Last();
                        unit        = IOHelper.ReadAttribute <string>(groupId, "unit_set").LastOrDefault();
                        hdf_transfer_function_t_set = IOHelper.ReadAttribute <hdf_transfer_function_t>(groupId, "transfer_function_set");
                        transferFunctionSet         = hdf_transfer_function_t_set.Select(tf => new TransferFunction(DateTime.ParseExact(tf.date_time, "yyyy-MM-ddTHH-mm-ssZ", CultureInfo.InvariantCulture), tf.type, tf.option, tf.argument)).ToList();

                        oneDasDataType = OneDasUtilities.GetOneDasDataTypeFromType(TypeConversionHelper.GetTypeFromHdfTypeId(typeId));
                        samplesPerDay  = OneDasUtilities.GetSamplesPerDayFromString(datasetName);

                        variableDescriptionSet.Add(new VariableDescription(new Guid(variableInfo.Key), displayName, datasetName, groupName, oneDasDataType, samplesPerDay, unit, transferFunctionSet, typeof(ISimpleDataStorage)));
                    }
                    finally
                    {
                        if (H5I.is_valid(datasetId) > 0)
                        {
                            H5D.close(datasetId);
                        }
                        if (H5I.is_valid(groupId) > 0)
                        {
                            H5G.close(groupId);
                        }
                        if (H5I.is_valid(typeId) > 0)
                        {
                            H5T.close(typeId);
                        }
                    }
                });
            });

            dataWriter = null; // REMOVE ONE IMPLEMENTED PROPERLY

            switch (fileFormat)
            {
            case FileFormat.CSV:

                //settings = new CsvSettings() { FileGranularity = fileGranularity };
                //dataWriter = new CsvWriter((CsvSettings)settings, new LoggerFactory());

                break;

            case FileFormat.GAM:

                //settings = new GamSettings() { FileGranularity = fileGranularity };
                //dataWriter = new GamWriter((GamSettings)settings, new LoggerFactory());

                break;

            case FileFormat.MAT73:

                //settings = new Mat73Settings() { FileGranularity = fileGranularity };
                //dataWriter = new Mat73Writer((Mat73Settings)settings, new LoggerFactory());

                break;

            default:
                throw new NotImplementedException();
            }

            // create temp directory
            directoryPath = Path.Combine(Path.GetTempPath(), "OneDas.Hdf.Explorer", Guid.NewGuid().ToString());
            Directory.CreateDirectory(directoryPath);

            // create custom meta data
            customMetadataEntrySet = new List <CustomMetadataEntry>();
            //customMetadataEntrySet.Add(new CustomMetadataEntry("system_name", "HDF Explorer", CustomMetadataEntryLevel.File));

            // initialize data writer
            campaignName_splitted = zipSettings.CampaignInfo.Key.Split('/');
            dataWriterContext     = new DataWriterContext("HDF Explorer", directoryPath, new OneDasCampaignDescription(Guid.Empty, 0, campaignName_splitted[1], campaignName_splitted[2], campaignName_splitted[3]), customMetadataEntrySet);
            dataWriter.Configure(dataWriterContext, variableDescriptionSet);

            // create temp files
            try
            {
                if (!this.CreateFiles(dataWriter, zipSettings))
                {
                    this.CleanUp(directoryPath);

                    return(false);
                }
            }
            finally
            {
                dataWriter.Dispose();
            }

            // write zip archive entries
            filePathSet = Directory.GetFiles(directoryPath, "*", SearchOption.AllDirectories);
            currentFile = 0;
            fileCount   = filePathSet.Count();

            foreach (string filePath in filePathSet)
            {
                zipArchiveEntry = zipArchive.CreateEntry(Path.GetFileName(filePath), CompressionLevel.Optimal);

                this.OnProgressUpdated(new ProgressUpdatedEventArgs(currentFile / (double)fileCount * 100, $"Writing file { currentFile + 1 } / { fileCount } to ZIP archive ..."));

                using (FileStream fileStream = File.Open(filePath, FileMode.Open, FileAccess.Read))
                {
                    using (Stream zipArchiveEntryStream = zipArchiveEntry.Open())
                    {
                        fileStream.CopyTo(zipArchiveEntryStream);
                    }
                }

                currentFile++;
            }

            this.CleanUp(directoryPath);

            return(true);
        }
Exemple #15
0
 private bool IsAssociationAllowed(DataPort dataPort, ChannelHubSettings channelHub)
 {
     return(OneDasUtilities.GetBitLength(dataPort.DataType, true) == OneDasUtilities.GetBitLength(channelHub.DataType, true));
 }