public ExtendedBuffer(int length) : base(length)
{
_buffer = MemoryPool <T> .Shared.Rent(length);
MemoryMarshal.AsBytes(_buffer.Memory.Span).Fill(0);
this.ElementSize = NexusUtilities.SizeOf(typeof(T));
}
public static double[] ApplyDatasetStatus2(Array dataset, byte[] status)
{
var methodName = nameof(BufferUtilities.ForwardApplyDatasetStatus);
var flags = BindingFlags.NonPublic | BindingFlags.Static;
var genericType = dataset.GetType().GetElementType();
var parameters = new object[] { dataset, status };
var result = NexusUtilities.InvokeGenericMethod(typeof(BufferUtilities), null, methodName, flags, genericType, parameters);
return((double[])result);
}
public void Validate()
{
Contract.Requires(this.ProductVersion != null);
string errorMessage;
if (!NexusUtilities.CheckNamingConvention(this.Id, out errorMessage))
{
throw new Exception($"The ID is invalid: { errorMessage }");
}
}
public long GetByteCount()
{
var totalDays = (this.DateTimeEnd - this.DateTimeBegin).TotalDays;
var frequency = string.IsNullOrWhiteSpace(this.SampleRate) ? 0 : new SampleRateContainer(this.SampleRate).SamplesPerDay;
return((long)this.GetSelectedDatasets().Sum(dataset =>
{
var elementSize = NexusUtilities.SizeOf(dataset.DataType);
return frequency * totalDays * elementSize;
}));
}
protected override ValidationResult IsValid(object value, ValidationContext validationContext)
{
var name = value as string;
if (NexusUtilities.CheckNamingConvention(name, out var errorDescription))
{
return(ValidationResult.Success);
}
else
{
return(new ValidationResult(errorDescription));
}
}
public DataWriterContext(string systemName, string dataDirectoryPath, NexusProjectDescription projectDescription, IList <CustomMetadataEntry> customMetadataEntrySet)
{
Contract.Requires(customMetadataEntrySet != null);
customMetadataEntrySet.ToList().ForEach(customMetaDataEntry =>
{
if (!NexusUtilities.CheckNamingConvention(customMetaDataEntry.Key, out errorDescription))
{
throw new ArgumentException($"Argument '{ nameof(customMetadataEntrySet) }', value '{ customMetaDataEntry.Key }': { errorDescription }");
}
});
this.SystemName = systemName;
this.DataDirectoryPath = dataDirectoryPath;
this.ProjectDescription = projectDescription;
this.CustomMetadataEntrySet = customMetadataEntrySet;
}
public DataReaderDoubleStream ReadAsDoubleStream(
DatasetInfo dataset,
DateTime begin,
DateTime end,
ulong upperBlockSize,
CancellationToken cancellationToken)
{
var progressRecords = this.Read(new List <DatasetInfo>()
{
dataset
}, begin, end, upperBlockSize, TimeSpan.FromMinutes(1), cancellationToken);
var samplesPerSecond = new SampleRateContainer(dataset.Id).SamplesPerSecond;
var length = (long)Math.Round(samplesPerSecond *
(decimal)(end - begin).TotalSeconds, MidpointRounding.AwayFromZero) *
NexusUtilities.SizeOf(NexusDataType.FLOAT64);
return(new DataReaderDoubleStream(length, progressRecords));
}
public void Validate()
{
string errorMessage;
if (this.Version < 0)
{
throw new Exception(ErrorMessage.NexusProjectDescription_InvalidVersion);
}
if (!NexusUtilities.CheckNamingConvention(this.PrimaryGroupName, out errorMessage))
{
throw new Exception($"The PrimaryGroupName is invalid: { errorMessage }");
}
if (!NexusUtilities.CheckNamingConvention(this.SecondaryGroupName, out errorMessage))
{
throw new Exception($"The SecondaryGroupName is invalid: { errorMessage }");
}
if (!NexusUtilities.CheckNamingConvention(this.ProjectName, out errorMessage))
{
throw new Exception($"The ProjectName is invalid: { errorMessage }");
}
}
public static IExtendedBuffer CreateExtendedBuffer(NexusDataType dataType, int length)
{
var type = typeof(ExtendedBuffer <>).MakeGenericType(new Type[] { NexusUtilities.GetTypeFromNexusDataType(dataType) });
return((IExtendedBuffer)Activator.CreateInstance(type, length));
}
private void AggregateProject(ClaimsPrincipal user,
string databaseFolderPath,
string projectId,
DateTime date,
AggregationSetup setup,
AggregationInstruction instruction,
CancellationToken cancellationToken)
{
foreach (var(registration, aggregationChannels) in instruction.DataReaderToAggregationsMap)
{
using var dataReader = _databaseManager.GetDataReader(user, registration);
// find reader configurations
foreach (var configuration in setup.ReaderConfigurations
.Where(configuration => configuration.ProjectId == projectId))
{
var tmpRegistration = new DataReaderRegistration()
{
RootPath = configuration.DataReaderRootPath,
DataReaderId = configuration.DataReaderId
};
if (dataReader.Registration.Equals(tmpRegistration))
{
dataReader.OptionalParameters = configuration.Parameters;
break;
}
}
// get files
if (!dataReader.IsDataOfDayAvailable(projectId, date))
{
return;
}
// project
var container = _databaseManager.Database.ProjectContainers.FirstOrDefault(container => container.Id == projectId);
if (container == null)
{
throw new Exception($"The requested project '{projectId}' could not be found.");
}
var targetDirectoryPath = Path.Combine(databaseFolderPath, "DATA", WebUtility.UrlEncode(container.Id), date.ToString("yyyy-MM"), date.ToString("dd"));
// for each channel
foreach (var aggregationChannel in aggregationChannels)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
var dataset = aggregationChannel.Channel.Datasets.First();
NexusUtilities.InvokeGenericMethod(this, nameof(this.OrchestrateAggregation),
BindingFlags.Instance | BindingFlags.NonPublic,
NexusUtilities.GetTypeFromNexusDataType(dataset.DataType),
new object[]
{
targetDirectoryPath,
dataReader,
dataset,
aggregationChannel.Aggregations,
date,
setup.Force,
cancellationToken
});
}
catch (TaskCanceledException)
{
throw;
}
catch (Exception ex)
{
_logger.LogError(ex.GetFullMessage());
}
}
}
}
private RenderFragment CreateComponent() => builder =>
{
var properties = this.DataContext.GetType().GetProperties(BindingFlags.Public | BindingFlags.Instance);
var sequence = 0; // Autoincrement is not recommended for performance reasons (https://docs.microsoft.com/en-us/aspnet/core/blazor/advanced-scenarios?view=aspnetcore-5.0#the-problem-with-generating-sequence-numbers-programmatically)
builder.OpenElement(sequence++, "div");
builder.AddAttribute(sequence++, "class", "form-generator");
foreach (var property in properties)
{
builder.OpenElement(sequence++, "div");
builder.AddAttribute(sequence++, "class", "form-generator-property");
var type = this.DataContext.GetType();
var constant = Expression.Constant(this.DataContext, type);
var expression = MemberExpression.Property(constant, property.Name);
var value = property.GetValue(this.DataContext);
if (property.PropertyType == typeof(string))
{
builder.OpenComponent(sequence++, typeof(MatTextField <string>));
builder.AddAttribute(sequence++, "Value", value);
builder.AddAttribute(sequence++, "ValueChanged",
EventCallback.Factory.Create <string>(this, value =>
{
property.SetValue(this.DataContext, value);
this.OnChanged();
}));
builder.AddAttribute(sequence++, "Label", property.Name);
builder.CloseComponent();
}
else if (property.PropertyType == typeof(bool))
{
builder.OpenComponent(sequence++, typeof(MatCheckbox <bool>));
builder.AddAttribute(sequence++, "Value", value);
builder.AddAttribute(sequence++, "ValueChanged",
EventCallback.Factory.Create <bool>(this, value =>
{
property.SetValue(this.DataContext, value);
this.OnChanged();
}));
builder.AddAttribute(sequence++, "Label", property.Name);
builder.CloseComponent();
}
else if (property.PropertyType == typeof(DateTime))
{
builder.OpenComponent(sequence++, typeof(MatDatePicker <DateTime>));
builder.AddAttribute(sequence++, "Value", value);
builder.AddAttribute(sequence++, "ValueChanged",
EventCallback.Factory.Create <DateTime>(this, value =>
{
property.SetValue(this.DataContext, value);
this.OnChanged();
}));
builder.AddAttribute(sequence++, "Label", property.Name);
builder.AddAttribute(sequence++, "Format", "dd.MM.yyyy HH:mm");
builder.AddAttribute(sequence++, "EnableTime", true);
builder.AddAttribute(sequence++, "EnableSeconds", false);
builder.AddAttribute(sequence++, "Enable24hours", true);
builder.AddAttribute(sequence++, "AllowInput", true);
builder.CloseComponent();
}
// List
else if (value is IList list)
{
var elementType = list.GetType().GetGenericArguments().Single();
builder.OpenElement(sequence++, "div");
builder.AddAttribute(sequence++, "class", "form-generator-list");
builder.OpenElement(sequence++, "div");
builder.AddAttribute(sequence++, "class", "form-generator-header");
builder.OpenElement(sequence++, "span");
builder.AddAttribute(sequence++, "class", "form-generator-title");
builder.AddContent(sequence++, $"{property.Name} ({list.Count})");
builder.CloseComponent();
builder.OpenElement(sequence++, "span");
builder.AddAttribute(sequence++, "class", "form-generator-add-button");
builder.AddAttribute(sequence++, "onclick",
EventCallback.Factory.Create <MouseEventArgs>(this, value =>
{
var element = this.Instantiate(elementType);
list.Add(element);
this.OnChanged();
}));
builder.OpenElement(sequence++, "i");
builder.AddAttribute(sequence++, "class", "fas fa-plus");
builder.CloseElement();
builder.CloseElement();
builder.CloseElement();
builder.OpenElement(sequence++, "div");
builder.AddAttribute(sequence++, "class", "form-generator-content");
if (list.Count > 0)
{
var index = 0;
foreach (var item in list)
{
builder.OpenElement(sequence++, "div");
builder.AddAttribute(sequence++, "class", "form-generator-list-item");
if (elementType.IsPrimitive || elementType.IsEnum || elementType == typeof(string) || elementType == typeof(decimal))
{
var localIndex = index;
Action <object> valueChanged = newValue =>
{
list[localIndex] = newValue;
this.OnChanged();
};
var onKeyDown = EventCallback.Factory.Create <KeyboardEventArgs>(this, e =>
{
if (e.Key == "Delete")
{
list.RemoveAt(localIndex);
this.OnChanged();
}
});
sequence = (int)NexusUtilities.InvokeGenericMethod(
this.GetType(),
this,
nameof(AddGenericPrimitive),
BindingFlags.NonPublic | BindingFlags.Instance,
elementType,
new object[] { builder, sequence, "Value", item, valueChanged, onKeyDown });
}
else
{
builder.OpenComponent(sequence++, typeof(FormGenerator));
builder.AddAttribute(sequence++, "DataContext", item);
builder.AddAttribute(sequence++, "Changed",
EventCallback.Factory.Create(this, value => this.OnChanged()));
builder.CloseComponent();
var localIndex = index;
builder.OpenElement(sequence++, "span");
builder.AddAttribute(sequence++, "class", "form-generator-remove-button");
builder.AddAttribute(sequence++, "onclick",
EventCallback.Factory.Create <MouseEventArgs>(this, e =>
{
var element = this.Instantiate(elementType);
list.RemoveAt(localIndex);
this.OnChanged();
}));
builder.OpenElement(sequence++, "i");
builder.AddAttribute(sequence++, "class", "fas fa-trash-alt");
builder.CloseElement();
builder.CloseElement();
}
builder.CloseElement();
index++;
}
}
builder.CloseElement();
builder.CloseElement();
}
// Dictionary
else if (value is IDictionary dict)
{
var elementTypes = dict.GetType().GetGenericArguments();
if (!(elementTypes[0].IsPrimitive || elementTypes[0].IsEnum || elementTypes[0] == typeof(string) || elementTypes[0] == typeof(decimal)))
{
throw new Exception("Only primitive keys are supported.");
}
builder.OpenElement(sequence++, "div");
builder.AddAttribute(sequence++, "class", "form-generator-dict");
builder.OpenElement(sequence++, "div");
builder.AddAttribute(sequence++, "class", "form-generator-header");
builder.OpenElement(sequence++, "span");
builder.AddAttribute(sequence++, "class", "form-generator-title");
builder.AddContent(sequence++, $"{property.Name} ({dict.Count})");
builder.CloseComponent();
builder.OpenElement(sequence++, "span");
builder.AddAttribute(sequence++, "class", "form-generator-add-button");
builder.AddAttribute(sequence++, "onclick",
EventCallback.Factory.Create <MouseEventArgs>(this, value =>
{
var dictKey = this.Instantiate(elementTypes[0]);
var dictValue = this.Instantiate(elementTypes[1]);
dict[dictKey] = dictValue;
this.OnChanged();
}));
builder.OpenElement(sequence++, "i");
builder.AddAttribute(sequence++, "class", "fas fa-plus");
builder.CloseElement();
builder.CloseElement();
builder.CloseElement();
builder.OpenElement(sequence++, "div");
builder.AddAttribute(sequence++, "class", "form-generator-content");
if (dict.Count > 0)
{
foreach (var key in dict.Keys)
{
builder.OpenElement(sequence++, "div");
builder.AddAttribute(sequence++, "class", "form-generator-dict-entry");
// key
Action <object> valueChanged = newKey =>
{
dict.Remove(key);
dict[newKey] = this.Instantiate(elementTypes[1]);
this.OnChanged();
};
var onKeyDown = EventCallback.Factory.Create <KeyboardEventArgs>(this, e =>
{
if (e.Key == "Delete")
{
dict.Remove(key);
this.OnChanged();
}
});
sequence = (int)NexusUtilities.InvokeGenericMethod(
this.GetType(),
this,
nameof(AddGenericPrimitive),
BindingFlags.NonPublic | BindingFlags.Instance,
elementTypes[0],
new object[] { builder, sequence, "Key", key, valueChanged, onKeyDown });
// space
builder.OpenElement(sequence++, "span");
builder.AddContent(sequence++, " ");
builder.CloseElement();
// value
if (elementTypes[1].IsPrimitive || elementTypes[1].IsEnum || elementTypes[1] == typeof(string) || elementTypes[1] == typeof(decimal))
{
Action <object> valueChanged2 = newValue =>
{
dict[key] = newValue;
this.OnChanged();
};
sequence = (int)NexusUtilities.InvokeGenericMethod(
this.GetType(),
this,
nameof(AddGenericPrimitive),
BindingFlags.NonPublic | BindingFlags.Instance,
elementTypes[1],
new object[] { builder, sequence, "Value", dict[key], valueChanged2, null });
}
else
{
builder.OpenComponent(sequence++, typeof(FormGenerator));
builder.AddAttribute(sequence++, "DataContext", dict[key]);
builder.AddAttribute(sequence++, "Changed",
EventCallback.Factory.Create(this, value => this.OnChanged()));
builder.CloseComponent();
}
builder.CloseElement();
}
}
builder.CloseElement();
builder.CloseElement();
}
builder.CloseElement();
}
builder.CloseComponent();
};
private IEnumerable <DataReaderProgressRecord> InternalRead(
List <DatasetInfo> datasets,
DateTime begin,
DateTime end,
ulong blockSizeLimit,
TimeSpan basePeriod,
TimeSpan fundamentalPeriod,
CancellationToken cancellationToken)
{
/*
* |....................|
* |
* |
* |....................
* |
* |
* |....................
* |
* |====================
* |....................
* |
* |
* |....................|
*
* | = base period (1 minute)
* ... = fundamental period (e.g. 10 minutes)
* |...| = begin & end markers
* === = block period
*/
if (cancellationToken.IsCancellationRequested)
{
yield break;
}
if (!datasets.Any() || begin == end)
{
yield break;
}
// calculation
var minutesPerFP = fundamentalPeriod.Ticks / basePeriod.Ticks;
var bytesPerFP = datasets.Sum(dataset =>
{
var bytesPerSample = NexusUtilities.SizeOf(dataset.DataType);
var samplesPerMinute = dataset.GetSampleRate().SamplesPerSecond * 60;
var bytesPerFP = bytesPerSample * samplesPerMinute * minutesPerFP;
return(bytesPerFP);
});
var FPCountPerBlock = blockSizeLimit / bytesPerFP;
var roundedFPCount = (long)Math.Floor(FPCountPerBlock);
if (roundedFPCount < 1)
{
throw new Exception("The block size limit is too small.");
}
var maxPeriodPerRequest = TimeSpan.FromTicks(fundamentalPeriod.Ticks * roundedFPCount);
// load data
var period = end - begin;
var currentBegin = begin;
var remainingPeriod = end - currentBegin;
while (remainingPeriod > TimeSpan.Zero)
{
var datasetToRecordMap = new Dictionary <DatasetInfo, DataRecord>();
var currentPeriod = TimeSpan.FromTicks(Math.Min(remainingPeriod.Ticks, maxPeriodPerRequest.Ticks));
var currentEnd = currentBegin + currentPeriod;
var index = 1;
var count = datasets.Count;
foreach (var dataset in datasets)
{
if (cancellationToken.IsCancellationRequested)
{
yield break;
}
(var data, var status) = this.ReadSingle(dataset, currentBegin, currentEnd);
datasetToRecordMap[dataset] = new DataRecord(data, status);
// update progress
var localProgress = TimeSpan.FromTicks(currentPeriod.Ticks * index / count);
var currentProgress = (currentBegin + localProgress - begin).Ticks / (double)period.Ticks;
((IProgress <double>) this.Progress).Report(currentProgress);
index++;
}
// notify about new data
yield return(new DataReaderProgressRecord(datasetToRecordMap, currentBegin, currentEnd));
// continue in time
currentBegin += currentPeriod;
remainingPeriod = end - currentBegin;
}
}
private void OpenFile(string dataFilePath, DateTime startDateTime, List <ChannelContextGroup> channelContextGroupSet)
{
if (File.Exists(dataFilePath))
{
throw new Exception($"The file {dataFilePath} already exists. Extending an already existing file with additional channels is not supported.");
}
var famosFile = new FamosFileHeader();
// file
var metadataGroup = new FamosFileGroup("Metadata");
metadataGroup.PropertyInfo = new FamosFilePropertyInfo(new List <FamosFileProperty>()
{
new FamosFileProperty("format_version", this.FormatVersion),
new FamosFileProperty("system_name", this.DataWriterContext.SystemName),
new FamosFileProperty("date_time", startDateTime),
});
foreach (var customMetadataEntry in this.DataWriterContext.CustomMetadataEntrySet.Where(customMetadataEntry => customMetadataEntry.CustomMetadataEntryLevel == CustomMetadataEntryLevel.File))
{
metadataGroup.PropertyInfo.Properties.Add(new FamosFileProperty(customMetadataEntry.Key, customMetadataEntry.Value));
}
famosFile.Groups.Add(metadataGroup);
// file -> project
var projectGroup = new FamosFileGroup($"{this.DataWriterContext.ProjectDescription.PrimaryGroupName} / {this.DataWriterContext.ProjectDescription.SecondaryGroupName} / {this.DataWriterContext.ProjectDescription.ProjectName}");
projectGroup.PropertyInfo = new FamosFilePropertyInfo(new List <FamosFileProperty>()
{
new FamosFileProperty("project_version", this.DataWriterContext.ProjectDescription.Version)
});
foreach (var customMetadataEntry in this.DataWriterContext.CustomMetadataEntrySet.Where(customMetadataEntry => customMetadataEntry.CustomMetadataEntryLevel == CustomMetadataEntryLevel.Project))
{
projectGroup.PropertyInfo.Properties.Add(new FamosFileProperty(customMetadataEntry.Key, customMetadataEntry.Value));
}
famosFile.Groups.Add(projectGroup);
// for each context group
foreach (var contextGroup in channelContextGroupSet)
{
var totalSeconds = (int)Math.Round(_settings.FilePeriod.TotalSeconds, MidpointRounding.AwayFromZero);
var totalLength = (int)(totalSeconds * contextGroup.SampleRate.SamplesPerSecond);
if (totalLength * (double)NexusUtilities.SizeOf(NexusDataType.FLOAT64) > 2 * Math.Pow(10, 9))
{
throw new Exception(ErrorMessage.FamosWriter_DataSizeExceedsLimit);
}
// file -> project -> channels
var field = new FamosFileField(FamosFileFieldType.MultipleYToSingleEquidistantTime);
foreach (ChannelContext channelContext in contextGroup.ChannelContextSet)
{
var dx = contextGroup.SampleRate.Period.TotalSeconds;
var channel = this.PrepareChannel(field, channelContext.ChannelDescription, (int)totalLength, startDateTime, dx);
projectGroup.Channels.Add(channel);
}
famosFile.Fields.Add(field);
_spdToFieldIndexMap[contextGroup.SampleRate.SamplesPerDay] = famosFile.Fields.Count - 1;
}
//
famosFile.Save(dataFilePath, _ => { });
_famosFile = FamosFile.OpenEditable(dataFilePath);
}