public async Task <long> GetCountOfEntities(Guid id, CollectedDataType collectedDataType, DateTime timeFrom, DateTime timeTo)
{
var count =
await _repository.GetCountOfEntities(id, collectedDataType, timeFrom, timeTo);
return(count);
}
public async Task <IEnumerable <CollectedDataDto> > GetEntitiesByTypeAsync(CollectedDataType collectedDataType)
{
var entities = await _repository.GetAllEntitiesByType(collectedDataType);
if (entities == null)
{
throw new NotFoundException();
}
return(_mapper.Map <IEnumerable <CollectedData>, IEnumerable <CollectedDataDto> >(entities));
}
public async Task <IEnumerable <CollectedDataDto> > GetSourceEntitiesAsync(CollectedDataType sourceType,
DateTime timeFrom, DateTime timeTo)
{
var entities = await _dataAggregatorRepository.GetEntitiesByTypeInTime(sourceType, timeFrom, timeTo);
if (entities == null)
{
throw new NotFoundException();
}
return(_mapper.Map <IEnumerable <CollectedData>, IEnumerable <CollectedDataDto> >(entities));
}
public Task <long> CountByParametersAsync(Guid id, CollectedDataType type, DateTime @from, DateTime to)
{
var query = _context.AnomalyReportsCollection.Find(d => d.ClientId == id &&
d.CollectedDataTypeOfReport == type &&
d.Date >= @from &&
d.Date <= to);
if (query != null)
{
return(query.CountAsync());
}
return(Task.FromResult((long)0));
}
public async Task <IEnumerable <CollectedData> > GetAllEntitiesByType(CollectedDataType collectedDataType)
{
try
{
return(await _context.Datasets
.Find(data => data.CollectedDataType == collectedDataType)
.ToListAsync());
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
}
// Query by type in time
public async Task <IEnumerable <CollectedData> > GetEntitiesByTypeInTime(CollectedDataType collectedDataType,
DateTime timeFrom, DateTime timeTo)
{
try
{
var query = _context.Datasets
.Find(d => d.CollectedDataType == collectedDataType && d.Time >= timeFrom && d.Time <= timeTo);
return(await query.ToListAsync());
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
}
public async Task <long> GetCountOfEntities(Guid id, CollectedDataType collectedDataType,
DateTime timeFrom, DateTime timeTo)
{
try
{
var query = _context.Datasets
.Find(d => d.ClientId == id && d.CollectedDataType == collectedDataType && d.Time >= timeFrom && d.Time <= timeTo);
return(await query.CountAsync());
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
}
public virtual async Task <ActionResult <IEnumerable <CollectedDataDto> > > GetDataByInstance(Guid id, CollectedDataType type, DateTime from, DateTime to)
{
var dtos = await _aggregateDataService.GetAggregatedDataInTime(id, type, from, to, 1, int.MaxValue);
return(Ok(dtos));
}
public async Task <IEnumerable <CollectedDataDto> > GetAggregatedDataInTime(Guid id, CollectedDataType collectedDataType, DateTime timeFrom, DateTime timeTo,
int page = 1, int count = 10)
{
var entities =
await _repository.GetEntitiesByInstanceIdAndTypeInTime(id, collectedDataType, timeFrom, timeTo, page, count);
var collectedDtos = _mapper.Map <IEnumerable <CollectedData>, IEnumerable <CollectedDataDto> >(entities);
return(collectedDtos);
}
public async Task <List <CollectedDataDto> > GetCollectedDataByInstanceId(Guid id, CollectedDataType dataType)
{
var entities = await _repository.GetCollectedDataByInstanceIdAsync(id, dataType);
if (entities == null)
{
return(null);
}
var dtos = _mapper.Map <IEnumerable <CollectedData>, List <CollectedDataDto> >(entities);
return(dtos);
}
public virtual async Task <ActionResult <int> > GetCount(Guid id, CollectedDataType type, DateTime from, DateTime to)
{
var dtos = await _service.GetCountOfReportsAsync(id, type, from, to);
return(Ok(dtos));
}
public Task <List <InstanceAnomalyReport> > GetReportsByParametersAsync(Guid instanceId, CollectedDataType type, DateTime @from, DateTime to, int page = 1, int count = 10)
{
var query = _context.AnomalyReportsCollection.Find(d => d.ClientId == instanceId &&
d.CollectedDataTypeOfReport == type &&
d.Date >= @from &&
d.Date <= to);
query.SortByDescending(report => report.Date).Skip(count * (page - 1)).Limit(count);
return(query.ToListAsync());
}
public virtual async Task <ActionResult <CollectedDataDto> > GetCollectedDataByInstanceId([FromRoute] Guid id, [FromQuery] CollectedDataType dataType)
{
var dtos = await _collectedDataService.GetCollectedDataByInstanceId(id, dataType);
if (dtos == null)
{
return(NoContent());
}
return(Ok(dtos));
}
public Task <List <CollectedData> > GetCollectedDataByInstanceIdAsync(Guid instanceId, CollectedDataType dataType)
{
try
{
return(_context.Datasets
.Find(data => data.CollectedDataType == dataType && // It will get all data for last hour(Aggregation) or for last day(AggregationForHour)
data.Time > DateTime.UtcNow.AddHours(-1) &&
data.ClientId == instanceId)
.SortByDescending(cd => cd.Time)
.Limit(400) // max get 400 documents(60 * 6 = 360) - for hour
.ToListAsync());
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
}
public virtual async Task <ActionResult <IEnumerable <CollectedDataDto> > > GetDataByInstancePagination(Guid id, CollectedDataType type, DateTime from, DateTime to,
int page, int count)
{
var dtos = await _aggregateDataService.GetAggregatedDataInTime(id, type, from, to, page, count);
return(Ok(dtos));
}
public virtual async Task <ActionResult <IEnumerable <InstanceAnomalyReport> > > GetDataByInstance(Guid id,
CollectedDataType type,
DateTime from,
DateTime to)
{
var dtos = await _service.GetReportsInTimeAsync(id, type, from, to, 1, int.MaxValue);
return(Ok(dtos));
}
public async Task <List <InstanceAnomalyReportDto> > GetReportsInTimeAsync(Guid id, CollectedDataType type, DateTime @from, DateTime to, int page, int count)
{
var reports = await _repository.GetReportsByParametersAsync(id, type, from, to, page, count);
if (reports != null && reports.Count > 0)
{
return(_mapper.Map <List <InstanceAnomalyReport>, List <InstanceAnomalyReportDto> >(reports));
}
return(new List <InstanceAnomalyReportDto>());
}
public virtual async Task <ActionResult <IEnumerable <InstanceAnomalyReport> > > GetDataByInstancePagination(Guid id,
CollectedDataType type,
DateTime from,
DateTime to,
int page,
int count)
{
var dtos = await _service.GetReportsInTimeAsync(id, type, from, to, page, count);
return(Ok(dtos));
}
// Query by instance and type in time
public async Task <IEnumerable <CollectedData> > GetEntitiesByInstanceIdAndTypeInTime(Guid id, CollectedDataType collectedDataType,
DateTime timeFrom, DateTime timeTo, int page = 1, int count = 10)
{
try
{
var query = _context.Datasets
.Find(d => d.ClientId == id && d.CollectedDataType == collectedDataType && d.Time >= timeFrom && d.Time <= timeTo);
query.Skip(count * (page - 1)).Limit(count);
return(await query.ToListAsync());
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
}
public Task <long> GetCountOfReportsAsync(Guid id, CollectedDataType type, DateTime @from, DateTime to)
{
return(_repository.CountByParametersAsync(id, type, @from, to));
}
private Task SendMLReport(IEnumerable <CollectedDataDto> data, CollectedDataType reportType)
{
var results = (from d in data
group d by d.ClientId into g
select new { ClientId = g.Key, CollectedDatas = g.ToList() }).ToList();
var tasks = results.Select(
async r =>
{
try
{
var result = await _anomalyDetector.AnalyzeData(r.CollectedDatas);
result.CollectedDataTypeOfReport = reportType;
var report = new InstanceAnomalyReport
{
Id = Guid.NewGuid(),
ClientId = r.ClientId,
Date = result.Date,
AnomalyGroups = result.AnomalyGroups,
CollectedDataTypeOfReport = result.CollectedDataTypeOfReport
};
await _reportsRepository.AddReportAsync(report);
var reportMessage =
new InstanceAnomalyReportMessage
{
AnomalyReportId = report.Id,
InstanceId = r.ClientId
};
Debug.WriteLine($"Generated id of report {report.Id}");
await _serviceBusProvider.SendAnomalyReportMessage(reportMessage);
}
catch (Exception ex)
{
_logger.LogError(ex, $"Unhandled error occurred while analyzing Anomalies of instance with id: {r?.ClientId}");
// TODO: Maybe send notification to user about that error that analyzing Anomalies of instance with id was unsuccessful
}
});
return(Task.WhenAll(tasks));
//var res = Parallel.ForEach(
// results,
// async obj =>
// {
// try
// {
// var result = await _anomalyDetector.AnalyzeData(obj.CollectedDatas);
// result.CollectedDataTypeOfReport = reportType;
// var report = new InstanceAnomalyReport
// {
// Id = Guid.NewGuid(),
// ClientId = obj.ClientId,
// Date = result.Date,
// AnomalyGroups = result.AnomalyGroups,
// CollectedDataTypeOfReport =
// result.CollectedDataTypeOfReport
// };
// await _reportsRepository.AddReportAsync(report);
// var reportMessage =
// new InstanceAnomalyReportMessage { AnomalyReportId = report.Id, InstanceId = obj.ClientId };
// Debug.WriteLine($"Generated id of report {report.Id}");
// await _serviceBusProvider.SendAnomalyReportMessage(reportMessage);
// }
// catch (Exception ex)
// {
// _logger.LogError(ex, $"Unhandled error occurred while analyzing Anomalies of instance with id: {obj?.ClientId}");
// // TODO: Maybe send notification to user about that error that analyzing Anomalies of instance with id was unsuccessful
// }
// });
}
public virtual async Task <ActionResult <long> > GetCount(Guid id, CollectedDataType type, DateTime from, DateTime to)
{
var dtos = await _aggregateDataService.GetCountOfEntities(id, type, from, to);
return(Ok(dtos));
}
private async Task <IEnumerable <CollectedDataDto> > FilterCollectedDataByInstanceSettings(List <CollectedDataDto> collectedDataDtos,
CollectedDataType destinationType)
{
var instanceSettingsDtos = await _aggregatorService.GetInstanceSettingsEntitiesAsync();
List <InstanceSettingsDto> instanceSettingsList = instanceSettingsDtos.ToList();
List <CollectedDataDto> filteredCollectedDataDtos = new List <CollectedDataDto>();
switch (destinationType)
{
case CollectedDataType.AggregationForHour:
{
instanceSettingsList.ForEach(instanceSettings =>
{
if (instanceSettings.IsActive && instanceSettings.AggregationForHour)
{
var data = collectedDataDtos.Where(d => d.ClientId == instanceSettings.ClientId).ToList();
filteredCollectedDataDtos.AddRange(data);
}
});
break;
}
case CollectedDataType.AggregationForDay:
{
instanceSettingsList.ForEach(instanceSettings =>
{
if (instanceSettings.IsActive && instanceSettings.AggregationForDay)
{
var data = collectedDataDtos.Where(d => d.ClientId == instanceSettings.ClientId).ToList();
filteredCollectedDataDtos.AddRange(data);
}
});
break;
}
case CollectedDataType.AggregationForWeek:
{
instanceSettingsList.ForEach(instanceSettings =>
{
if (instanceSettings.IsActive && instanceSettings.AggregationForWeek)
{
var data = collectedDataDtos.Where(d => d.ClientId == instanceSettings.ClientId).ToList();
filteredCollectedDataDtos.AddRange(data);
}
});
break;
}
case CollectedDataType.AggregationForMonth:
{
instanceSettingsList.ForEach(instanceSettings =>
{
if (instanceSettings.IsActive && instanceSettings.AggregationForMonth)
{
var data = collectedDataDtos.Where(d => d.ClientId == instanceSettings.ClientId).ToList();
filteredCollectedDataDtos.AddRange(data);
}
});
break;
}
}
return(filteredCollectedDataDtos);
}
public async Task AggregatingData(CollectedDataType sourceType, CollectedDataType destinationType, TimeSpan interval, bool deleteSource)
{
// Need destinationType,
// Subtract interval from the current time
var timeFrom = DateTime.UtcNow.Add(-interval);
var timeTo = DateTime.UtcNow;
var sourceCollectedDataDtos = (await _aggregatorService.GetSourceEntitiesAsync(sourceType, timeFrom, timeTo)).ToList();
var filteredCollectedDataDtos = await FilterCollectedDataByInstanceSettings(sourceCollectedDataDtos, destinationType);
var collectedDataDtos = filteredCollectedDataDtos as CollectedDataDto[] ?? filteredCollectedDataDtos.ToArray();
if (collectedDataDtos.Any())
{
var collectedDataDtosAverage =
from collectedDataDto in collectedDataDtos
group collectedDataDto by collectedDataDto.ClientId
into collectedDataGroup
select new CollectedDataDto
{
ClientId = collectedDataGroup.Key,
CollectedDataType = destinationType,
ProcessesCount = Convert.ToInt32(collectedDataGroup
.Average(d => d.ProcessesCount)),
Processes = collectedDataGroup
.SelectMany(d => d.Processes)
.GroupBy(p => p.Name)
.Select(g => new ProcessData()
{
Name = g.Key,
PCpu = g.Average(c => c.PCpu),
PCpuMax = g.Max(c => c.PCpu),
PCpuMin = g.Min(c => c.PCpu),
PRam = g.Average(r => r.PRam),
PRamMax = g.Max(r => r.PRam),
PRamMin = g.Min(r => r.PRam),
RamMBytes = g.Average(b => b.RamMBytes),
RamMBytesMax = g.Max(b => b.RamMBytes),
RamMBytesMin = g.Min(b => b.RamMBytes)
}).ToList(),
UsageRamMBytes = collectedDataGroup
.Average(d => d.UsageRamMBytes),
TotalRamMBytes = collectedDataGroup
.Average(d => d.TotalRamMBytes),
RamUsagePercentage = collectedDataGroup
.Average(d => d.RamUsagePercentage),
InterruptsPerSeconds = collectedDataGroup
.Average(d => d.InterruptsPerSeconds),
LocalDiskUsageMBytes = collectedDataGroup
.Average(d => d.LocalDiskUsageMBytes),
LocalDiskTotalMBytes = collectedDataGroup
.Average(d => d.LocalDiskTotalMBytes),
LocalDiskUsagePercentage = collectedDataGroup
.Average(d => d.LocalDiskUsagePercentage),
CpuUsagePercentage = collectedDataGroup
.Average(d => d.CpuUsagePercentage),
Time = collectedDataGroup
.Max(d => d.Time)
};
var tasks = new List <Task> {
SendMLReport(collectedDataDtos, destinationType),
_aggregatorService.AddAggregatorEntitiesAsync(collectedDataDtosAverage)
};
// await SendMLReport(collectedDataDtos, destinationType);
// Save aggregated CollectedDataDto to destination table MongoDb
// await _aggregatorService.AddAggregatorEntitiesAsync(collectedDataDtosAverage);
if (deleteSource)
{
// Delete already aggregated CollectedDataDto from source table MongoDb
// await _aggregatorService.DeleteManyAggregatedEntitiesAsync(collectedDataDtos.Select(dto => dto.Id));
tasks.Add(_aggregatorService.DeleteManyAggregatedEntitiesAsync(collectedDataDtos.Select(dto => dto.Id)));
}
await Task.WhenAll(tasks);
}
}
public DataAccumulatorRepository(string ConnectionString, string Database, CollectedDataType collectedDataType)
{
_context = new DataAccumulatorContext(ConnectionString, Database);
_collectedDataType = collectedDataType;
}