private async Task DequeueReturnsSingleItem()
{
var expected = new DataSourceQueue
{
DataSource = new DataSource
{
Name = "ds1",
Priority = DataSourcePriority.NotTrusted,
AllowedOperations = DataSourceAllowedOperation.Alter,
},
StartImportDate = DateTime.MinValue,
Status = DataSourceQueueStatuses.InQueue,
};
DataSourceQueue actual;
using (var ctx = CreateDbContext())
{
ctx.Add(expected);
await ctx.SaveChangesAsync();
actual = await new QueueService(ctx).Dequeue();
}
Assert.NotNull(actual);
Assert.Equal(DataSourceQueueStatuses.Loading, actual.Status);
Assert.NotNull(actual.StartImportDate);
Assert.Equal(actual.StartImportDate.Value.FlushSeconds(), DateTime.Now.FlushSeconds());
}
private async Task LogStatUnitUplaodTest(DataUploadingLogStatuses status)
{
var unit = new LegalUnit {
StatId = "123", Name = "name42"
};
var started = DateTime.Now;
var ended = DateTime.Now;
DataUploadingLog actual;
using (var ctx = CreateDbContext())
{
var queueItem = new DataSourceQueue();
ctx.DataSourceQueues.Add(queueItem);
await ctx.SaveChangesAsync();
await new QueueService(ctx).LogUnitUpload(
queueItem,
JsonConvert.SerializeObject(unit),
started,
unit,
ended,
status,
string.Empty,
null,
null);
actual = queueItem.DataUploadingLogs.FirstOrDefault();
}
Assert.NotNull(actual);
Assert.Equal(started, actual.StartImportDate);
Assert.Equal(ended, actual.EndImportDate);
Assert.Equal(status, actual.Status);
}
private async Task DequeuedItemIncludesDataSourceEntity()
{
var expected = new DataSourceQueue
{
DataSource = new DataSource
{
Name = "ds1",
Priority = DataSourcePriority.NotTrusted,
AllowedOperations = DataSourceAllowedOperation.Alter,
},
StartImportDate = DateTime.MinValue,
Status = DataSourceQueueStatuses.InQueue,
};
DataSourceQueue actual;
using (var ctx = CreateDbContext())
{
ctx.Add(expected);
await ctx.SaveChangesAsync();
actual = await new QueueService(ctx).Dequeue();
}
Assert.NotNull(actual.DataSource);
Assert.Equal(expected.DataSource.Name, actual.DataSource.Name);
}
private async Task DequeueReturnsNullIfNoItemsWithInQueueStatus()
{
var expected = new DataSourceQueue
{
DataSource = new DataSource
{
Name = "ds1",
Priority = DataSourcePriority.NotTrusted,
AllowedOperations = DataSourceAllowedOperation.Alter,
},
StartImportDate = DateTime.MinValue,
Status = DataSourceQueueStatuses.DataLoadCompleted,
};
DataSourceQueue actual;
using (var ctx = CreateDbContext())
{
ctx.Add(expected);
await ctx.SaveChangesAsync();
actual = await new QueueService(ctx).Dequeue();
}
Assert.Null(actual);
}
/// <summary>
///
/// </summary>
/// <param name="queueItem"></param>
/// <param name="parsedUnit"></param>
/// <returns></returns>
private async Task <(string, StatisticalUnit)> PopulateUnit(
DataSourceQueue queueItem,
IReadOnlyDictionary <string, object> parsedUnit)
{
StatisticalUnit unit;
try
{
unit = await _queueSvc.GetStatUnitFromRawEntity(
parsedUnit,
queueItem.DataSource.StatUnitType,
queueItem.DataSource.VariablesMappingArray,
queueItem.DataSource.DataSourceUploadType,
queueItem.DataSource.AllowedOperations);
}
catch (Exception ex)
{
var data = ex.Data.Keys
.Cast <string>()
.Where(key => key != "unit")
.Select(key => $"`{key}` = `{ex.Data[key]}`");
return(
$"message: {ex.Message}, data: {string.Join(", ", data)}",
ex.Data["unit"] as StatisticalUnit);
}
unit.DataSource = queueItem.DataSourceFileName;
unit.ChangeReason = ChangeReasons.Edit;
unit.EditComment = "Uploaded from data source file";
return(null, unit);
}
private QueueVm(DataSourceQueue item)
{
Id = item.Id;
FileName = item.DataSourceFileName;
DataSourceTemplateName = item.DataSource.Name;
UploadDateTime = item.StartImportDate;
UserName = item.User.Name;
Status = (int)item.Status;
}
public DataSourceQueueModel ToDataSourceQueueModel(DataSourceQueue dataSourceQueue)
{
return(new DataSourceQueueModel
{
Id = dataSourceQueue.Id.ToString(),
DatasourceId = dataSourceQueue.DatasourceId.ToString(),
Status = dataSourceQueue.Status
});
}
private async Task FinishQueueItemTest(DataSourceQueueStatuses expectedStatus)
{
var actual = new DataSourceQueue();
using (var ctx = CreateDbContext())
{
ctx.DataSourceQueues.Add(actual);
await ctx.SaveChangesAsync();
await new QueueService(ctx).FinishQueueItem(actual, expectedStatus);
}
Assert.Equal(expectedStatus, actual.Status);
}
/// <summary> /// Method of creating a view of the queue model /// </summary> /// <param name="item">item</param> /// <returns></returns> public static QueueVm Create(DataSourceQueue item) => new QueueVm(item);
private (string, (IReadOnlyDictionary <string, string[]>, string[])) AnalyzeUnit(IStatisticalUnit unit, DataSourceQueue queueItem)
{
if (queueItem.DataSource.DataSourceUploadType != DataSourceUploadTypes.StatUnits)
{
return(null, (new Dictionary <string, string[]>(), new string[0]));
}
AnalysisResult analysisResult;
try
{
analysisResult = _analysisSvc.AnalyzeStatUnit(unit);
}
catch (Exception ex)
{
return(ex.Message, (null, null));
}
return(null, (
analysisResult.Messages,
analysisResult.SummaryMessages?.ToArray() ?? Array.Empty <string>()));
}
private static async Task <(string error, IReadOnlyDictionary <string, object>[] result)> ParseFile(DataSourceQueue queueItem)
{
IEnumerable <IReadOnlyDictionary <string, object> > parsed;
try
{
switch (queueItem.DataSourceFileName)
{
case var name when name.EndsWith(".xml", StringComparison.OrdinalIgnoreCase):
parsed = FileParser.GetRawEntitiesFromXml(queueItem.DataSourcePath);
break;
case var name when name.EndsWith(".csv", StringComparison.OrdinalIgnoreCase):
parsed = await FileParser.GetRawEntitiesFromCsv(
queueItem.DataSourcePath,
queueItem.DataSource.CsvSkipCount,
queueItem.DataSource.CsvDelimiter);
break;
default: return("Unsupported type of file", null);
}
}
catch (Exception ex)
{
return(ex.Message, null);
}
var parsedArr = parsed.ToArray();
if (parsedArr.Length == 0)
{
return(Resource.UploadFileEmpty, parsedArr);
}
if (parsedArr.Any(x => x.Count == 0))
{
return(Resource.FileHasEmptyUnit, parsedArr);
}
return(null, parsedArr);
}
public static void AddDataSources(NSCRegDbContext context)
{
var ds1 = new DataSource
{
Name = "data source #1",
Description = "data source #1 detailed description",
Priority = DataSourcePriority.Ok,
StatUnitType = StatUnitTypes.LegalUnit,
Restrictions = null,
VariablesMapping = "id-RegId,name-Name",
AttributesToCheck = "id,name,something",
AllowedOperations = DataSourceAllowedOperation.CreateAndAlter,
CsvDelimiter = ",",
CsvSkipCount = 0,
};
var ds2 = new DataSource
{
Name = "data source #2",
Description = "data source #2 detailed description",
Priority = DataSourcePriority.Trusted,
StatUnitType = StatUnitTypes.LocalUnit,
Restrictions = null,
VariablesMapping = "id-RegId,whatever-Name",
AttributesToCheck = "id,salary,whatever",
AllowedOperations = DataSourceAllowedOperation.Create,
CsvDelimiter = ",",
CsvSkipCount = 0,
};
context.DataSources.AddRange(ds1, ds2);
context.SaveChanges();
var dsq1 = new DataSourceQueue
{
DataSource = ds1,
DataSourceFileName = "qwe.xml",
DataSourcePath = ".\\",
DataUploadingLogs = new[]
{
new DataUploadingLog
{
StartImportDate = DateTime.Now,
EndImportDate = DateTime.Now,
StatUnitName = "qwe",
SerializedUnit =
JsonConvert.SerializeObject(new LegalUnit {
Name = "42", DataSource = "qwe.xml"
}),
Status = DataUploadingLogStatuses.Warning,
Errors = JsonConvert.SerializeObject(
new Dictionary <string, string[]> {
[nameof(LegalUnit.UnitStatusId)] = new[] { "err1", "err2" }
}),
}
},
EndImportDate = DateTime.Now,
StartImportDate = DateTime.Now,
User = context.Users.FirstOrDefault(),
Status = DataSourceQueueStatuses.DataLoadCompletedPartially,
};
context.DataSourceQueues.Add(dsq1);
context.SaveChanges();
}
