private async Task DequeueReturnsSingleItem() { var expected = new DataSourceQueue { DataSource = new DataSource { Name = "ds1", Priority = DataSourcePriority.NotTrusted, AllowedOperations = DataSourceAllowedOperation.Alter, }, StartImportDate = DateTime.MinValue, Status = DataSourceQueueStatuses.InQueue, }; DataSourceQueue actual; using (var ctx = CreateDbContext()) { ctx.Add(expected); await ctx.SaveChangesAsync(); actual = await new QueueService(ctx).Dequeue(); } Assert.NotNull(actual); Assert.Equal(DataSourceQueueStatuses.Loading, actual.Status); Assert.NotNull(actual.StartImportDate); Assert.Equal(actual.StartImportDate.Value.FlushSeconds(), DateTime.Now.FlushSeconds()); }
private async Task LogStatUnitUplaodTest(DataUploadingLogStatuses status) { var unit = new LegalUnit { StatId = "123", Name = "name42" }; var started = DateTime.Now; var ended = DateTime.Now; DataUploadingLog actual; using (var ctx = CreateDbContext()) { var queueItem = new DataSourceQueue(); ctx.DataSourceQueues.Add(queueItem); await ctx.SaveChangesAsync(); await new QueueService(ctx).LogUnitUpload( queueItem, JsonConvert.SerializeObject(unit), started, unit, ended, status, string.Empty, null, null); actual = queueItem.DataUploadingLogs.FirstOrDefault(); } Assert.NotNull(actual); Assert.Equal(started, actual.StartImportDate); Assert.Equal(ended, actual.EndImportDate); Assert.Equal(status, actual.Status); }
private async Task DequeuedItemIncludesDataSourceEntity() { var expected = new DataSourceQueue { DataSource = new DataSource { Name = "ds1", Priority = DataSourcePriority.NotTrusted, AllowedOperations = DataSourceAllowedOperation.Alter, }, StartImportDate = DateTime.MinValue, Status = DataSourceQueueStatuses.InQueue, }; DataSourceQueue actual; using (var ctx = CreateDbContext()) { ctx.Add(expected); await ctx.SaveChangesAsync(); actual = await new QueueService(ctx).Dequeue(); } Assert.NotNull(actual.DataSource); Assert.Equal(expected.DataSource.Name, actual.DataSource.Name); }
private async Task DequeueReturnsNullIfNoItemsWithInQueueStatus() { var expected = new DataSourceQueue { DataSource = new DataSource { Name = "ds1", Priority = DataSourcePriority.NotTrusted, AllowedOperations = DataSourceAllowedOperation.Alter, }, StartImportDate = DateTime.MinValue, Status = DataSourceQueueStatuses.DataLoadCompleted, }; DataSourceQueue actual; using (var ctx = CreateDbContext()) { ctx.Add(expected); await ctx.SaveChangesAsync(); actual = await new QueueService(ctx).Dequeue(); } Assert.Null(actual); }
/// <summary> /// /// </summary> /// <param name="queueItem"></param> /// <param name="parsedUnit"></param> /// <returns></returns> private async Task <(string, StatisticalUnit)> PopulateUnit( DataSourceQueue queueItem, IReadOnlyDictionary <string, object> parsedUnit) { StatisticalUnit unit; try { unit = await _queueSvc.GetStatUnitFromRawEntity( parsedUnit, queueItem.DataSource.StatUnitType, queueItem.DataSource.VariablesMappingArray, queueItem.DataSource.DataSourceUploadType, queueItem.DataSource.AllowedOperations); } catch (Exception ex) { var data = ex.Data.Keys .Cast <string>() .Where(key => key != "unit") .Select(key => $"`{key}` = `{ex.Data[key]}`"); return( $"message: {ex.Message}, data: {string.Join(", ", data)}", ex.Data["unit"] as StatisticalUnit); } unit.DataSource = queueItem.DataSourceFileName; unit.ChangeReason = ChangeReasons.Edit; unit.EditComment = "Uploaded from data source file"; return(null, unit); }
private QueueVm(DataSourceQueue item) { Id = item.Id; FileName = item.DataSourceFileName; DataSourceTemplateName = item.DataSource.Name; UploadDateTime = item.StartImportDate; UserName = item.User.Name; Status = (int)item.Status; }
public DataSourceQueueModel ToDataSourceQueueModel(DataSourceQueue dataSourceQueue) { return(new DataSourceQueueModel { Id = dataSourceQueue.Id.ToString(), DatasourceId = dataSourceQueue.DatasourceId.ToString(), Status = dataSourceQueue.Status }); }
private async Task FinishQueueItemTest(DataSourceQueueStatuses expectedStatus) { var actual = new DataSourceQueue(); using (var ctx = CreateDbContext()) { ctx.DataSourceQueues.Add(actual); await ctx.SaveChangesAsync(); await new QueueService(ctx).FinishQueueItem(actual, expectedStatus); } Assert.Equal(expectedStatus, actual.Status); }
/// <summary> /// Method of creating a view of the queue model /// </summary> /// <param name="item">item</param> /// <returns></returns> public static QueueVm Create(DataSourceQueue item) => new QueueVm(item);
private (string, (IReadOnlyDictionary <string, string[]>, string[])) AnalyzeUnit(IStatisticalUnit unit, DataSourceQueue queueItem) { if (queueItem.DataSource.DataSourceUploadType != DataSourceUploadTypes.StatUnits) { return(null, (new Dictionary <string, string[]>(), new string[0])); } AnalysisResult analysisResult; try { analysisResult = _analysisSvc.AnalyzeStatUnit(unit); } catch (Exception ex) { return(ex.Message, (null, null)); } return(null, ( analysisResult.Messages, analysisResult.SummaryMessages?.ToArray() ?? Array.Empty <string>())); }
private static async Task <(string error, IReadOnlyDictionary <string, object>[] result)> ParseFile(DataSourceQueue queueItem) { IEnumerable <IReadOnlyDictionary <string, object> > parsed; try { switch (queueItem.DataSourceFileName) { case var name when name.EndsWith(".xml", StringComparison.OrdinalIgnoreCase): parsed = FileParser.GetRawEntitiesFromXml(queueItem.DataSourcePath); break; case var name when name.EndsWith(".csv", StringComparison.OrdinalIgnoreCase): parsed = await FileParser.GetRawEntitiesFromCsv( queueItem.DataSourcePath, queueItem.DataSource.CsvSkipCount, queueItem.DataSource.CsvDelimiter); break; default: return("Unsupported type of file", null); } } catch (Exception ex) { return(ex.Message, null); } var parsedArr = parsed.ToArray(); if (parsedArr.Length == 0) { return(Resource.UploadFileEmpty, parsedArr); } if (parsedArr.Any(x => x.Count == 0)) { return(Resource.FileHasEmptyUnit, parsedArr); } return(null, parsedArr); }
public static void AddDataSources(NSCRegDbContext context) { var ds1 = new DataSource { Name = "data source #1", Description = "data source #1 detailed description", Priority = DataSourcePriority.Ok, StatUnitType = StatUnitTypes.LegalUnit, Restrictions = null, VariablesMapping = "id-RegId,name-Name", AttributesToCheck = "id,name,something", AllowedOperations = DataSourceAllowedOperation.CreateAndAlter, CsvDelimiter = ",", CsvSkipCount = 0, }; var ds2 = new DataSource { Name = "data source #2", Description = "data source #2 detailed description", Priority = DataSourcePriority.Trusted, StatUnitType = StatUnitTypes.LocalUnit, Restrictions = null, VariablesMapping = "id-RegId,whatever-Name", AttributesToCheck = "id,salary,whatever", AllowedOperations = DataSourceAllowedOperation.Create, CsvDelimiter = ",", CsvSkipCount = 0, }; context.DataSources.AddRange(ds1, ds2); context.SaveChanges(); var dsq1 = new DataSourceQueue { DataSource = ds1, DataSourceFileName = "qwe.xml", DataSourcePath = ".\\", DataUploadingLogs = new[] { new DataUploadingLog { StartImportDate = DateTime.Now, EndImportDate = DateTime.Now, StatUnitName = "qwe", SerializedUnit = JsonConvert.SerializeObject(new LegalUnit { Name = "42", DataSource = "qwe.xml" }), Status = DataUploadingLogStatuses.Warning, Errors = JsonConvert.SerializeObject( new Dictionary <string, string[]> { [nameof(LegalUnit.UnitStatusId)] = new[] { "err1", "err2" } }), } }, EndImportDate = DateTime.Now, StartImportDate = DateTime.Now, User = context.Users.FirstOrDefault(), Status = DataSourceQueueStatuses.DataLoadCompletedPartially, }; context.DataSourceQueues.Add(dsq1); context.SaveChanges(); }