/// <inheritdoc cref="IChildImportEntitiesGetter"/> public virtual IEnumerable <Entity> Get(ImportParameters parameters) { List <Entity> entities = new List <Entity>(); foreach (ImportEntity importEntity in parameters.Entities) { if (importEntity.PrimaryEntity == null) { continue; } SchemasKeyColumnsValues schemasKeyColumnsValues = GetSchemasKeyColumnsValues(parameters, importEntity); AddChildEntitiesQueryFilters(parameters, importEntity, schemasKeyColumnsValues); foreach (EntitySchemaQuery esq in ChildEntitiesQueries.Values) { if (esq.Filters.Count >= FileImporterConstants.MaxQueryChildFiltersCount) { EntityCollection entityCollection = esq.GetEntityCollection(UserConnection); entities.AddRange(entityCollection); esq.ResetSelectQuery(); esq.Filters.Clear(); } } } foreach (EntitySchemaQuery esq in ChildEntitiesQueries.Values) { if (esq.Filters.Any()) { EntityCollection entityCollection = esq.GetEntityCollection(UserConnection); entities.AddRange(entityCollection); } } return(entities); }
private void InitEntityForSave(ImportParameters parameters, ImportEntity importEntity) { importEntity.InitPrimaryEntity(UserConnection, parameters); foreach (var column in parameters.Columns) { var columnValue = importEntity.FindColumnValue(column); if (columnValue == null) { continue; } foreach (var destination in column.Destinations) { var entity = importEntity.GetEntityForSave(UserConnection, destination); var columnValueName = destination.ColumnValueName; var valueForSave = ColumnsProcessor.FindValueForSave(destination, columnValue); if (valueForSave == null) { continue; } if (entity.StoringState != StoringObjectState.New) { var entityValue = entity.GetColumnValue(columnValueName); if (valueForSave.Equals(entityValue) || destination.IsKey) { continue; } } if (entity.Schema.Columns.GetByName(destination.ColumnName).DataValueType is TextDataValueType) { valueForSave = valueForSave.ToString().Trim(); } entity.SetColumnValue(columnValueName, valueForSave); } } }
private void UpdateImportResults(ImportParameters parameters) { parameters.ImportedRowsCount = _successProcessedRowsCount; parameters.NotImportedRowsCount = _processedWithErrorRowsCount + parameters.NotImportedRowsCount; parameters.ProcessedRowsCount = parameters.ImportedRowsCount + parameters.NotImportedRowsCount; ImportParametersRepository.Update(parameters); }
private void CreateChunkFromFileProcessor(ImportParameters importParameters) { var fileProcessor = ClassFactory.Get <ISaxFileProcessor>( new ConstructorArgument("userConnection", UserConnection)); var importSessionId = importParameters.ImportSessionId; var chunkSize = importParameters.ChunkSize; var importEntities = fileProcessor.ReadEntities(importSessionId); var chunk = new List <ImportEntity>(chunkSize); var counter = 0; foreach (var importEntity in importEntities) { chunk.Add(importEntity); counter++; importParameters.TotalRowsCount++; if (counter < chunkSize) { continue; } Add(CreateEntityDataChunk(importSessionId, chunk)); chunk = new List <ImportEntity>(chunkSize); counter = 0; } if (chunk.Any()) { Add(CreateEntityDataChunk(importSessionId, chunk)); } }
private void ProcessChunk(IFileImportEntitiesChunkProcessor chunkProcessor, ImportParameters importParameters, ImportDataChunk <EntityChunkData> chunk) { EventHandler <ImportEntitySavedEventArgs> successImportEntitySavedHadler = (o, arg) => { chunk.Data.SuccessProcessedEntityCount++; _importEntitiesChunksDataProvider.SaveProcessedRow(chunk.ChunkId, (int)arg.RowIndex); }; chunkProcessor.ImportEntitySaved += successImportEntitySavedHadler; EventHandler <ImportEntitySaveErrorEventArgs> importEntitySavedWithErrorHandler = (o, arg) => { chunk.Data.ProcessedWithErrorEntityCount++; _importEntitiesChunksDataProvider.SaveProcessedRow(chunk.ChunkId, (int)arg.ImportEntity.RowIndex, false); }; chunkProcessor.ImportEntitySaveError += importEntitySavedWithErrorHandler; ImportParameters newImportParameters = PrepareImportParameters(importParameters, chunk); SaveProcessedChunk(chunk, ImportChunkState.InProcess); chunkProcessor.ProcessChunk(newImportParameters); SaveProcessedChunk(chunk, ImportChunkState.Processed); _importEntitiesChunksDataProvider.RemoveProcessedRows(chunk.ChunkId); chunkProcessor.ImportEntitySaved -= successImportEntitySavedHadler; chunkProcessor.ImportEntitySaveError -= importEntitySavedWithErrorHandler; }
/// <inheritdoc /> public void Process(ImportParameters importParameters) { var lookupChunksData = _importDataProvider.Get(importParameters.ImportSessionId); if (!lookupChunksData.Any()) { lookupChunksData = GetLookupChunkData(importParameters); SaveLookupChunksData(lookupChunksData); } var handler = new ChunkLookupValuesHandler(UserConnection); handler.ProcessError += SendProcessError; try { foreach (var item in lookupChunksData.Where(l => l.State == ImportChunkState.ToProcess)) { if (_importParametersRepository.GetIsImportSessionCanceled(importParameters.ImportSessionId)) { break; } handler.Execute(item); } } finally { handler.ProcessError -= SendProcessError; } _lookupProcessedValues = new LookupProcessedValues(UserConnection); _lookupProcessedValues.RestoreState(JoinLookupProcessedValues(lookupChunksData.Select(c => c.Data.ProcessedValuesState).ToList())); }
/// <inheritdoc cref="BaseFileImportStage"/> protected override FileImportStagesEnum?InternalProcess(ImportParameters parameters) { ColumnsProcessor.ProcessError += ImportLogger.HandleError; ColumnsProcessor.Process(parameters); ColumnsProcessor.ProcessError -= ImportLogger.HandleError; return(FileImportStagesEnum.ProcessEntitiesFileImportStage); }
/// <inheritdoc /> public override void CreateDataChunks(ImportParameters importParameters) { if (importParameters.AnyLookupColumns()) { CreateLookupChunks(importParameters); } }
private void SaveImportEntities(ImportParameters parameters) { Guid sessionId = parameters.ImportSessionId; var entities = parameters.Entities; ImportEntity importEntity; while ((importEntity = entities.FirstOrDefault()) != null) { if (importEntity.ImportEntityException != null) { OnImportEntitySaveError(importEntity.ImportEntityException, importEntity, sessionId); entities.Remove(importEntity); continue; } try { InitEntityForSave(parameters, importEntity); importEntity.Save(SaveInBackgroundMode); OnImportEntitySaved(importEntity.PrimaryEntity, parameters, importEntity.RowIndex); } catch (OutOfMemoryException) { throw; } catch (Exception e) { OnImportEntitySaveError(e, importEntity, sessionId); } finally { entities.Remove(importEntity); } } }
private void SetImportTags(ImportParameters parameters) { if (parameters.ImportTags.IsNullOrEmpty()) { parameters.ImportTags = CreateTags(); } }
/// <summary> /// Gets key columns values. /// </summary> /// <param name="parameters">Import parameters.</param> /// <param name="importEntity">Import entity.</param> /// <param name="keyColumns">Key columns.</param> /// <returns>Key columns values.</returns> private List <ColumnExpression> GetKeyColumnsValues(ImportParameters parameters, ImportEntity importEntity, IEnumerable <ImportColumn> keyColumns) { var columnsExpressions = new List <ColumnExpression>(); foreach (var column in keyColumns) { var columnValue = importEntity.FindColumnValue(column); if (columnValue == null) { continue; } foreach (var destination in column.Destinations) { if (!destination.SchemaUId.Equals(parameters.RootSchemaUId) || !destination.IsKey) { continue; } var valueForSave = ColumnsProcessor.FindValueForSave(destination, columnValue); if (valueForSave == null) { continue; } columnsExpressions.Add(new ColumnExpression { ColumnValueName = destination.ColumnName, ComparisonType = FilterComparisonType.Equal, ColumnValue = valueForSave, DataValueTypeUId = Guid.Parse(destination.Properties[_typeUIdPropertyName].ToString()) }); } } return(columnsExpressions); }
/// <summary> /// <inheritdoc cref="IPersistentFileImporter"/> /// </summary> public void Import(Guid importSessionId, CancellationToken token) { ImportParameters parameters = GetImportParametersWithStage(importSessionId); parameters.ImportCancellationToken = token; Import(parameters); }
private void InitializationOfAuxiliaryData(ImportParameters parameters) { SetImportTags(parameters); SetChunkSize(parameters); SetCurrentUser(parameters); SaveImportParameters(parameters); }
private void CreateDataChunks(ImportParameters parameters) { var chunkDataProvider = ClassFactory.Get <IImportEntitiesChunksDataProvider>( new ConstructorArgument("userConnection", UserConnection)); chunkDataProvider.CreateDataChunks(parameters); }
private IEnumerable <ImportDataChunk <LookupChunkData> > GetLookupChunkData(ImportParameters importParameters) { var importColumns = importParameters.Columns.Where(c => c.Destinations.Any(d => d.FindColumnTypeUId() == DataValueTypeUId)); var entitiesChunks = importParameters.Entities.SplitOnChunks(importParameters.ChunkSize); var result = new List <ImportDataChunk <LookupChunkData> >(); if (!importColumns.Any()) { return(result); } foreach (var entities in entitiesChunks) { var processor = new LookupValuesToProcess(UserConnection); foreach (var entity in entities) { foreach (var column in importColumns) { var columnValue = entity.FindColumnValue(column); if (columnValue == null) { continue; } foreach (var destination in column.Destinations) { processor.Add(entity, column, columnValue, destination); } } } result.Add(GetImportDataChunk(importParameters.ImportSessionId, processor.SaveState())); } return(result); }
private void CreateImportLogger(ImportParameters parameters) { var userConnectionArg = new ConstructorArgument("userConnection", UserConnection); var importParametersArg = new ConstructorArgument("importParameters", parameters); ImportLogger = ClassFactory.Get <IImportLogger>(userConnectionArg, importParametersArg); }
private void CreateReminding(ImportParameters parameters) { Guid contactId = parameters.AuthorId; DateTime dateTime = TimeZoneInfo.ConvertTimeFromUtc(new DateTimeProvider().UtcNow, parameters.AuthorTimeZone); uint notImportedRowsCount = parameters.NotImportedRowsCount; string description = string.Format(CompleteRemindingDescriptionTemplate, parameters.ImportedRowsCount, parameters.TotalRowsCount, parameters.FileName); if (notImportedRowsCount > 0) { description += string.Format(NotImportedRowsCountMessageTemplate, notImportedRowsCount); } string caption = string.Format("{0} {1}", CompleteRemindingSubject, description); ISchemaManagerItem <EntitySchema> importSessionItem = UserConnection.EntitySchemaManager.GetItemByName("ImportSession"); ISchemaManagerItem <EntitySchema> sysProcessLogItem = UserConnection.EntitySchemaManager.GetItemByName("VwSysProcessLog"); EntitySchema remindingSchema = UserConnection.EntitySchemaManager.GetInstanceByName("Reminding"); Entity reminding = remindingSchema.CreateEntity(UserConnection); reminding.SetDefColumnValues(); reminding.SetColumnValue("AuthorId", contactId); reminding.SetColumnValue("ContactId", contactId); reminding.SetColumnValue("SourceId", RemindingConsts.RemindingSourceAuthorId); reminding.SetColumnValue("RemindTime", dateTime); reminding.SetColumnValue("Description", description); reminding.SetColumnValue("SubjectId", parameters.ImportSessionId); reminding.SetColumnValue("SysEntitySchemaId", sysProcessLogItem.UId); reminding.SetColumnValue("SubjectCaption", caption); reminding.SetColumnValue("LoaderId", importSessionItem.UId); reminding.Save(); }
/// <inheritdoc cref="BaseFileImportStage"/> protected override FileImportStagesEnum?InternalProcess(ImportParameters parameters) { InitializationOfAuxiliaryData(parameters); ValidateColumnsMapping(parameters); MergeImportEntities(parameters); CreateDataChunks(parameters); return(FileImportStagesEnum.ProcessColumnsFileImportStage); }
/// <inheritdoc cref="IBaseFileImportStage"/> public IBaseFileImportStage ProcessStage(ImportParameters parameters) { CreateImportLogger(parameters); var nextStageId = InternalProcess(parameters); ImportLogger.SaveLog(); return(nextStageId.HasValue ? CreateNextStage(nextStageId.Value) : null); }
private void SendNotificationMessage(ImportParameters parameters) { var importNotifier = ClassFactory.Get <IImportNotifier>( new ConstructorArgument("userConnection", UserConnection), new ConstructorArgument("importParameters", parameters)); importNotifier.NotifyEnd(); }
private void DeleteImportOperationalData(ImportParameters parameters) { parameters.Entities = null; ImportParametersRepository.Update(parameters); ImportParametersRepository.DeleteFile(parameters.ImportSessionId); ImportEntitiesChunksDataProvider.Delete(parameters.ImportSessionId); ImportLookupChunksDataProvider.Delete(parameters.ImportSessionId); }
private void InitImportEntities(ImportParameters parameters) { var keyColumns = GetKeyColumns(parameters); PrimaryEntityFinder.LoadPrimaryEntity(parameters, keyColumns); var childEntities = ChildImportEntitiesGetter.Get(parameters); ChildImportEntitiesSetter.Set(parameters, childEntities); }
private void SetCurrentUser(ImportParameters parameters) { if (parameters.AuthorId == default(Guid)) { var currentUser = UserConnection.CurrentUser; parameters.AuthorTimeZone = currentUser.TimeZone; parameters.AuthorId = currentUser.ContactId; } }
private void CheckIsImportCancelled(ImportParameters parameters) { if (parameters.GetIsImportCancelled(UserConnection)) { WriteFileImportLog(FileImportLogMessageType.CancelFileImport, parameters.ImportSessionId); DeleteImportParameters(parameters.ImportSessionId); throw new OperationCanceledException(); } }
///<inheritdoc cref="IFileImportEntitiesChunkProcessor"/> public void ProcessChunk(ImportParameters importParameters) { if (GetIsImportSessionCanceled(importParameters.ImportSessionId)) { return; } InitImportEntities(importParameters); SaveImportEntities(importParameters); }
///<inheritdoc cref="IFileImportEntitiesChunkProcessor"/> public void ProcessChunk(ImportParameters importParameters) { if (importParameters.GetIsImportCancelled(UserConnection)) { return; } InitImportEntities(importParameters); SaveImportEntities(importParameters); }
/// <summary> /// Gets schemas key columns values. /// </summary> /// <param name="parameters">Import parameters.</param> /// <param name="importEntity">Import entity.</param> /// <returns>Schemas key columns values.</returns> private SchemasKeyColumnsValues GetSchemasKeyColumnsValues( ImportParameters parameters, ImportEntity importEntity) { var schemasKeyColumnsValues = new SchemasKeyColumnsValues(); foreach (ImportColumn column in parameters.Columns) { ImportColumnValue columnValue = importEntity.FindColumnValue(column); if (columnValue == null) { continue; } foreach (ImportColumnDestination destination in column.Destinations) { Guid schemaUId = destination.SchemaUId; if (schemaUId.Equals(parameters.RootSchemaUId)) { continue; } object valueForSave = ColumnsProcessor.FindValueForSave(destination, columnValue); if (valueForSave == null) { continue; } EntitySchema schema = UserConnection.EntitySchemaManager.GetInstanceByUId(schemaUId); EntitiesKeyColumnsValues entitiesKeyColumnsValues; if (!schemasKeyColumnsValues.TryGetValue(schema, out entitiesKeyColumnsValues)) { entitiesKeyColumnsValues = new EntitiesKeyColumnsValues(); schemasKeyColumnsValues.Add(schema, entitiesKeyColumnsValues); } int destinationIndex = destination.GetIndex(); object attributeColumnValue = destination.FindAttributeColumnValue(); string destinationKey = string.Concat(destinationIndex, attributeColumnValue); KeyColumnsValues keyColumnsValues; if (!entitiesKeyColumnsValues.TryGetValue(destinationKey, out keyColumnsValues)) { keyColumnsValues = new KeyColumnsValues(); entitiesKeyColumnsValues.Add(destinationKey, keyColumnsValues); } keyColumnsValues.Add(destination.ColumnName, valueForSave); string attributeColumnName = GetDestinationAttributeColumnName(destination, schema); if (!keyColumnsValues.ContainsKey(attributeColumnName)) { keyColumnsValues.Add(attributeColumnName, attributeColumnValue); } EntitySchemaColumn connectionColumn = importEntity.GetReferenceColumn(schema); string connectionColumnName = connectionColumn.Name; if (!keyColumnsValues.ContainsKey(connectionColumnName)) { Guid primaryColumnValue = importEntity.PrimaryEntity.PrimaryColumnValue; keyColumnsValues.Add(connectionColumnName, primaryColumnValue); } } } return(schemasKeyColumnsValues); }
private void InitImportNotifier(ImportParameters parameters) { if ((_isNeedNotify = !parameters.NeedSendNotify)) { return; } _importNotifier = ClassFactory.Get <IImportNotifier>(GetUserConnectionArgument(), new ConstructorArgument("importParameters", parameters)); }
/// <summary> /// Sets existing entities to import entities. /// </summary> /// <param name="parameters">Import parameters.</param> /// <param name="entities">Existing entities.</param> public void Set(ImportParameters parameters, IEnumerable <Entity> entities) { parameters.CheckArgumentNull(nameof(parameters)); if (entities.Any()) { ImportParameters = parameters; ProcessEntities(entities); Logger.SaveLog(); } }
/// <summary> /// Creates instance of type <see cref="ImportNotifier"/>. /// </summary> /// <param name="userConnection">User connection.</param> /// <param name="importParameters">Import parameters.</param> public ImportNotifier(UserConnection userConnection, ImportParameters importParameters) { UserConnection = userConnection; ImportParameters = importParameters; _channel = MsgChannelManager.Instance.FindItemByUId(UserConnection.CurrentUser.Id); if (_channel != null) { _channel.OnMessage += ImportMsgHandler; } }