/// <inheritdoc cref="IChildImportEntitiesGetter"/>
public virtual IEnumerable <Entity> Get(ImportParameters parameters)
{
List <Entity> entities = new List <Entity>();
foreach (ImportEntity importEntity in parameters.Entities)
{
if (importEntity.PrimaryEntity == null)
{
continue;
}
SchemasKeyColumnsValues schemasKeyColumnsValues =
GetSchemasKeyColumnsValues(parameters, importEntity);
AddChildEntitiesQueryFilters(parameters, importEntity, schemasKeyColumnsValues);
foreach (EntitySchemaQuery esq in ChildEntitiesQueries.Values)
{
if (esq.Filters.Count >= FileImporterConstants.MaxQueryChildFiltersCount)
{
EntityCollection entityCollection = esq.GetEntityCollection(UserConnection);
entities.AddRange(entityCollection);
esq.ResetSelectQuery();
esq.Filters.Clear();
}
}
}
foreach (EntitySchemaQuery esq in ChildEntitiesQueries.Values)
{
if (esq.Filters.Any())
{
EntityCollection entityCollection = esq.GetEntityCollection(UserConnection);
entities.AddRange(entityCollection);
}
}
return(entities);
}
private void InitEntityForSave(ImportParameters parameters, ImportEntity importEntity)
{
importEntity.InitPrimaryEntity(UserConnection, parameters);
foreach (var column in parameters.Columns)
{
var columnValue = importEntity.FindColumnValue(column);
if (columnValue == null)
{
continue;
}
foreach (var destination in column.Destinations)
{
var entity = importEntity.GetEntityForSave(UserConnection, destination);
var columnValueName = destination.ColumnValueName;
var valueForSave = ColumnsProcessor.FindValueForSave(destination, columnValue);
if (valueForSave == null)
{
continue;
}
if (entity.StoringState != StoringObjectState.New)
{
var entityValue = entity.GetColumnValue(columnValueName);
if (valueForSave.Equals(entityValue) || destination.IsKey)
{
continue;
}
}
if (entity.Schema.Columns.GetByName(destination.ColumnName).DataValueType is TextDataValueType)
{
valueForSave = valueForSave.ToString().Trim();
}
entity.SetColumnValue(columnValueName, valueForSave);
}
}
}
private void UpdateImportResults(ImportParameters parameters)
{
parameters.ImportedRowsCount = _successProcessedRowsCount;
parameters.NotImportedRowsCount = _processedWithErrorRowsCount + parameters.NotImportedRowsCount;
parameters.ProcessedRowsCount = parameters.ImportedRowsCount + parameters.NotImportedRowsCount;
ImportParametersRepository.Update(parameters);
}
private void CreateChunkFromFileProcessor(ImportParameters importParameters)
{
var fileProcessor = ClassFactory.Get <ISaxFileProcessor>(
new ConstructorArgument("userConnection", UserConnection));
var importSessionId = importParameters.ImportSessionId;
var chunkSize = importParameters.ChunkSize;
var importEntities = fileProcessor.ReadEntities(importSessionId);
var chunk = new List <ImportEntity>(chunkSize);
var counter = 0;
foreach (var importEntity in importEntities)
{
chunk.Add(importEntity);
counter++;
importParameters.TotalRowsCount++;
if (counter < chunkSize)
{
continue;
}
Add(CreateEntityDataChunk(importSessionId, chunk));
chunk = new List <ImportEntity>(chunkSize);
counter = 0;
}
if (chunk.Any())
{
Add(CreateEntityDataChunk(importSessionId, chunk));
}
}
private void ProcessChunk(IFileImportEntitiesChunkProcessor chunkProcessor, ImportParameters importParameters, ImportDataChunk <EntityChunkData> chunk)
{
EventHandler <ImportEntitySavedEventArgs> successImportEntitySavedHadler = (o, arg) => {
chunk.Data.SuccessProcessedEntityCount++;
_importEntitiesChunksDataProvider.SaveProcessedRow(chunk.ChunkId, (int)arg.RowIndex);
};
chunkProcessor.ImportEntitySaved += successImportEntitySavedHadler;
EventHandler <ImportEntitySaveErrorEventArgs> importEntitySavedWithErrorHandler = (o, arg) => {
chunk.Data.ProcessedWithErrorEntityCount++;
_importEntitiesChunksDataProvider.SaveProcessedRow(chunk.ChunkId, (int)arg.ImportEntity.RowIndex, false);
};
chunkProcessor.ImportEntitySaveError += importEntitySavedWithErrorHandler;
ImportParameters newImportParameters = PrepareImportParameters(importParameters, chunk);
SaveProcessedChunk(chunk, ImportChunkState.InProcess);
chunkProcessor.ProcessChunk(newImportParameters);
SaveProcessedChunk(chunk, ImportChunkState.Processed);
_importEntitiesChunksDataProvider.RemoveProcessedRows(chunk.ChunkId);
chunkProcessor.ImportEntitySaved -= successImportEntitySavedHadler;
chunkProcessor.ImportEntitySaveError -= importEntitySavedWithErrorHandler;
}
/// <inheritdoc />
public void Process(ImportParameters importParameters)
{
var lookupChunksData = _importDataProvider.Get(importParameters.ImportSessionId);
if (!lookupChunksData.Any())
{
lookupChunksData = GetLookupChunkData(importParameters);
SaveLookupChunksData(lookupChunksData);
}
var handler = new ChunkLookupValuesHandler(UserConnection);
handler.ProcessError += SendProcessError;
try {
foreach (var item in lookupChunksData.Where(l => l.State == ImportChunkState.ToProcess))
{
if (_importParametersRepository.GetIsImportSessionCanceled(importParameters.ImportSessionId))
{
break;
}
handler.Execute(item);
}
} finally {
handler.ProcessError -= SendProcessError;
}
_lookupProcessedValues = new LookupProcessedValues(UserConnection);
_lookupProcessedValues.RestoreState(JoinLookupProcessedValues(lookupChunksData.Select(c => c.Data.ProcessedValuesState).ToList()));
}
/// <inheritdoc cref="BaseFileImportStage"/>
protected override FileImportStagesEnum?InternalProcess(ImportParameters parameters)
{
ColumnsProcessor.ProcessError += ImportLogger.HandleError;
ColumnsProcessor.Process(parameters);
ColumnsProcessor.ProcessError -= ImportLogger.HandleError;
return(FileImportStagesEnum.ProcessEntitiesFileImportStage);
}
/// <inheritdoc />
public override void CreateDataChunks(ImportParameters importParameters)
{
if (importParameters.AnyLookupColumns())
{
CreateLookupChunks(importParameters);
}
}
private void SaveImportEntities(ImportParameters parameters)
{
Guid sessionId = parameters.ImportSessionId;
var entities = parameters.Entities;
ImportEntity importEntity;
while ((importEntity = entities.FirstOrDefault()) != null)
{
if (importEntity.ImportEntityException != null)
{
OnImportEntitySaveError(importEntity.ImportEntityException, importEntity, sessionId);
entities.Remove(importEntity);
continue;
}
try {
InitEntityForSave(parameters, importEntity);
importEntity.Save(SaveInBackgroundMode);
OnImportEntitySaved(importEntity.PrimaryEntity, parameters, importEntity.RowIndex);
} catch (OutOfMemoryException) {
throw;
} catch (Exception e) {
OnImportEntitySaveError(e, importEntity, sessionId);
} finally {
entities.Remove(importEntity);
}
}
}
private void SetImportTags(ImportParameters parameters)
{
if (parameters.ImportTags.IsNullOrEmpty())
{
parameters.ImportTags = CreateTags();
}
}
/// <summary>
/// Gets key columns values.
/// </summary>
/// <param name="parameters">Import parameters.</param>
/// <param name="importEntity">Import entity.</param>
/// <param name="keyColumns">Key columns.</param>
/// <returns>Key columns values.</returns>
private List <ColumnExpression> GetKeyColumnsValues(ImportParameters parameters, ImportEntity importEntity,
IEnumerable <ImportColumn> keyColumns)
{
var columnsExpressions = new List <ColumnExpression>();
foreach (var column in keyColumns)
{
var columnValue = importEntity.FindColumnValue(column);
if (columnValue == null)
{
continue;
}
foreach (var destination in column.Destinations)
{
if (!destination.SchemaUId.Equals(parameters.RootSchemaUId) || !destination.IsKey)
{
continue;
}
var valueForSave = ColumnsProcessor.FindValueForSave(destination, columnValue);
if (valueForSave == null)
{
continue;
}
columnsExpressions.Add(new ColumnExpression {
ColumnValueName = destination.ColumnName,
ComparisonType = FilterComparisonType.Equal,
ColumnValue = valueForSave,
DataValueTypeUId = Guid.Parse(destination.Properties[_typeUIdPropertyName].ToString())
});
}
}
return(columnsExpressions);
}
/// <summary>
/// <inheritdoc cref="IPersistentFileImporter"/>
/// </summary>
public void Import(Guid importSessionId, CancellationToken token)
{
ImportParameters parameters = GetImportParametersWithStage(importSessionId);
parameters.ImportCancellationToken = token;
Import(parameters);
}
private void InitializationOfAuxiliaryData(ImportParameters parameters)
{
SetImportTags(parameters);
SetChunkSize(parameters);
SetCurrentUser(parameters);
SaveImportParameters(parameters);
}
private void CreateDataChunks(ImportParameters parameters)
{
var chunkDataProvider = ClassFactory.Get <IImportEntitiesChunksDataProvider>(
new ConstructorArgument("userConnection", UserConnection));
chunkDataProvider.CreateDataChunks(parameters);
}
private IEnumerable <ImportDataChunk <LookupChunkData> > GetLookupChunkData(ImportParameters importParameters)
{
var importColumns = importParameters.Columns.Where(c => c.Destinations.Any(d => d.FindColumnTypeUId() == DataValueTypeUId));
var entitiesChunks = importParameters.Entities.SplitOnChunks(importParameters.ChunkSize);
var result = new List <ImportDataChunk <LookupChunkData> >();
if (!importColumns.Any())
{
return(result);
}
foreach (var entities in entitiesChunks)
{
var processor = new LookupValuesToProcess(UserConnection);
foreach (var entity in entities)
{
foreach (var column in importColumns)
{
var columnValue = entity.FindColumnValue(column);
if (columnValue == null)
{
continue;
}
foreach (var destination in column.Destinations)
{
processor.Add(entity, column, columnValue, destination);
}
}
}
result.Add(GetImportDataChunk(importParameters.ImportSessionId,
processor.SaveState()));
}
return(result);
}
private void CreateImportLogger(ImportParameters parameters)
{
var userConnectionArg = new ConstructorArgument("userConnection", UserConnection);
var importParametersArg = new ConstructorArgument("importParameters", parameters);
ImportLogger = ClassFactory.Get <IImportLogger>(userConnectionArg, importParametersArg);
}
private void CreateReminding(ImportParameters parameters)
{
Guid contactId = parameters.AuthorId;
DateTime dateTime = TimeZoneInfo.ConvertTimeFromUtc(new DateTimeProvider().UtcNow, parameters.AuthorTimeZone);
uint notImportedRowsCount = parameters.NotImportedRowsCount;
string description = string.Format(CompleteRemindingDescriptionTemplate, parameters.ImportedRowsCount,
parameters.TotalRowsCount, parameters.FileName);
if (notImportedRowsCount > 0)
{
description += string.Format(NotImportedRowsCountMessageTemplate, notImportedRowsCount);
}
string caption = string.Format("{0} {1}", CompleteRemindingSubject, description);
ISchemaManagerItem <EntitySchema> importSessionItem =
UserConnection.EntitySchemaManager.GetItemByName("ImportSession");
ISchemaManagerItem <EntitySchema> sysProcessLogItem =
UserConnection.EntitySchemaManager.GetItemByName("VwSysProcessLog");
EntitySchema remindingSchema = UserConnection.EntitySchemaManager.GetInstanceByName("Reminding");
Entity reminding = remindingSchema.CreateEntity(UserConnection);
reminding.SetDefColumnValues();
reminding.SetColumnValue("AuthorId", contactId);
reminding.SetColumnValue("ContactId", contactId);
reminding.SetColumnValue("SourceId", RemindingConsts.RemindingSourceAuthorId);
reminding.SetColumnValue("RemindTime", dateTime);
reminding.SetColumnValue("Description", description);
reminding.SetColumnValue("SubjectId", parameters.ImportSessionId);
reminding.SetColumnValue("SysEntitySchemaId", sysProcessLogItem.UId);
reminding.SetColumnValue("SubjectCaption", caption);
reminding.SetColumnValue("LoaderId", importSessionItem.UId);
reminding.Save();
}
/// <inheritdoc cref="BaseFileImportStage"/>
protected override FileImportStagesEnum?InternalProcess(ImportParameters parameters)
{
InitializationOfAuxiliaryData(parameters);
ValidateColumnsMapping(parameters);
MergeImportEntities(parameters);
CreateDataChunks(parameters);
return(FileImportStagesEnum.ProcessColumnsFileImportStage);
}
/// <inheritdoc cref="IBaseFileImportStage"/>
public IBaseFileImportStage ProcessStage(ImportParameters parameters)
{
CreateImportLogger(parameters);
var nextStageId = InternalProcess(parameters);
ImportLogger.SaveLog();
return(nextStageId.HasValue ? CreateNextStage(nextStageId.Value) : null);
}
private void SendNotificationMessage(ImportParameters parameters)
{
var importNotifier = ClassFactory.Get <IImportNotifier>(
new ConstructorArgument("userConnection", UserConnection),
new ConstructorArgument("importParameters", parameters));
importNotifier.NotifyEnd();
}
private void DeleteImportOperationalData(ImportParameters parameters)
{
parameters.Entities = null;
ImportParametersRepository.Update(parameters);
ImportParametersRepository.DeleteFile(parameters.ImportSessionId);
ImportEntitiesChunksDataProvider.Delete(parameters.ImportSessionId);
ImportLookupChunksDataProvider.Delete(parameters.ImportSessionId);
}
private void InitImportEntities(ImportParameters parameters)
{
var keyColumns = GetKeyColumns(parameters);
PrimaryEntityFinder.LoadPrimaryEntity(parameters, keyColumns);
var childEntities = ChildImportEntitiesGetter.Get(parameters);
ChildImportEntitiesSetter.Set(parameters, childEntities);
}
private void SetCurrentUser(ImportParameters parameters)
{
if (parameters.AuthorId == default(Guid))
{
var currentUser = UserConnection.CurrentUser;
parameters.AuthorTimeZone = currentUser.TimeZone;
parameters.AuthorId = currentUser.ContactId;
}
}
private void CheckIsImportCancelled(ImportParameters parameters)
{
if (parameters.GetIsImportCancelled(UserConnection))
{
WriteFileImportLog(FileImportLogMessageType.CancelFileImport, parameters.ImportSessionId);
DeleteImportParameters(parameters.ImportSessionId);
throw new OperationCanceledException();
}
}
///<inheritdoc cref="IFileImportEntitiesChunkProcessor"/>
public void ProcessChunk(ImportParameters importParameters)
{
if (GetIsImportSessionCanceled(importParameters.ImportSessionId))
{
return;
}
InitImportEntities(importParameters);
SaveImportEntities(importParameters);
}
///<inheritdoc cref="IFileImportEntitiesChunkProcessor"/>
public void ProcessChunk(ImportParameters importParameters)
{
if (importParameters.GetIsImportCancelled(UserConnection))
{
return;
}
InitImportEntities(importParameters);
SaveImportEntities(importParameters);
}
/// <summary>
/// Gets schemas key columns values.
/// </summary>
/// <param name="parameters">Import parameters.</param>
/// <param name="importEntity">Import entity.</param>
/// <returns>Schemas key columns values.</returns>
private SchemasKeyColumnsValues GetSchemasKeyColumnsValues(
ImportParameters parameters, ImportEntity importEntity)
{
var schemasKeyColumnsValues = new SchemasKeyColumnsValues();
foreach (ImportColumn column in parameters.Columns)
{
ImportColumnValue columnValue = importEntity.FindColumnValue(column);
if (columnValue == null)
{
continue;
}
foreach (ImportColumnDestination destination in column.Destinations)
{
Guid schemaUId = destination.SchemaUId;
if (schemaUId.Equals(parameters.RootSchemaUId))
{
continue;
}
object valueForSave = ColumnsProcessor.FindValueForSave(destination, columnValue);
if (valueForSave == null)
{
continue;
}
EntitySchema schema = UserConnection.EntitySchemaManager.GetInstanceByUId(schemaUId);
EntitiesKeyColumnsValues entitiesKeyColumnsValues;
if (!schemasKeyColumnsValues.TryGetValue(schema, out entitiesKeyColumnsValues))
{
entitiesKeyColumnsValues = new EntitiesKeyColumnsValues();
schemasKeyColumnsValues.Add(schema, entitiesKeyColumnsValues);
}
int destinationIndex = destination.GetIndex();
object attributeColumnValue = destination.FindAttributeColumnValue();
string destinationKey = string.Concat(destinationIndex, attributeColumnValue);
KeyColumnsValues keyColumnsValues;
if (!entitiesKeyColumnsValues.TryGetValue(destinationKey, out keyColumnsValues))
{
keyColumnsValues = new KeyColumnsValues();
entitiesKeyColumnsValues.Add(destinationKey, keyColumnsValues);
}
keyColumnsValues.Add(destination.ColumnName, valueForSave);
string attributeColumnName = GetDestinationAttributeColumnName(destination, schema);
if (!keyColumnsValues.ContainsKey(attributeColumnName))
{
keyColumnsValues.Add(attributeColumnName, attributeColumnValue);
}
EntitySchemaColumn connectionColumn = importEntity.GetReferenceColumn(schema);
string connectionColumnName = connectionColumn.Name;
if (!keyColumnsValues.ContainsKey(connectionColumnName))
{
Guid primaryColumnValue = importEntity.PrimaryEntity.PrimaryColumnValue;
keyColumnsValues.Add(connectionColumnName, primaryColumnValue);
}
}
}
return(schemasKeyColumnsValues);
}
private void InitImportNotifier(ImportParameters parameters)
{
if ((_isNeedNotify = !parameters.NeedSendNotify))
{
return;
}
_importNotifier = ClassFactory.Get <IImportNotifier>(GetUserConnectionArgument(),
new ConstructorArgument("importParameters", parameters));
}
/// <summary>
/// Sets existing entities to import entities.
/// </summary>
/// <param name="parameters">Import parameters.</param>
/// <param name="entities">Existing entities.</param>
public void Set(ImportParameters parameters, IEnumerable <Entity> entities)
{
parameters.CheckArgumentNull(nameof(parameters));
if (entities.Any())
{
ImportParameters = parameters;
ProcessEntities(entities);
Logger.SaveLog();
}
}
/// <summary>
/// Creates instance of type <see cref="ImportNotifier"/>.
/// </summary>
/// <param name="userConnection">User connection.</param>
/// <param name="importParameters">Import parameters.</param>
public ImportNotifier(UserConnection userConnection, ImportParameters importParameters)
{
UserConnection = userConnection;
ImportParameters = importParameters;
_channel = MsgChannelManager.Instance.FindItemByUId(UserConnection.CurrentUser.Id);
if (_channel != null)
{
_channel.OnMessage += ImportMsgHandler;
}
}
