/// <summary>Snippet for ImportDataAsync</summary>
/// <remarks>
/// This snippet has been automatically generated for illustrative purposes only.
/// It may require modifications to work in your environment.
/// </remarks>
public async Task ImportDataRequestObjectAsync()
{
// Create client
AutoMlClient autoMlClient = await AutoMlClient.CreateAsync();
// Initialize request argument(s)
ImportDataRequest request = new ImportDataRequest
{
DatasetName = DatasetName.FromProjectLocationDataset("[PROJECT]", "[LOCATION]", "[DATASET]"),
InputConfig = new InputConfig(),
};
// Make the request
Operation <Empty, OperationMetadata> response = await autoMlClient.ImportDataAsync(request);
// Poll until the returned long-running operation is complete
Operation <Empty, OperationMetadata> completedResponse = await response.PollUntilCompletedAsync();
// Retrieve the operation result
Empty result = completedResponse.Result;
// Or get the name of the operation
string operationName = response.Name;
// This name can be stored, then the long-running operation retrieved later by name
Operation <Empty, OperationMetadata> retrievedResponse = await autoMlClient.PollOnceImportDataAsync(operationName);
// Check if the retrieved long-running operation has completed
if (retrievedResponse.IsCompleted)
{
// If it has completed, then access the result
Empty retrievedResult = retrievedResponse.Result;
}
}
public async Task ImportBackgroundAsync(ImportDataRequest request, ImportPushNotification pushNotification, IJobCancellationToken jobCancellationToken, PerformContext context)
{
ValidateParameters(request, pushNotification);
try
{
var importer = _customerDataImporters.First(x => x.MemberType == request.MemberType);
await importer.ImportAsync(request,
progressInfo => ProgressCallback(progressInfo, pushNotification, context),
new JobCancellationTokenWrapper(jobCancellationToken));
}
catch (JobAbortedException)
{
// job is aborted, do nothing
}
catch (Exception ex)
{
pushNotification.Errors.Add(ex.ExpandExceptionMessage());
}
finally
{
pushNotification.Description = "Import finished";
pushNotification.Finished = DateTime.UtcNow;
await _pushNotificationManager.SendAsync(pushNotification);
}
}
/// <summary>Snippet for ImportDataAsync</summary>
public async Task ImportDataRequestObjectAsync()
{
// Snippet: ImportDataAsync(ImportDataRequest, CallSettings)
// Additional: ImportDataAsync(ImportDataRequest, CancellationToken)
// Create client
AutoMlClient autoMlClient = await AutoMlClient.CreateAsync();
// Initialize request argument(s)
ImportDataRequest request = new ImportDataRequest
{
Name = "",
InputConfig = new InputConfig(),
};
// Make the request
Operation <Empty, OperationMetadata> response = await autoMlClient.ImportDataAsync(request);
// Poll until the returned long-running operation is complete
Operation <Empty, OperationMetadata> completedResponse = await response.PollUntilCompletedAsync();
// Retrieve the operation result
Empty result = completedResponse.Result;
// Or get the name of the operation
string operationName = response.Name;
// This name can be stored, then the long-running operation retrieved later by name
Operation <Empty, OperationMetadata> retrievedResponse = await autoMlClient.PollOnceImportDataAsync(operationName);
// Check if the retrieved long-running operation has completed
if (retrievedResponse.IsCompleted)
{
// If it has completed, then access the result
Empty retrievedResult = retrievedResponse.Result;
}
// End snippet
}
private async Task <Price[]> GetAndPatchExistingPrices(ImportDataRequest request, ImportProductPrice[] importProductPrices)
{
var updateProductIds = importProductPrices.Select(x => x.ProductId).ToArray();
var existingPricesSearchResult = await _pricingSearchService.SearchPricesAsync(
new PricesSearchCriteria { ProductIds = updateProductIds, PriceListIds = new[] { request.PricelistId }, Take = int.MaxValue });
var existingPrices = existingPricesSearchResult
.Results
.Where(x => importProductPrices.Select(i => i.Price).Any(p => p.MinQuantity == x.MinQuantity && p.ProductId == x.ProductId))
.ToArray();
foreach (var existingPrice in existingPrices)
{
var priceForUpdating = importProductPrices
.FirstOrDefault(x =>
x.ProductId == existingPrice.ProductId &&
x.Price.MinQuantity == existingPrice.MinQuantity);
if (priceForUpdating != null)
{
existingPrice.Sale = priceForUpdating.Price.Sale;
existingPrice.List = priceForUpdating.Price.List;
}
}
return(existingPrices);
}
public async Task <ActionResult <ImportPushNotification> > RunImport([FromBody] ImportDataRequest request)
{
var notification = new ImportPushNotification(_userNameResolver.GetCurrentUserName())
{
Title = "Prices import",
Description = "Starting import task..."
};
await _pushNotificationManager.SendAsync(notification);
notification.JobId = BackgroundJob.Enqueue <ImportJob>(importJob => importJob.ImportBackgroundAsync(request, notification, JobCancellationToken.Null, null));
return(Ok(notification));
}
private void ValidateParameters(ImportDataRequest request, ImportPushNotification pushNotification)
{
if (pushNotification == null)
{
throw new ArgumentNullException(nameof(pushNotification));
}
var importer = _customerDataImporters.FirstOrDefault(x => x.MemberType == request.MemberType);
if (importer == null)
{
throw new ArgumentException($"Not allowed argument value in field {nameof(request.MemberType)}", nameof(request));
}
}
public ActionResult InsertImportData(string importViewModel)
{
ImportDataListViewModel viewModel = JsonConvert.DeserializeObject <ImportDataListViewModel>(importViewModel);
viewModel.AccountID = UserExtensions.ToAccountID(this.Identity);
viewModel.UserId = UserExtensions.ToUserID(this.Identity);
ImportDataRequest request = new ImportDataRequest()
{
ImportDataListViewModel = viewModel,
RequestedBy = viewModel.OwnerId
};
accountService.ImportData(request);
return(Json(new { success = true, response = "" }, JsonRequestBehavior.AllowGet));
}
private static void ValidateParameters(ImportDataRequest request, Action <ImportProgressInfo> progressCallback, ICancellationToken cancellationToken)
{
if (request == null)
{
throw new ArgumentNullException(nameof(request));
}
if (progressCallback == null)
{
throw new ArgumentNullException(nameof(progressCallback));
}
if (cancellationToken == null)
{
throw new ArgumentNullException(nameof(cancellationToken));
}
}
public virtual async Task ImportAsync(ImportDataRequest request, Action <ImportProgressInfo> progressCallback, ICancellationToken cancellationToken)
{
ValidateParameters(request, progressCallback, cancellationToken);
var errorsContext = new ImportErrorsContext();
var csvPriceDataValidationResult = await _dataValidator.ValidateAsync <TCsvMember>(request.FilePath);
if (csvPriceDataValidationResult.Errors.Any())
{
throw new InvalidDataException();
}
var configuration = new ImportConfiguration();
var reportFilePath = GetReportFilePath(request.FilePath);
await using var importReporter = await _importReporterFactory.CreateAsync(reportFilePath, configuration.Delimiter);
cancellationToken.ThrowIfCancellationRequested();
var importProgress = new ImportProgressInfo {
Description = "Import has started"
};
using var dataSource = await _dataSourceFactory.CreateAsync <TCsvMember, TMember>(request.FilePath, ModuleConstants.Settings.PageSize, configuration);
var headerRaw = dataSource.GetHeaderRaw();
if (!headerRaw.IsNullOrEmpty())
{
await importReporter.WriteHeaderAsync(headerRaw);
}
importProgress.TotalCount = dataSource.GetTotalCount();
progressCallback(importProgress);
const string importDescription = "{0} out of {1} have been imported.";
SetupErrorHandlers(progressCallback, configuration, errorsContext, importProgress, importReporter);
try
{
importProgress.Description = "Fetching...";
progressCallback(importProgress);
while (await dataSource.FetchAsync())
{
await ProcessChunkAsync(request, progressCallback, dataSource, errorsContext, importProgress, importReporter);
if (importProgress.ProcessedCount != importProgress.TotalCount)
{
importProgress.Description = string.Format(importDescription, importProgress.ProcessedCount, importProgress.TotalCount);
progressCallback(importProgress);
}
}
}
catch (Exception e)
{
HandleError(progressCallback, importProgress, e.Message);
}
finally
{
var completedMessage = importProgress.ErrorCount > 0 ? "Import completed with errors" : "Import completed";
importProgress.Description = $"{completedMessage}: {string.Format(importDescription, importProgress.ProcessedCount, importProgress.TotalCount)}";
if (importReporter.ReportIsNotEmpty)
{
importProgress.ReportUrl = _blobUrlResolver.GetAbsoluteUrl(reportFilePath);
}
progressCallback(importProgress);
}
}
protected abstract Task ProcessChunkAsync(ImportDataRequest request, Action <ImportProgressInfo> progressCallback, ICustomerImportPagedDataSource <TCsvMember> dataSource,
ImportErrorsContext errorsContext, ImportProgressInfo importProgress, ICsvCustomerImportReporter importReporter);
public async Task ImportAsync(ImportDataRequest request, Action <ImportProgressInfo> progressCallback, ICancellationToken cancellationToken)
{
ValidateParameters(request, progressCallback, cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
var errorsContext = new ImportErrorsContext();
var csvPriceDataValidationResult = await _csvPriceDataValidator.ValidateAsync(request.FilePath);
if (csvPriceDataValidationResult.Errors.Any())
{
throw new InvalidDataException();
}
var reportFilePath = GetReportFilePath(request.FilePath);
var configuration = _importConfigurationFactory.Create();
await using var importReporter = await _importReporterFactory.CreateAsync(reportFilePath, configuration.Delimiter);
cancellationToken.ThrowIfCancellationRequested();
var importProgress = new ImportProgressInfo {
ProcessedCount = 0, CreatedCount = 0, UpdatedCount = 0, Description = "Import has started"
};
SetupErrorHandlers(progressCallback, configuration, errorsContext, importProgress, importReporter);
using var dataSource = _dataSourceFactory.Create(request.FilePath, ModuleConstants.Settings.PageSize, configuration);
var headerRaw = dataSource.GetHeaderRaw();
if (!headerRaw.IsNullOrEmpty())
{
importReporter.WriteHeader(headerRaw);
}
importProgress.TotalCount = dataSource.GetTotalCount();
progressCallback(importProgress);
const string importDescription = "{0} out of {1} have been imported.";
try
{
importProgress.Description = "Fetching...";
progressCallback(importProgress);
var importProductPricesExistValidator = new ImportProductPricesExistenceValidator(_pricingSearchService, ImportProductPricesExistenceValidationMode.Exists);
while (await dataSource.FetchAsync())
{
cancellationToken.ThrowIfCancellationRequested();
var importProductPrices = dataSource.Items
// expect records that was parsed with errors
.Where(importProductPrice => !errorsContext.ErrorsRows.Contains(importProductPrice.Row))
.Select(importProductPrice =>
{
importProductPrice.Price.PricelistId = request.PricelistId;
return(importProductPrice);
}).ToArray();
try
{
var createdPrices = new List <Price>();
var updatedPrices = new List <Price>();
var validationResult = await _importProductPricesValidator.ValidateAsync(importProductPrices, options => options.IncludeRuleSets(request.ImportMode.ToString()));
var invalidImportProductPrices = validationResult.Errors.Select(x => (x.CustomState as ImportValidationState)?.InvalidImportProductPrice).Distinct().ToArray();
var errorsInfos = validationResult.Errors.Select(x => new { Message = x.ErrorMessage, ImportProductPrice = (x.CustomState as ImportValidationState)?.InvalidImportProductPrice }).ToArray();
var errorsGroups = errorsInfos.GroupBy(x => x.ImportProductPrice);
foreach (var group in errorsGroups)
{
var importPrice = group.Key;
var errorMessages = string.Join(" ", group.Select(x => x.Message).ToArray());
var importError = new ImportError {
Error = errorMessages, RawRow = importPrice.RawRecord
};
await importReporter.WriteAsync(importError);
}
importProgress.ErrorCount += invalidImportProductPrices.Length;
importProductPrices = importProductPrices.Except(invalidImportProductPrices).ToArray();
switch (request.ImportMode)
{
case ImportMode.CreateOnly:
createdPrices.AddRange(importProductPrices.Select(importProductPrice => importProductPrice.Price));
break;
case ImportMode.UpdateOnly:
var existingPrices = await GetAndPatchExistingPrices(request, importProductPrices);
updatedPrices.AddRange(existingPrices);
break;
case ImportMode.CreateAndUpdate:
var importProductPriceNotExistValidationResult = await importProductPricesExistValidator.ValidateAsync(importProductPrices);
var importProductPricesToCreate = importProductPriceNotExistValidationResult.Errors
.Select(x => (x.CustomState as ImportValidationState)?.InvalidImportProductPrice).Distinct().ToArray();
var importProductPricesToUpdate = importProductPrices.Except(importProductPricesToCreate).ToArray();
createdPrices.AddRange(importProductPricesToCreate.Select(importProductPrice => importProductPrice.Price));
var existingPricesToUpdate = await GetAndPatchExistingPrices(request, importProductPricesToUpdate);
updatedPrices.AddRange(existingPricesToUpdate);
break;
default:
throw new ArgumentException("Import mode has invalid value", nameof(request));
}
var allChangingPrices = createdPrices.Concat(updatedPrices).ToArray();
await _pricingService.SavePricesAsync(allChangingPrices);
importProgress.CreatedCount += createdPrices.Count;
importProgress.UpdatedCount += updatedPrices.Count;
}
catch (Exception e)
{
HandleError(progressCallback, importProgress, e.Message);
}
finally
{
importProgress.ProcessedCount = Math.Min(dataSource.CurrentPageNumber * dataSource.PageSize, importProgress.TotalCount);
}
if (importProgress.ProcessedCount != importProgress.TotalCount)
{
importProgress.Description = string.Format(importDescription, importProgress.ProcessedCount, importProgress.TotalCount);
progressCallback(importProgress);
}
}
}
catch (Exception e)
{
HandleError(progressCallback, importProgress, e.Message);
}
finally
{
var completedMessage = importProgress.ErrorCount > 0 ? "Import completed with errors" : "Import completed";
importProgress.Description = $"{completedMessage}: {string.Format(importDescription, importProgress.ProcessedCount, importProgress.TotalCount)}";
if (importReporter.ReportIsNotEmpty)
{
importProgress.ReportUrl = _blobUrlResolver.GetAbsoluteUrl(reportFilePath);
}
progressCallback(importProgress);
}
}
protected override async Task ProcessChunkAsync(ImportDataRequest request, Action <ImportProgressInfo> progressCallback, ICustomerImportPagedDataSource <ImportableContact> dataSource,
ImportErrorsContext errorsContext, ImportProgressInfo importProgress, ICsvCustomerImportReporter importReporter)
{
var importContacts = dataSource.Items
// expect records that was parsed with errors
.Where(importContact => !errorsContext.ErrorsRows.Contains(importContact.Row))
.ToArray();
try
{
var internalIds = importContacts.Select(x => x.Record?.Id).Distinct()
.Where(x => !string.IsNullOrEmpty(x))
.ToArray();
var outerIds = importContacts.Select(x => x.Record?.OuterId).Distinct()
.Where(x => !string.IsNullOrEmpty(x))
.ToArray();
var existedContacts =
(await SearchMembersByIdAndOuterIdAsync(internalIds, outerIds, new[] { nameof(Contact) }, true))
.OfType <Contact>().ToArray();
SetIdToNullForNotExisted(importContacts, existedContacts);
SetIdToRealForExistedOuterId(importContacts, existedContacts);
var validationResult = await ValidateAsync(importContacts, importReporter);
var invalidImportContacts = validationResult.Errors
.Select(x => (x.CustomState as ImportValidationState <ImportableContact>)?.InvalidRecord).Distinct().ToArray();
importContacts = importContacts.Except(invalidImportContacts).ToArray();
//reduce existed set after validation
existedContacts = existedContacts.Where(ec => importContacts.Any(ic =>
ec.Id.EqualsInvariant(ic.Record.Id) ||
!string.IsNullOrEmpty(ec.OuterId) && ec.OuterId.EqualsInvariant(ic.Record.OuterId)))
.ToArray();
var updateImportContacts = importContacts.Where(x => existedContacts.Any(ec =>
ec.Id.EqualsInvariant(x.Record.Id) ||
(!ec.OuterId.IsNullOrEmpty() && ec.OuterId.EqualsInvariant(x.Record.OuterId)))
).ToArray();
existedContacts = GetReducedExistedByWrongOuterId(updateImportContacts, existedContacts).OfType <Contact>().ToArray();
var createImportContacts = importContacts.Except(updateImportContacts).ToArray();
var internalOrgIds = importContacts.Select(x => x.Record?.OrganizationId).Distinct()
.Where(x => !string.IsNullOrEmpty(x)).ToArray();
var outerOrgIds = importContacts.Select(x => x.Record?.OrganizationOuterId).Distinct()
.Where(x => !string.IsNullOrEmpty(x)).ToArray();
var existedOrganizations =
(await SearchMembersByIdAndOuterIdAsync(internalOrgIds, outerOrgIds,
new[] { nameof(Organization) })).OfType <Organization>().ToArray();
var newContacts = CreateNewContacts(createImportContacts, existedOrganizations, request);
PatchExistedContacts(existedContacts, updateImportContacts, existedOrganizations, request);
var contactsForSave = newContacts.Union(existedContacts).ToArray();
await _memberService.SaveChangesAsync(contactsForSave);
await CreateAccountsForContacts(contactsForSave);
importProgress.ContactsCreated += newContacts.Length;
importProgress.ContactsUpdated += existedContacts.Length;
}
catch (Exception e)
{
HandleError(progressCallback, importProgress, e.Message);
}
finally
{
importProgress.ProcessedCount = Math.Min(dataSource.CurrentPageNumber * dataSource.PageSize, importProgress.TotalCount);
importProgress.ErrorCount = importProgress.ProcessedCount - importProgress.ContactsCreated - importProgress.ContactsUpdated;
}
}
private static Contact[] CreateNewContacts(ImportRecord <ImportableContact>[] createImportContacts, Organization[] existedOrganizations, ImportDataRequest request)
{
var newContacts = createImportContacts.Select(x =>
{
var contact = AbstractTypeFactory <Contact> .TryCreateInstance <Contact>();
x.Record.PatchModel(contact);
var existedOrg = existedOrganizations.FirstOrDefault(o => o.Id == x.Record.OrganizationId)
?? existedOrganizations.FirstOrDefault(o =>
o.OuterId == x.Record.OrganizationOuterId);
var orgIdForNewContact = existedOrg?.Id ?? request.OrganizationId;
contact.Organizations =
orgIdForNewContact != null ? new[] { orgIdForNewContact }.ToList() : new List <string>();
return(contact);
}).ToArray();
return(newContacts);
}
private static void PatchExistedContacts(IEnumerable <Contact> existedContacts, ImportRecord <ImportableContact>[] updateImportContacts, Organization[] existedOrganizations, ImportDataRequest request)
{
foreach (var existedContact in existedContacts)
{
var importContact = updateImportContacts.LastOrDefault(x => existedContact.Id.EqualsInvariant(x.Record.Id) ||
(!existedContact.OuterId.IsNullOrEmpty() && existedContact.OuterId.EqualsInvariant(x.Record.OuterId)));
var existedOrg = existedOrganizations.FirstOrDefault(o => o.Id.EqualsInvariant(importContact.Record.OrganizationId))
?? existedOrganizations.FirstOrDefault(o =>
!o.OuterId.IsNullOrEmpty() && o.OuterId.EqualsInvariant(importContact.Record.OrganizationOuterId));
var orgIdForNewContact = existedOrg?.Id ?? request.OrganizationId;
importContact?.Record.PatchModel(existedContact);
if (!orgIdForNewContact.IsNullOrEmpty() && !existedContact.Organizations.Contains(orgIdForNewContact))
{
existedContact.Organizations ??= new List <string>();
existedContact.Organizations.Add(orgIdForNewContact);
}
}
}
