/// <summary>Snippet for ImportDataAsync</summary> /// <remarks> /// This snippet has been automatically generated for illustrative purposes only. /// It may require modifications to work in your environment. /// </remarks> public async Task ImportDataRequestObjectAsync() { // Create client AutoMlClient autoMlClient = await AutoMlClient.CreateAsync(); // Initialize request argument(s) ImportDataRequest request = new ImportDataRequest { DatasetName = DatasetName.FromProjectLocationDataset("[PROJECT]", "[LOCATION]", "[DATASET]"), InputConfig = new InputConfig(), }; // Make the request Operation <Empty, OperationMetadata> response = await autoMlClient.ImportDataAsync(request); // Poll until the returned long-running operation is complete Operation <Empty, OperationMetadata> completedResponse = await response.PollUntilCompletedAsync(); // Retrieve the operation result Empty result = completedResponse.Result; // Or get the name of the operation string operationName = response.Name; // This name can be stored, then the long-running operation retrieved later by name Operation <Empty, OperationMetadata> retrievedResponse = await autoMlClient.PollOnceImportDataAsync(operationName); // Check if the retrieved long-running operation has completed if (retrievedResponse.IsCompleted) { // If it has completed, then access the result Empty retrievedResult = retrievedResponse.Result; } }
public async Task ImportBackgroundAsync(ImportDataRequest request, ImportPushNotification pushNotification, IJobCancellationToken jobCancellationToken, PerformContext context) { ValidateParameters(request, pushNotification); try { var importer = _customerDataImporters.First(x => x.MemberType == request.MemberType); await importer.ImportAsync(request, progressInfo => ProgressCallback(progressInfo, pushNotification, context), new JobCancellationTokenWrapper(jobCancellationToken)); } catch (JobAbortedException) { // job is aborted, do nothing } catch (Exception ex) { pushNotification.Errors.Add(ex.ExpandExceptionMessage()); } finally { pushNotification.Description = "Import finished"; pushNotification.Finished = DateTime.UtcNow; await _pushNotificationManager.SendAsync(pushNotification); } }
/// <summary>Snippet for ImportDataAsync</summary> public async Task ImportDataRequestObjectAsync() { // Snippet: ImportDataAsync(ImportDataRequest, CallSettings) // Additional: ImportDataAsync(ImportDataRequest, CancellationToken) // Create client AutoMlClient autoMlClient = await AutoMlClient.CreateAsync(); // Initialize request argument(s) ImportDataRequest request = new ImportDataRequest { Name = "", InputConfig = new InputConfig(), }; // Make the request Operation <Empty, OperationMetadata> response = await autoMlClient.ImportDataAsync(request); // Poll until the returned long-running operation is complete Operation <Empty, OperationMetadata> completedResponse = await response.PollUntilCompletedAsync(); // Retrieve the operation result Empty result = completedResponse.Result; // Or get the name of the operation string operationName = response.Name; // This name can be stored, then the long-running operation retrieved later by name Operation <Empty, OperationMetadata> retrievedResponse = await autoMlClient.PollOnceImportDataAsync(operationName); // Check if the retrieved long-running operation has completed if (retrievedResponse.IsCompleted) { // If it has completed, then access the result Empty retrievedResult = retrievedResponse.Result; } // End snippet }
private async Task <Price[]> GetAndPatchExistingPrices(ImportDataRequest request, ImportProductPrice[] importProductPrices) { var updateProductIds = importProductPrices.Select(x => x.ProductId).ToArray(); var existingPricesSearchResult = await _pricingSearchService.SearchPricesAsync( new PricesSearchCriteria { ProductIds = updateProductIds, PriceListIds = new[] { request.PricelistId }, Take = int.MaxValue }); var existingPrices = existingPricesSearchResult .Results .Where(x => importProductPrices.Select(i => i.Price).Any(p => p.MinQuantity == x.MinQuantity && p.ProductId == x.ProductId)) .ToArray(); foreach (var existingPrice in existingPrices) { var priceForUpdating = importProductPrices .FirstOrDefault(x => x.ProductId == existingPrice.ProductId && x.Price.MinQuantity == existingPrice.MinQuantity); if (priceForUpdating != null) { existingPrice.Sale = priceForUpdating.Price.Sale; existingPrice.List = priceForUpdating.Price.List; } } return(existingPrices); }
public async Task <ActionResult <ImportPushNotification> > RunImport([FromBody] ImportDataRequest request) { var notification = new ImportPushNotification(_userNameResolver.GetCurrentUserName()) { Title = "Prices import", Description = "Starting import task..." }; await _pushNotificationManager.SendAsync(notification); notification.JobId = BackgroundJob.Enqueue <ImportJob>(importJob => importJob.ImportBackgroundAsync(request, notification, JobCancellationToken.Null, null)); return(Ok(notification)); }
private void ValidateParameters(ImportDataRequest request, ImportPushNotification pushNotification) { if (pushNotification == null) { throw new ArgumentNullException(nameof(pushNotification)); } var importer = _customerDataImporters.FirstOrDefault(x => x.MemberType == request.MemberType); if (importer == null) { throw new ArgumentException($"Not allowed argument value in field {nameof(request.MemberType)}", nameof(request)); } }
public ActionResult InsertImportData(string importViewModel) { ImportDataListViewModel viewModel = JsonConvert.DeserializeObject <ImportDataListViewModel>(importViewModel); viewModel.AccountID = UserExtensions.ToAccountID(this.Identity); viewModel.UserId = UserExtensions.ToUserID(this.Identity); ImportDataRequest request = new ImportDataRequest() { ImportDataListViewModel = viewModel, RequestedBy = viewModel.OwnerId }; accountService.ImportData(request); return(Json(new { success = true, response = "" }, JsonRequestBehavior.AllowGet)); }
private static void ValidateParameters(ImportDataRequest request, Action <ImportProgressInfo> progressCallback, ICancellationToken cancellationToken) { if (request == null) { throw new ArgumentNullException(nameof(request)); } if (progressCallback == null) { throw new ArgumentNullException(nameof(progressCallback)); } if (cancellationToken == null) { throw new ArgumentNullException(nameof(cancellationToken)); } }
public virtual async Task ImportAsync(ImportDataRequest request, Action <ImportProgressInfo> progressCallback, ICancellationToken cancellationToken) { ValidateParameters(request, progressCallback, cancellationToken); var errorsContext = new ImportErrorsContext(); var csvPriceDataValidationResult = await _dataValidator.ValidateAsync <TCsvMember>(request.FilePath); if (csvPriceDataValidationResult.Errors.Any()) { throw new InvalidDataException(); } var configuration = new ImportConfiguration(); var reportFilePath = GetReportFilePath(request.FilePath); await using var importReporter = await _importReporterFactory.CreateAsync(reportFilePath, configuration.Delimiter); cancellationToken.ThrowIfCancellationRequested(); var importProgress = new ImportProgressInfo { Description = "Import has started" }; using var dataSource = await _dataSourceFactory.CreateAsync <TCsvMember, TMember>(request.FilePath, ModuleConstants.Settings.PageSize, configuration); var headerRaw = dataSource.GetHeaderRaw(); if (!headerRaw.IsNullOrEmpty()) { await importReporter.WriteHeaderAsync(headerRaw); } importProgress.TotalCount = dataSource.GetTotalCount(); progressCallback(importProgress); const string importDescription = "{0} out of {1} have been imported."; SetupErrorHandlers(progressCallback, configuration, errorsContext, importProgress, importReporter); try { importProgress.Description = "Fetching..."; progressCallback(importProgress); while (await dataSource.FetchAsync()) { await ProcessChunkAsync(request, progressCallback, dataSource, errorsContext, importProgress, importReporter); if (importProgress.ProcessedCount != importProgress.TotalCount) { importProgress.Description = string.Format(importDescription, importProgress.ProcessedCount, importProgress.TotalCount); progressCallback(importProgress); } } } catch (Exception e) { HandleError(progressCallback, importProgress, e.Message); } finally { var completedMessage = importProgress.ErrorCount > 0 ? "Import completed with errors" : "Import completed"; importProgress.Description = $"{completedMessage}: {string.Format(importDescription, importProgress.ProcessedCount, importProgress.TotalCount)}"; if (importReporter.ReportIsNotEmpty) { importProgress.ReportUrl = _blobUrlResolver.GetAbsoluteUrl(reportFilePath); } progressCallback(importProgress); } }
protected abstract Task ProcessChunkAsync(ImportDataRequest request, Action <ImportProgressInfo> progressCallback, ICustomerImportPagedDataSource <TCsvMember> dataSource, ImportErrorsContext errorsContext, ImportProgressInfo importProgress, ICsvCustomerImportReporter importReporter);
public async Task ImportAsync(ImportDataRequest request, Action <ImportProgressInfo> progressCallback, ICancellationToken cancellationToken) { ValidateParameters(request, progressCallback, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); var errorsContext = new ImportErrorsContext(); var csvPriceDataValidationResult = await _csvPriceDataValidator.ValidateAsync(request.FilePath); if (csvPriceDataValidationResult.Errors.Any()) { throw new InvalidDataException(); } var reportFilePath = GetReportFilePath(request.FilePath); var configuration = _importConfigurationFactory.Create(); await using var importReporter = await _importReporterFactory.CreateAsync(reportFilePath, configuration.Delimiter); cancellationToken.ThrowIfCancellationRequested(); var importProgress = new ImportProgressInfo { ProcessedCount = 0, CreatedCount = 0, UpdatedCount = 0, Description = "Import has started" }; SetupErrorHandlers(progressCallback, configuration, errorsContext, importProgress, importReporter); using var dataSource = _dataSourceFactory.Create(request.FilePath, ModuleConstants.Settings.PageSize, configuration); var headerRaw = dataSource.GetHeaderRaw(); if (!headerRaw.IsNullOrEmpty()) { importReporter.WriteHeader(headerRaw); } importProgress.TotalCount = dataSource.GetTotalCount(); progressCallback(importProgress); const string importDescription = "{0} out of {1} have been imported."; try { importProgress.Description = "Fetching..."; progressCallback(importProgress); var importProductPricesExistValidator = new ImportProductPricesExistenceValidator(_pricingSearchService, ImportProductPricesExistenceValidationMode.Exists); while (await dataSource.FetchAsync()) { cancellationToken.ThrowIfCancellationRequested(); var importProductPrices = dataSource.Items // expect records that was parsed with errors .Where(importProductPrice => !errorsContext.ErrorsRows.Contains(importProductPrice.Row)) .Select(importProductPrice => { importProductPrice.Price.PricelistId = request.PricelistId; return(importProductPrice); }).ToArray(); try { var createdPrices = new List <Price>(); var updatedPrices = new List <Price>(); var validationResult = await _importProductPricesValidator.ValidateAsync(importProductPrices, options => options.IncludeRuleSets(request.ImportMode.ToString())); var invalidImportProductPrices = validationResult.Errors.Select(x => (x.CustomState as ImportValidationState)?.InvalidImportProductPrice).Distinct().ToArray(); var errorsInfos = validationResult.Errors.Select(x => new { Message = x.ErrorMessage, ImportProductPrice = (x.CustomState as ImportValidationState)?.InvalidImportProductPrice }).ToArray(); var errorsGroups = errorsInfos.GroupBy(x => x.ImportProductPrice); foreach (var group in errorsGroups) { var importPrice = group.Key; var errorMessages = string.Join(" ", group.Select(x => x.Message).ToArray()); var importError = new ImportError { Error = errorMessages, RawRow = importPrice.RawRecord }; await importReporter.WriteAsync(importError); } importProgress.ErrorCount += invalidImportProductPrices.Length; importProductPrices = importProductPrices.Except(invalidImportProductPrices).ToArray(); switch (request.ImportMode) { case ImportMode.CreateOnly: createdPrices.AddRange(importProductPrices.Select(importProductPrice => importProductPrice.Price)); break; case ImportMode.UpdateOnly: var existingPrices = await GetAndPatchExistingPrices(request, importProductPrices); updatedPrices.AddRange(existingPrices); break; case ImportMode.CreateAndUpdate: var importProductPriceNotExistValidationResult = await importProductPricesExistValidator.ValidateAsync(importProductPrices); var importProductPricesToCreate = importProductPriceNotExistValidationResult.Errors .Select(x => (x.CustomState as ImportValidationState)?.InvalidImportProductPrice).Distinct().ToArray(); var importProductPricesToUpdate = importProductPrices.Except(importProductPricesToCreate).ToArray(); createdPrices.AddRange(importProductPricesToCreate.Select(importProductPrice => importProductPrice.Price)); var existingPricesToUpdate = await GetAndPatchExistingPrices(request, importProductPricesToUpdate); updatedPrices.AddRange(existingPricesToUpdate); break; default: throw new ArgumentException("Import mode has invalid value", nameof(request)); } var allChangingPrices = createdPrices.Concat(updatedPrices).ToArray(); await _pricingService.SavePricesAsync(allChangingPrices); importProgress.CreatedCount += createdPrices.Count; importProgress.UpdatedCount += updatedPrices.Count; } catch (Exception e) { HandleError(progressCallback, importProgress, e.Message); } finally { importProgress.ProcessedCount = Math.Min(dataSource.CurrentPageNumber * dataSource.PageSize, importProgress.TotalCount); } if (importProgress.ProcessedCount != importProgress.TotalCount) { importProgress.Description = string.Format(importDescription, importProgress.ProcessedCount, importProgress.TotalCount); progressCallback(importProgress); } } } catch (Exception e) { HandleError(progressCallback, importProgress, e.Message); } finally { var completedMessage = importProgress.ErrorCount > 0 ? "Import completed with errors" : "Import completed"; importProgress.Description = $"{completedMessage}: {string.Format(importDescription, importProgress.ProcessedCount, importProgress.TotalCount)}"; if (importReporter.ReportIsNotEmpty) { importProgress.ReportUrl = _blobUrlResolver.GetAbsoluteUrl(reportFilePath); } progressCallback(importProgress); } }
protected override async Task ProcessChunkAsync(ImportDataRequest request, Action <ImportProgressInfo> progressCallback, ICustomerImportPagedDataSource <ImportableContact> dataSource, ImportErrorsContext errorsContext, ImportProgressInfo importProgress, ICsvCustomerImportReporter importReporter) { var importContacts = dataSource.Items // expect records that was parsed with errors .Where(importContact => !errorsContext.ErrorsRows.Contains(importContact.Row)) .ToArray(); try { var internalIds = importContacts.Select(x => x.Record?.Id).Distinct() .Where(x => !string.IsNullOrEmpty(x)) .ToArray(); var outerIds = importContacts.Select(x => x.Record?.OuterId).Distinct() .Where(x => !string.IsNullOrEmpty(x)) .ToArray(); var existedContacts = (await SearchMembersByIdAndOuterIdAsync(internalIds, outerIds, new[] { nameof(Contact) }, true)) .OfType <Contact>().ToArray(); SetIdToNullForNotExisted(importContacts, existedContacts); SetIdToRealForExistedOuterId(importContacts, existedContacts); var validationResult = await ValidateAsync(importContacts, importReporter); var invalidImportContacts = validationResult.Errors .Select(x => (x.CustomState as ImportValidationState <ImportableContact>)?.InvalidRecord).Distinct().ToArray(); importContacts = importContacts.Except(invalidImportContacts).ToArray(); //reduce existed set after validation existedContacts = existedContacts.Where(ec => importContacts.Any(ic => ec.Id.EqualsInvariant(ic.Record.Id) || !string.IsNullOrEmpty(ec.OuterId) && ec.OuterId.EqualsInvariant(ic.Record.OuterId))) .ToArray(); var updateImportContacts = importContacts.Where(x => existedContacts.Any(ec => ec.Id.EqualsInvariant(x.Record.Id) || (!ec.OuterId.IsNullOrEmpty() && ec.OuterId.EqualsInvariant(x.Record.OuterId))) ).ToArray(); existedContacts = GetReducedExistedByWrongOuterId(updateImportContacts, existedContacts).OfType <Contact>().ToArray(); var createImportContacts = importContacts.Except(updateImportContacts).ToArray(); var internalOrgIds = importContacts.Select(x => x.Record?.OrganizationId).Distinct() .Where(x => !string.IsNullOrEmpty(x)).ToArray(); var outerOrgIds = importContacts.Select(x => x.Record?.OrganizationOuterId).Distinct() .Where(x => !string.IsNullOrEmpty(x)).ToArray(); var existedOrganizations = (await SearchMembersByIdAndOuterIdAsync(internalOrgIds, outerOrgIds, new[] { nameof(Organization) })).OfType <Organization>().ToArray(); var newContacts = CreateNewContacts(createImportContacts, existedOrganizations, request); PatchExistedContacts(existedContacts, updateImportContacts, existedOrganizations, request); var contactsForSave = newContacts.Union(existedContacts).ToArray(); await _memberService.SaveChangesAsync(contactsForSave); await CreateAccountsForContacts(contactsForSave); importProgress.ContactsCreated += newContacts.Length; importProgress.ContactsUpdated += existedContacts.Length; } catch (Exception e) { HandleError(progressCallback, importProgress, e.Message); } finally { importProgress.ProcessedCount = Math.Min(dataSource.CurrentPageNumber * dataSource.PageSize, importProgress.TotalCount); importProgress.ErrorCount = importProgress.ProcessedCount - importProgress.ContactsCreated - importProgress.ContactsUpdated; } }
private static Contact[] CreateNewContacts(ImportRecord <ImportableContact>[] createImportContacts, Organization[] existedOrganizations, ImportDataRequest request) { var newContacts = createImportContacts.Select(x => { var contact = AbstractTypeFactory <Contact> .TryCreateInstance <Contact>(); x.Record.PatchModel(contact); var existedOrg = existedOrganizations.FirstOrDefault(o => o.Id == x.Record.OrganizationId) ?? existedOrganizations.FirstOrDefault(o => o.OuterId == x.Record.OrganizationOuterId); var orgIdForNewContact = existedOrg?.Id ?? request.OrganizationId; contact.Organizations = orgIdForNewContact != null ? new[] { orgIdForNewContact }.ToList() : new List <string>(); return(contact); }).ToArray(); return(newContacts); }
private static void PatchExistedContacts(IEnumerable <Contact> existedContacts, ImportRecord <ImportableContact>[] updateImportContacts, Organization[] existedOrganizations, ImportDataRequest request) { foreach (var existedContact in existedContacts) { var importContact = updateImportContacts.LastOrDefault(x => existedContact.Id.EqualsInvariant(x.Record.Id) || (!existedContact.OuterId.IsNullOrEmpty() && existedContact.OuterId.EqualsInvariant(x.Record.OuterId))); var existedOrg = existedOrganizations.FirstOrDefault(o => o.Id.EqualsInvariant(importContact.Record.OrganizationId)) ?? existedOrganizations.FirstOrDefault(o => !o.OuterId.IsNullOrEmpty() && o.OuterId.EqualsInvariant(importContact.Record.OrganizationOuterId)); var orgIdForNewContact = existedOrg?.Id ?? request.OrganizationId; importContact?.Record.PatchModel(existedContact); if (!orgIdForNewContact.IsNullOrEmpty() && !existedContact.Organizations.Contains(orgIdForNewContact)) { existedContact.Organizations ??= new List <string>(); existedContact.Organizations.Add(orgIdForNewContact); } } }