public async Task <ActionResult <PlatformExportPushNotification> > RunExport([FromBody] ExportDataRequest request) { var authorizationResult = await _authorizationService.AuthorizeAsync(User, request.DataQuery, request.ExportTypeName + "ExportDataPolicy"); if (!authorizationResult.Succeeded) { return(Unauthorized()); } var typeTitle = request.ExportTypeName.LastIndexOf('.') > 0 ? request.ExportTypeName.Substring(request.ExportTypeName.LastIndexOf('.') + 1) : request.ExportTypeName; var notification = new ExportPushNotification(_userNameResolver.GetCurrentUserName()) { NotifyType = "PlatformExportPushNotification", Title = $"{typeTitle} export", Description = "Starting export task..." }; _pushNotificationManager.Send(notification); var jobId = BackgroundJob.Enqueue <ExportJob>(x => x.ExportBackgroundAsync(request, notification, JobCancellationToken.Null, null)); notification.JobId = jobId; return(Ok(notification)); }
/// <summary>Snippet for ExportData</summary> /// <remarks> /// This snippet has been automatically generated for illustrative purposes only. /// It may require modifications to work in your environment. /// </remarks> public void ExportDataRequestObject() { // Create client AutoMlClient autoMlClient = AutoMlClient.Create(); // Initialize request argument(s) ExportDataRequest request = new ExportDataRequest { DatasetName = DatasetName.FromProjectLocationDataset("[PROJECT]", "[LOCATION]", "[DATASET]"), OutputConfig = new OutputConfig(), }; // Make the request Operation <Empty, OperationMetadata> response = autoMlClient.ExportData(request); // Poll until the returned long-running operation is complete Operation <Empty, OperationMetadata> completedResponse = response.PollUntilCompleted(); // Retrieve the operation result Empty result = completedResponse.Result; // Or get the name of the operation string operationName = response.Name; // This name can be stored, then the long-running operation retrieved later by name Operation <Empty, OperationMetadata> retrievedResponse = autoMlClient.PollOnceExportData(operationName); // Check if the retrieved long-running operation has completed if (retrievedResponse.IsCompleted) { // If it has completed, then access the result Empty retrievedResult = retrievedResponse.Result; } }
private static string SerializeToString(params IExportable[] objects) { var exportDataRequest = new ExportDataRequest() { ProviderName = nameof(JsonExportProvider), ProviderConfig = new JsonProviderConfiguration() { Settings = new JsonSerializerSettings() { NullValueHandling = NullValueHandling.Ignore } }, }; using (var stream = new MemoryStream()) using (var writer = new StreamWriter(stream, Encoding.UTF8) { AutoFlush = true }) { using (var jsonExportProvider = new JsonExportProvider(exportDataRequest)) { foreach (var obj in objects) { jsonExportProvider.WriteRecord(writer, obj); } } stream.Seek(0, SeekOrigin.Begin); var reader = new StreamReader(stream); return(reader.ReadToEnd()); } }
public async Task ExportBackgroundAsync(ExportDataRequest request, ExportPushNotification pushNotification, IJobCancellationToken jobCancellationToken, PerformContext context) { ValidateParameters(pushNotification); try { await _customerDataExporter.ExportAsync(request, progressInfo => ProgressCallback(progressInfo, pushNotification, context), new JobCancellationTokenWrapper(jobCancellationToken)); } catch (JobAbortedException) { // job is aborted, do nothing } catch (Exception ex) { pushNotification.Errors.Add(ex.ExpandExceptionMessage()); } finally { pushNotification.Description = "Export finished"; pushNotification.Finished = DateTime.UtcNow; await _pushNotificationManager.SendAsync(pushNotification); } }
private string SerializeAndRead <T>(ExportDataRequest exportDataRequest, IEnumerable <T> items) where T : IExportable { exportDataRequest.ProviderConfig = exportDataRequest.ProviderConfig ?? new CsvProviderConfiguration() { }; using (var stream = new MemoryStream()) using (var writer = new StreamWriter(stream, Encoding.UTF8) { AutoFlush = true }) { using (var csvExportProvider = new CsvExportProvider(exportDataRequest)) { foreach (var item in items) { csvExportProvider.WriteRecord(writer, item); } } stream.Seek(0, SeekOrigin.Begin); return(new StreamReader(stream).ReadToEnd()); } }
/// <summary>Snippet for ExportDataAsync</summary> public async Task ExportDataRequestObjectAsync() { // Snippet: ExportDataAsync(ExportDataRequest, CallSettings) // Additional: ExportDataAsync(ExportDataRequest, CancellationToken) // Create client AutoMlClient autoMlClient = await AutoMlClient.CreateAsync(); // Initialize request argument(s) ExportDataRequest request = new ExportDataRequest { Name = "", OutputConfig = new OutputConfig(), }; // Make the request Operation <Empty, OperationMetadata> response = await autoMlClient.ExportDataAsync(request); // Poll until the returned long-running operation is complete Operation <Empty, OperationMetadata> completedResponse = await response.PollUntilCompletedAsync(); // Retrieve the operation result Empty result = completedResponse.Result; // Or get the name of the operation string operationName = response.Name; // This name can be stored, then the long-running operation retrieved later by name Operation <Empty, OperationMetadata> retrievedResponse = await autoMlClient.PollOnceExportDataAsync(operationName); // Check if the retrieved long-running operation has completed if (retrievedResponse.IsCompleted) { // If it has completed, then access the result Empty retrievedResult = retrievedResponse.Result; } // End snippet }
public string ExportFuelOrdersByCustClient(ExportDataRequest model) { var table = ExportFuelOrdersByCust(model.ClientID, model.ListOfIDs, model.StartDate.ToShortDateString(), model.EndDate.ToShortDateString()); DataTable copyDataTable; copyDataTable = table.Copy(); copyDataTable.PrimaryKey = null; copyDataTable.Columns["Name"].SetOrdinal(0); copyDataTable.Columns["RequestedBy"].SetOrdinal(1); copyDataTable.Columns["TailNumber"].SetOrdinal(4); copyDataTable.Columns["Product"].SetOrdinal(5); copyDataTable.Columns["PPG"].SetOrdinal(6); copyDataTable.Columns["DateRequested"].SetOrdinal(7); copyDataTable.Columns["FuelingDateTime"].SetOrdinal(8); copyDataTable.Columns["PostedRetail"].SetOrdinal(9); copyDataTable.Columns["QuotedPPG"].SetOrdinal(10); copyDataTable.Columns["InvoicedPPG"].SetOrdinal(11); copyDataTable.Columns["QuotedVolume"].SetOrdinal(12); copyDataTable.Columns["InvoicedVolume"].SetOrdinal(13); copyDataTable.Columns["RampFee"].SetOrdinal(14); copyDataTable.Columns["RampFeeWaivedAt"].SetOrdinal(15); copyDataTable.Columns["InvoiceStatus"].SetOrdinal(16); copyDataTable.Columns["CustNotes"].SetOrdinal(17); string filePath = ExportHelper.CreateFilePath("FuelOrderTransactions.csv", model.ClientName); CSVGenerator.GenerateCSV(copyDataTable, filePath); return(ExportHelper.GetFilePathFromRoot(filePath)); }
public string ExportContacts(ExportDataRequest model) { var table = GetCompanies(model.ClientID, model.ListOfIDs); string filePath = ExportHelper.CreateFilePath("Companies.csv", model.ClientName); CSVGenerator.GenerateCSV(table, filePath); return(ExportHelper.GetFilePathFromRoot(filePath)); }
public string ExportFuelOrdersByAdminClient(ExportDataRequest model) { var table = ExportFuelOrders(model.ClientID, model.ListOfIDs, model.StartDate.ToShortDateString(), model.EndDate.ToShortDateString()); string filePath = ExportHelper.CreateFilePath("FuelOrderTransactions.csv", model.ClientName); CSVGenerator.GenerateCSV(table, filePath); return(ExportHelper.GetFilePathFromRoot(filePath)); }
public void ExportBackground(ExportDataRequest request, ExportPushNotification notification, IJobCancellationToken cancellationToken, PerformContext context) { void progressCallback(ExportProgressInfo x) { notification.Patch(x); notification.JobId = context.BackgroundJob.Id; _pushNotificationManager.Upsert(notification); } try { var localTmpFolder = HostingEnvironment.MapPath(_defaultExportFolder); var fileName = Path.Combine(localTmpFolder, string.Format(FileNameTemplate, DateTime.UtcNow)); // Do not like provider creation here to get file extension, maybe need to pass created provider to Exporter. // Create stream inside Exporter is not good as it is not Exporter resposibility to decide where to write. var provider = _exportProviderFactory.CreateProvider(request); if (!string.IsNullOrEmpty(provider.ExportedFileExtension)) { fileName = Path.ChangeExtension(string.Format(FileNameTemplate, DateTime.UtcNow), provider.ExportedFileExtension); } var localTmpPath = Path.Combine(localTmpFolder, fileName); if (!Directory.Exists(localTmpFolder)) { Directory.CreateDirectory(localTmpFolder); } if (File.Exists(localTmpPath)) { File.Delete(localTmpPath); } //Import first to local tmp folder because Azure blob storage doesn't support some special file access mode using (var stream = File.OpenWrite(localTmpPath)) { _dataExporter.Export(stream, request, progressCallback, new JobCancellationTokenWrapper(cancellationToken)); notification.DownloadUrl = $"api/export/download/{fileName}"; } } catch (JobAbortedException) { //do nothing } catch (Exception ex) { notification.Errors.Add(ex.ExpandExceptionMessage()); } finally { notification.Description = "Export finished"; notification.Finished = DateTime.UtcNow; _pushNotificationManager.Upsert(notification); } }
public CustomerExportPagedDataSource(IMemberService memberService, IMemberSearchService memberSearchService, IStoreService storeService, int pageSize, ExportDataRequest request) { _memberService = memberService; _memberSearchService = memberSearchService; _storeService = storeService; _request = request; PageSize = pageSize; }
public CsvExportProvider(ExportDataRequest exportDataRequest) { if (exportDataRequest == null) { throw new ArgumentNullException(nameof(exportDataRequest)); } Configuration = exportDataRequest.ProviderConfig as CsvProviderConfiguration ?? new CsvProviderConfiguration(); IncludedProperties = exportDataRequest.DataQuery?.IncludedProperties; }
public virtual IExportProvider CreateProvider(ExportDataRequest exportDataRequest) { if (exportDataRequest == null) { throw new ArgumentNullException(nameof(exportDataRequest)); } var result = _providerFactories.FirstOrDefault(x => x(exportDataRequest).TypeName.EqualsInvariant(exportDataRequest.ProviderName)); return(result != null?result(exportDataRequest) : null); }
private ExportDataRequest CreatExportDataRequest(ExportDataQuery dataQuery, IExportProviderConfiguration providerConfig = null) { var result = new ExportDataRequest() { ProviderName = nameof(CsvExportProvider), DataQuery = dataQuery, ProviderConfig = providerConfig, }; return(result); }
public async Task ExportDataV1_ShouldReturn() { using (var httpClient = new HttpClient()) { var requestData = new ExportDataRequest(); var content = new StringContent(JsonConvert.SerializeObject(requestData), Encoding.UTF8, "application/json"); var result = await httpClient.PostAsync(new Uri(_url + "/v1/exportdata"), content); result.EnsureSuccessStatusCode(); } }
public string ExportByClientID(ExportDataRequest model) { if (!ModelState.IsValid) { return((Request.CreateErrorResponse(HttpStatusCode.BadRequest, ModelState)).ToString()); } var table = GetAirports(model.ClientID, model.ListOfIDs); string filePath = ExportHelper.CreateFilePath("Airports.csv", model.ClientName); CSVGenerator.GenerateCSV(table, filePath); return(ExportHelper.GetFilePathFromRoot(filePath)); }
private static string Export(ExportDataServiceClient client, DataType type) { var start = DateTime.Now.Date.Subtract(TimeSpan.FromDays(1)).Subtract(TimeSpan.FromSeconds(1)); var end = DateTime.Now.Date.Subtract(TimeSpan.FromSeconds(1)); var request = new ExportDataRequest() { StartDate = start, EndDate = end, Target = type }; var resultTask = client.GetOrders(request); resultTask.Wait(); return(resultTask.Result.ToString()); }
public async Task ExportBackgroundAsync(ExportDataRequest request, ExportPushNotification notification, IJobCancellationToken cancellationToken, PerformContext context) { void progressCallback(ExportProgressInfo x) { notification.Patch(x); notification.JobId = context.BackgroundJob.Id; _pushNotificationManager.Send(notification); } try { if (string.IsNullOrEmpty(_platformOptions.DefaultExportFolder)) { throw new PlatformException($"{nameof(_platformOptions.DefaultExportFolder)} should be set."); } var fileName = string.Format(FileNameTemplate, DateTime.UtcNow); // Do not like provider creation here to get file extension, maybe need to pass created provider to Exporter. // Create stream inside Exporter is not good as it is not Exporter resposibility to decide where to write. var provider = _exportProviderFactory.CreateProvider(request); if (!string.IsNullOrEmpty(provider.ExportedFileExtension)) { fileName = Path.ChangeExtension(fileName, provider.ExportedFileExtension); } var url = UrlHelperExtensions.Combine(_platformOptions.DefaultExportFolder, fileName); using (var blobStream = _blobStorageProvider.OpenWrite(url)) { _dataExporter.Export(blobStream, request, progressCallback, new JobCancellationTokenWrapper(cancellationToken)); } notification.DownloadUrl = _blobUrlResolver.GetAbsoluteUrl(url); } catch (JobAbortedException) { //do nothing } catch (Exception ex) { notification.Errors.Add(ex.ExpandExceptionMessage()); } finally { notification.Description = "Export finished"; notification.Finished = DateTime.UtcNow; await _pushNotificationManager.SendAsync(notification); } }
public async Task <ActionResult <ExportPushNotification> > RunExport([FromBody] ExportDataRequest request) { var notification = new ExportPushNotification(_userNameResolver.GetCurrentUserName()) { Title = "Customers export", Description = "Starting export task..." }; await _pushNotificationManager.SendAsync(notification); notification.JobId = BackgroundJob.Enqueue <ExportJob>(importJob => importJob.ExportBackgroundAsync(request, notification, JobCancellationToken.Null, null)); return(Ok(notification)); }
public string ExportPrices(ExportDataRequest model /*int adminId, string adminName*/) { var table = GetSupplierFuelsPricesDataTable(model.ClientID); DataTable copyDataTable; copyDataTable = table.Copy(); copyDataTable.PrimaryKey = null; copyDataTable.Columns.Remove("Id"); copyDataTable.Columns.Remove("AdminClientID"); copyDataTable.Columns.Remove("SupplierID"); copyDataTable.Columns.Remove("DateUploaded"); copyDataTable.Columns["EffectiveDate"].SetOrdinal(7); string filePath = ExportHelper.CreateFilePath("VendorFuelsPrices.csv", model.ClientName); CSVGenerator.GenerateCSV(copyDataTable, filePath); return(ExportHelper.GetFilePathFromRoot(filePath)); }
public JsonExportProvider(ExportDataRequest exportDataRequest) { if (exportDataRequest == null) { throw new ArgumentNullException(nameof(exportDataRequest)); } var jsonProviderConfiguration = exportDataRequest.ProviderConfig as JsonProviderConfiguration ?? new JsonProviderConfiguration(); Configuration = jsonProviderConfiguration; IncludedProperties = exportDataRequest.DataQuery?.IncludedProperties; var jsonSettings = jsonProviderConfiguration.Settings; _serializer = JsonSerializer.Create(jsonSettings); _serializer.Converters.Add(new ObjectDiscriminatorJsonConverter(jsonSettings, typeof(Entity))); }
//[ResponseType(typeof(PlatformExportPushNotification))] public IHttpActionResult RunExport([FromBody] ExportDataRequest request) { if (_exportSecurityHandlerRegistrar.GetHandler(request.ExportTypeName + "ExportDataPolicy")?.Authorize(User.Identity.Name, request) != true) { return(Unauthorized()); } var notification = new ExportPushNotification(_userNameResolver.GetCurrentUserName()) { Title = $"{request.ExportTypeName} export task", Description = "starting export...." }; //_pushNotificationManager.Send(notification); var jobId = BackgroundJob.Enqueue <ExportJob>(x => x.ExportBackground(request, notification, JobCancellationToken.Null, null)); notification.JobId = jobId; return(Ok(notification)); }
public IHttpActionResult GetData([FromBody] ExportDataRequest request) { if (_exportSecurityHandlerRegistrar.GetHandler(request.ExportTypeName + "ExportDataPolicy")?.Authorize(User.Identity.Name, request) != true) { return(Unauthorized()); } var exportedTypeDefinition = _knownExportTypesResolver.ResolveExportedTypeDefinition(request.ExportTypeName); var pagedDataSource = exportedTypeDefinition.ExportedDataSourceFactory(request.DataQuery); pagedDataSource.Fetch(); var queryResult = pagedDataSource.Items; var result = new ExportableSearchResult() { TotalCount = pagedDataSource.GetTotalCount(), Results = queryResult.ToList() }; return(Ok(result)); }
public async Task <ActionResult <ExportableSearchResult> > GetData([FromBody] ExportDataRequest request) { var authorizationResult = await _authorizationService.AuthorizeAsync(User, request.DataQuery, request.ExportTypeName + "ExportDataPolicy"); if (!authorizationResult.Succeeded) { return(Unauthorized()); } var exportedTypeDefinition = _knownExportTypesResolver.ResolveExportedTypeDefinition(request.ExportTypeName); var pagedDataSource = (exportedTypeDefinition.DataSourceFactory ?? throw new ArgumentNullException(nameof(ExportedTypeDefinition.DataSourceFactory))).Create(request.DataQuery); pagedDataSource.Fetch(); var queryResult = pagedDataSource.Items; var result = new ExportableSearchResult() { TotalCount = pagedDataSource.GetTotalCount(), Results = queryResult.ToList() }; return(Ok(result)); }
// --- private async Task <T> ExportData <T>(ExportDataRequest ExportDataRequest) { if (string.IsNullOrEmpty(this.Config.AccessToken)) { await this.RefreshToken(); } var response = await this.HttpService.POST_ASYNC <ExportDataRequest, T>($"https://{this.Config.ServerName}/ExportDataApi/exportData", ExportDataRequest, headers : new Dictionary <string, string> { ["Accept"] = "application/json", ["Content-Type"] = "application/json", ["Authorization"] = $"Bearer {this.Config.AccessToken}" }); var status = this.ParseResponse(response.Content); // Unauthorized - refresh token and try again if (!status.Success && status.Error == "Token is not valid") { await this.RefreshToken(); response = await this.HttpService.POST_ASYNC <ExportDataRequest, T>($"https://{this.Config.ServerName}/ExportDataApi/exportData", ExportDataRequest, headers : new Dictionary <string, string> { ["Accept"] = "application/json", ["Content-Type"] = "application/json", ["Authorization"] = $"Bearer {this.Config.AccessToken}" }); status = this.ParseResponse(response.Content); } if (!status.Success) { throw new Exception(response.Content); } return(response.Model); }
public JsonExportProvider(ExportDataRequest exportDataRequest) { if (exportDataRequest == null) { throw new ArgumentNullException(nameof(exportDataRequest)); } var jsonProviderConfiguration = exportDataRequest.ProviderConfig as JsonProviderConfiguration ?? new JsonProviderConfiguration(); Configuration = jsonProviderConfiguration; IncludedProperties = exportDataRequest.DataQuery?.IncludedProperties; var jsonSettings = new JsonSerializerSettings() { NullValueHandling = NullValueHandling.Ignore, DefaultValueHandling = DefaultValueHandling.Ignore, ReferenceLoopHandling = ReferenceLoopHandling.Ignore, Formatting = jsonProviderConfiguration.Indented ? Formatting.Indented : Formatting.None }; _serializer = JsonSerializer.Create(jsonSettings); _serializer.Converters.Add(new ObjectDiscriminatorJsonConverter(jsonSettings, typeof(Entity))); }
public async Task <ExportDataResponse> ExportDataAsync(ExportDataRequest request) => await Task.FromResult(new ExportDataResponse());
public void Export(Stream stream, ExportDataRequest request, Action <ExportProgressInfo> progressCallback, ICancellationToken token) { if (request == null) { throw new ArgumentNullException(nameof(request)); } token.ThrowIfCancellationRequested(); var exportedTypeDefinition = _exportTypesResolver.ResolveExportedTypeDefinition(request.ExportTypeName); var pagedDataSource = (exportedTypeDefinition.DataSourceFactory ?? throw new ArgumentNullException(nameof(ExportedTypeDefinition.DataSourceFactory))).Create(request.DataQuery); var completedMessage = $"Export completed"; var totalCount = pagedDataSource.GetTotalCount(); var exportedCount = 0; var exportProgress = new ExportProgressInfo() { ProcessedCount = 0, TotalCount = totalCount, Description = "Export has started", }; progressCallback(exportProgress); try { exportProgress.Description = "Creating provider…"; progressCallback(exportProgress); using (var writer = new StreamWriter(stream, Encoding.UTF8, 1024, true) { AutoFlush = true }) using (var exportProvider = _exportProviderFactory.CreateProvider(request)) { var needTabularData = exportProvider.IsTabular; if (needTabularData && !exportedTypeDefinition.IsTabularExportSupported) { throw new NotSupportedException($"Provider \"{exportProvider.TypeName}\" does not support tabular export."); } exportProgress.Description = "Fetching…"; progressCallback(exportProgress); while (pagedDataSource.Fetch()) { token.ThrowIfCancellationRequested(); var objectBatch = pagedDataSource.Items; foreach (var obj in objectBatch) { try { var preparedObject = obj.Clone() as IExportable; request.DataQuery.FilterProperties(preparedObject); if (needTabularData) { preparedObject = (preparedObject as ITabularConvertible)?.ToTabular() ?? throw new NotSupportedException($"Object should be {nameof(ITabularConvertible)} to be exported using tabular provider."); } exportProvider.WriteRecord(writer, preparedObject); } catch (Exception e) { exportProgress.Errors.Add(e.Message); progressCallback(exportProgress); } exportedCount++; } exportProgress.ProcessedCount = exportedCount; if (exportedCount != totalCount) { exportProgress.Description = $"{exportedCount} out of {totalCount} have been exported."; progressCallback(exportProgress); } } } } catch (Exception e) { exportProgress.Errors.Add(e.Message); } finally { if (exportProgress.Errors.Count > 0) { completedMessage = $"Export completed with errors"; } exportProgress.Description = $"{completedMessage}: {exportedCount} out of {totalCount} have been exported."; progressCallback(exportProgress); } }
/// <summary> /// 数据导出 /// </summary> /// <param name="request">请求参数信息</param> /// <returns>请求结果信息</returns> public ExportDataResponse ExportData(ExportDataRequest request) { return(new ExportDataExecutor().Client(this).Execute <ExportDataResponse, ExportDataResult, ExportDataRequest>(request)); }
/// <summary> /// 数据导出 /// </summary> /// <param name="request">请求参数信息</param> /// <returns>请求结果信息</returns> public async Task <ExportDataResponse> ExportData(ExportDataRequest request) { return(await new ExportDataExecutor().Client(this).Execute <ExportDataResponse, ExportDataResult, ExportDataRequest>(request).ConfigureAwait(false)); }