public async Task <ActionResult <PlatformExportPushNotification> > RunExport([FromBody] ExportDataRequest request)
{
var authorizationResult = await _authorizationService.AuthorizeAsync(User, request.DataQuery, request.ExportTypeName + "ExportDataPolicy");
if (!authorizationResult.Succeeded)
{
return(Unauthorized());
}
var typeTitle = request.ExportTypeName.LastIndexOf('.') > 0 ?
request.ExportTypeName.Substring(request.ExportTypeName.LastIndexOf('.') + 1) : request.ExportTypeName;
var notification = new ExportPushNotification(_userNameResolver.GetCurrentUserName())
{
NotifyType = "PlatformExportPushNotification",
Title = $"{typeTitle} export",
Description = "Starting export task..."
};
_pushNotificationManager.Send(notification);
var jobId = BackgroundJob.Enqueue <ExportJob>(x => x.ExportBackgroundAsync(request, notification, JobCancellationToken.Null, null));
notification.JobId = jobId;
return(Ok(notification));
}
/// <summary>Snippet for ExportData</summary>
/// <remarks>
/// This snippet has been automatically generated for illustrative purposes only.
/// It may require modifications to work in your environment.
/// </remarks>
public void ExportDataRequestObject()
{
// Create client
AutoMlClient autoMlClient = AutoMlClient.Create();
// Initialize request argument(s)
ExportDataRequest request = new ExportDataRequest
{
DatasetName = DatasetName.FromProjectLocationDataset("[PROJECT]", "[LOCATION]", "[DATASET]"),
OutputConfig = new OutputConfig(),
};
// Make the request
Operation <Empty, OperationMetadata> response = autoMlClient.ExportData(request);
// Poll until the returned long-running operation is complete
Operation <Empty, OperationMetadata> completedResponse = response.PollUntilCompleted();
// Retrieve the operation result
Empty result = completedResponse.Result;
// Or get the name of the operation
string operationName = response.Name;
// This name can be stored, then the long-running operation retrieved later by name
Operation <Empty, OperationMetadata> retrievedResponse = autoMlClient.PollOnceExportData(operationName);
// Check if the retrieved long-running operation has completed
if (retrievedResponse.IsCompleted)
{
// If it has completed, then access the result
Empty retrievedResult = retrievedResponse.Result;
}
}
private static string SerializeToString(params IExportable[] objects)
{
var exportDataRequest = new ExportDataRequest()
{
ProviderName = nameof(JsonExportProvider),
ProviderConfig = new JsonProviderConfiguration()
{
Settings = new JsonSerializerSettings()
{
NullValueHandling = NullValueHandling.Ignore
}
},
};
using (var stream = new MemoryStream())
using (var writer = new StreamWriter(stream, Encoding.UTF8)
{
AutoFlush = true
})
{
using (var jsonExportProvider = new JsonExportProvider(exportDataRequest))
{
foreach (var obj in objects)
{
jsonExportProvider.WriteRecord(writer, obj);
}
}
stream.Seek(0, SeekOrigin.Begin);
var reader = new StreamReader(stream);
return(reader.ReadToEnd());
}
}
public async Task ExportBackgroundAsync(ExportDataRequest request, ExportPushNotification pushNotification, IJobCancellationToken jobCancellationToken, PerformContext context)
{
ValidateParameters(pushNotification);
try
{
await _customerDataExporter.ExportAsync(request,
progressInfo => ProgressCallback(progressInfo, pushNotification, context),
new JobCancellationTokenWrapper(jobCancellationToken));
}
catch (JobAbortedException)
{
// job is aborted, do nothing
}
catch (Exception ex)
{
pushNotification.Errors.Add(ex.ExpandExceptionMessage());
}
finally
{
pushNotification.Description = "Export finished";
pushNotification.Finished = DateTime.UtcNow;
await _pushNotificationManager.SendAsync(pushNotification);
}
}
private string SerializeAndRead <T>(ExportDataRequest exportDataRequest, IEnumerable <T> items) where T : IExportable
{
exportDataRequest.ProviderConfig = exportDataRequest.ProviderConfig ?? new CsvProviderConfiguration()
{
};
using (var stream = new MemoryStream())
using (var writer = new StreamWriter(stream, Encoding.UTF8)
{
AutoFlush = true
})
{
using (var csvExportProvider = new CsvExportProvider(exportDataRequest))
{
foreach (var item in items)
{
csvExportProvider.WriteRecord(writer, item);
}
}
stream.Seek(0, SeekOrigin.Begin);
return(new StreamReader(stream).ReadToEnd());
}
}
/// <summary>Snippet for ExportDataAsync</summary>
public async Task ExportDataRequestObjectAsync()
{
// Snippet: ExportDataAsync(ExportDataRequest, CallSettings)
// Additional: ExportDataAsync(ExportDataRequest, CancellationToken)
// Create client
AutoMlClient autoMlClient = await AutoMlClient.CreateAsync();
// Initialize request argument(s)
ExportDataRequest request = new ExportDataRequest
{
Name = "",
OutputConfig = new OutputConfig(),
};
// Make the request
Operation <Empty, OperationMetadata> response = await autoMlClient.ExportDataAsync(request);
// Poll until the returned long-running operation is complete
Operation <Empty, OperationMetadata> completedResponse = await response.PollUntilCompletedAsync();
// Retrieve the operation result
Empty result = completedResponse.Result;
// Or get the name of the operation
string operationName = response.Name;
// This name can be stored, then the long-running operation retrieved later by name
Operation <Empty, OperationMetadata> retrievedResponse = await autoMlClient.PollOnceExportDataAsync(operationName);
// Check if the retrieved long-running operation has completed
if (retrievedResponse.IsCompleted)
{
// If it has completed, then access the result
Empty retrievedResult = retrievedResponse.Result;
}
// End snippet
}
public string ExportFuelOrdersByCustClient(ExportDataRequest model)
{
var table = ExportFuelOrdersByCust(model.ClientID, model.ListOfIDs, model.StartDate.ToShortDateString(), model.EndDate.ToShortDateString());
DataTable copyDataTable;
copyDataTable = table.Copy();
copyDataTable.PrimaryKey = null;
copyDataTable.Columns["Name"].SetOrdinal(0);
copyDataTable.Columns["RequestedBy"].SetOrdinal(1);
copyDataTable.Columns["TailNumber"].SetOrdinal(4);
copyDataTable.Columns["Product"].SetOrdinal(5);
copyDataTable.Columns["PPG"].SetOrdinal(6);
copyDataTable.Columns["DateRequested"].SetOrdinal(7);
copyDataTable.Columns["FuelingDateTime"].SetOrdinal(8);
copyDataTable.Columns["PostedRetail"].SetOrdinal(9);
copyDataTable.Columns["QuotedPPG"].SetOrdinal(10);
copyDataTable.Columns["InvoicedPPG"].SetOrdinal(11);
copyDataTable.Columns["QuotedVolume"].SetOrdinal(12);
copyDataTable.Columns["InvoicedVolume"].SetOrdinal(13);
copyDataTable.Columns["RampFee"].SetOrdinal(14);
copyDataTable.Columns["RampFeeWaivedAt"].SetOrdinal(15);
copyDataTable.Columns["InvoiceStatus"].SetOrdinal(16);
copyDataTable.Columns["CustNotes"].SetOrdinal(17);
string filePath = ExportHelper.CreateFilePath("FuelOrderTransactions.csv", model.ClientName);
CSVGenerator.GenerateCSV(copyDataTable, filePath);
return(ExportHelper.GetFilePathFromRoot(filePath));
}
public string ExportContacts(ExportDataRequest model)
{
var table = GetCompanies(model.ClientID, model.ListOfIDs);
string filePath = ExportHelper.CreateFilePath("Companies.csv", model.ClientName);
CSVGenerator.GenerateCSV(table, filePath);
return(ExportHelper.GetFilePathFromRoot(filePath));
}
public string ExportFuelOrdersByAdminClient(ExportDataRequest model)
{
var table = ExportFuelOrders(model.ClientID, model.ListOfIDs, model.StartDate.ToShortDateString(), model.EndDate.ToShortDateString());
string filePath = ExportHelper.CreateFilePath("FuelOrderTransactions.csv", model.ClientName);
CSVGenerator.GenerateCSV(table, filePath);
return(ExportHelper.GetFilePathFromRoot(filePath));
}
public void ExportBackground(ExportDataRequest request, ExportPushNotification notification, IJobCancellationToken cancellationToken, PerformContext context)
{
void progressCallback(ExportProgressInfo x)
{
notification.Patch(x);
notification.JobId = context.BackgroundJob.Id;
_pushNotificationManager.Upsert(notification);
}
try
{
var localTmpFolder = HostingEnvironment.MapPath(_defaultExportFolder);
var fileName = Path.Combine(localTmpFolder, string.Format(FileNameTemplate, DateTime.UtcNow));
// Do not like provider creation here to get file extension, maybe need to pass created provider to Exporter.
// Create stream inside Exporter is not good as it is not Exporter resposibility to decide where to write.
var provider = _exportProviderFactory.CreateProvider(request);
if (!string.IsNullOrEmpty(provider.ExportedFileExtension))
{
fileName = Path.ChangeExtension(string.Format(FileNameTemplate, DateTime.UtcNow), provider.ExportedFileExtension);
}
var localTmpPath = Path.Combine(localTmpFolder, fileName);
if (!Directory.Exists(localTmpFolder))
{
Directory.CreateDirectory(localTmpFolder);
}
if (File.Exists(localTmpPath))
{
File.Delete(localTmpPath);
}
//Import first to local tmp folder because Azure blob storage doesn't support some special file access mode
using (var stream = File.OpenWrite(localTmpPath))
{
_dataExporter.Export(stream, request, progressCallback, new JobCancellationTokenWrapper(cancellationToken));
notification.DownloadUrl = $"api/export/download/{fileName}";
}
}
catch (JobAbortedException)
{
//do nothing
}
catch (Exception ex)
{
notification.Errors.Add(ex.ExpandExceptionMessage());
}
finally
{
notification.Description = "Export finished";
notification.Finished = DateTime.UtcNow;
_pushNotificationManager.Upsert(notification);
}
}
public CustomerExportPagedDataSource(IMemberService memberService, IMemberSearchService memberSearchService, IStoreService storeService, int pageSize, ExportDataRequest request)
{
_memberService = memberService;
_memberSearchService = memberSearchService;
_storeService = storeService;
_request = request;
PageSize = pageSize;
}
public CsvExportProvider(ExportDataRequest exportDataRequest)
{
if (exportDataRequest == null)
{
throw new ArgumentNullException(nameof(exportDataRequest));
}
Configuration = exportDataRequest.ProviderConfig as CsvProviderConfiguration ?? new CsvProviderConfiguration();
IncludedProperties = exportDataRequest.DataQuery?.IncludedProperties;
}
public virtual IExportProvider CreateProvider(ExportDataRequest exportDataRequest)
{
if (exportDataRequest == null)
{
throw new ArgumentNullException(nameof(exportDataRequest));
}
var result = _providerFactories.FirstOrDefault(x => x(exportDataRequest).TypeName.EqualsInvariant(exportDataRequest.ProviderName));
return(result != null?result(exportDataRequest) : null);
}
private ExportDataRequest CreatExportDataRequest(ExportDataQuery dataQuery, IExportProviderConfiguration providerConfig = null)
{
var result = new ExportDataRequest()
{
ProviderName = nameof(CsvExportProvider),
DataQuery = dataQuery,
ProviderConfig = providerConfig,
};
return(result);
}
public async Task ExportDataV1_ShouldReturn()
{
using (var httpClient = new HttpClient())
{
var requestData = new ExportDataRequest();
var content = new StringContent(JsonConvert.SerializeObject(requestData), Encoding.UTF8, "application/json");
var result = await httpClient.PostAsync(new Uri(_url + "/v1/exportdata"), content);
result.EnsureSuccessStatusCode();
}
}
public string ExportByClientID(ExportDataRequest model)
{
if (!ModelState.IsValid)
{
return((Request.CreateErrorResponse(HttpStatusCode.BadRequest, ModelState)).ToString());
}
var table = GetAirports(model.ClientID, model.ListOfIDs);
string filePath = ExportHelper.CreateFilePath("Airports.csv", model.ClientName);
CSVGenerator.GenerateCSV(table, filePath);
return(ExportHelper.GetFilePathFromRoot(filePath));
}
private static string Export(ExportDataServiceClient client, DataType type)
{
var start = DateTime.Now.Date.Subtract(TimeSpan.FromDays(1)).Subtract(TimeSpan.FromSeconds(1));
var end = DateTime.Now.Date.Subtract(TimeSpan.FromSeconds(1));
var request = new ExportDataRequest()
{
StartDate = start, EndDate = end, Target = type
};
var resultTask = client.GetOrders(request);
resultTask.Wait();
return(resultTask.Result.ToString());
}
public async Task ExportBackgroundAsync(ExportDataRequest request, ExportPushNotification notification, IJobCancellationToken cancellationToken, PerformContext context)
{
void progressCallback(ExportProgressInfo x)
{
notification.Patch(x);
notification.JobId = context.BackgroundJob.Id;
_pushNotificationManager.Send(notification);
}
try
{
if (string.IsNullOrEmpty(_platformOptions.DefaultExportFolder))
{
throw new PlatformException($"{nameof(_platformOptions.DefaultExportFolder)} should be set.");
}
var fileName = string.Format(FileNameTemplate, DateTime.UtcNow);
// Do not like provider creation here to get file extension, maybe need to pass created provider to Exporter.
// Create stream inside Exporter is not good as it is not Exporter resposibility to decide where to write.
var provider = _exportProviderFactory.CreateProvider(request);
if (!string.IsNullOrEmpty(provider.ExportedFileExtension))
{
fileName = Path.ChangeExtension(fileName, provider.ExportedFileExtension);
}
var url = UrlHelperExtensions.Combine(_platformOptions.DefaultExportFolder, fileName);
using (var blobStream = _blobStorageProvider.OpenWrite(url))
{
_dataExporter.Export(blobStream, request, progressCallback, new JobCancellationTokenWrapper(cancellationToken));
}
notification.DownloadUrl = _blobUrlResolver.GetAbsoluteUrl(url);
}
catch (JobAbortedException)
{
//do nothing
}
catch (Exception ex)
{
notification.Errors.Add(ex.ExpandExceptionMessage());
}
finally
{
notification.Description = "Export finished";
notification.Finished = DateTime.UtcNow;
await _pushNotificationManager.SendAsync(notification);
}
}
public async Task <ActionResult <ExportPushNotification> > RunExport([FromBody] ExportDataRequest request)
{
var notification = new ExportPushNotification(_userNameResolver.GetCurrentUserName())
{
Title = "Customers export",
Description = "Starting export task..."
};
await _pushNotificationManager.SendAsync(notification);
notification.JobId = BackgroundJob.Enqueue <ExportJob>(importJob => importJob.ExportBackgroundAsync(request, notification, JobCancellationToken.Null, null));
return(Ok(notification));
}
public string ExportPrices(ExportDataRequest model /*int adminId, string adminName*/)
{
var table = GetSupplierFuelsPricesDataTable(model.ClientID);
DataTable copyDataTable;
copyDataTable = table.Copy();
copyDataTable.PrimaryKey = null;
copyDataTable.Columns.Remove("Id");
copyDataTable.Columns.Remove("AdminClientID");
copyDataTable.Columns.Remove("SupplierID");
copyDataTable.Columns.Remove("DateUploaded");
copyDataTable.Columns["EffectiveDate"].SetOrdinal(7);
string filePath = ExportHelper.CreateFilePath("VendorFuelsPrices.csv", model.ClientName);
CSVGenerator.GenerateCSV(copyDataTable, filePath);
return(ExportHelper.GetFilePathFromRoot(filePath));
}
public JsonExportProvider(ExportDataRequest exportDataRequest)
{
if (exportDataRequest == null)
{
throw new ArgumentNullException(nameof(exportDataRequest));
}
var jsonProviderConfiguration = exportDataRequest.ProviderConfig as JsonProviderConfiguration ?? new JsonProviderConfiguration();
Configuration = jsonProviderConfiguration;
IncludedProperties = exportDataRequest.DataQuery?.IncludedProperties;
var jsonSettings = jsonProviderConfiguration.Settings;
_serializer = JsonSerializer.Create(jsonSettings);
_serializer.Converters.Add(new ObjectDiscriminatorJsonConverter(jsonSettings, typeof(Entity)));
}
//[ResponseType(typeof(PlatformExportPushNotification))]
public IHttpActionResult RunExport([FromBody] ExportDataRequest request)
{
if (_exportSecurityHandlerRegistrar.GetHandler(request.ExportTypeName + "ExportDataPolicy")?.Authorize(User.Identity.Name, request) != true)
{
return(Unauthorized());
}
var notification = new ExportPushNotification(_userNameResolver.GetCurrentUserName())
{
Title = $"{request.ExportTypeName} export task",
Description = "starting export...."
};
//_pushNotificationManager.Send(notification);
var jobId = BackgroundJob.Enqueue <ExportJob>(x => x.ExportBackground(request, notification, JobCancellationToken.Null, null));
notification.JobId = jobId;
return(Ok(notification));
}
public IHttpActionResult GetData([FromBody] ExportDataRequest request)
{
if (_exportSecurityHandlerRegistrar.GetHandler(request.ExportTypeName + "ExportDataPolicy")?.Authorize(User.Identity.Name, request) != true)
{
return(Unauthorized());
}
var exportedTypeDefinition = _knownExportTypesResolver.ResolveExportedTypeDefinition(request.ExportTypeName);
var pagedDataSource = exportedTypeDefinition.ExportedDataSourceFactory(request.DataQuery);
pagedDataSource.Fetch();
var queryResult = pagedDataSource.Items;
var result = new ExportableSearchResult()
{
TotalCount = pagedDataSource.GetTotalCount(),
Results = queryResult.ToList()
};
return(Ok(result));
}
public async Task <ActionResult <ExportableSearchResult> > GetData([FromBody] ExportDataRequest request)
{
var authorizationResult = await _authorizationService.AuthorizeAsync(User, request.DataQuery, request.ExportTypeName + "ExportDataPolicy");
if (!authorizationResult.Succeeded)
{
return(Unauthorized());
}
var exportedTypeDefinition = _knownExportTypesResolver.ResolveExportedTypeDefinition(request.ExportTypeName);
var pagedDataSource = (exportedTypeDefinition.DataSourceFactory ?? throw new ArgumentNullException(nameof(ExportedTypeDefinition.DataSourceFactory))).Create(request.DataQuery);
pagedDataSource.Fetch();
var queryResult = pagedDataSource.Items;
var result = new ExportableSearchResult()
{
TotalCount = pagedDataSource.GetTotalCount(),
Results = queryResult.ToList()
};
return(Ok(result));
}
// ---
private async Task <T> ExportData <T>(ExportDataRequest ExportDataRequest)
{
if (string.IsNullOrEmpty(this.Config.AccessToken))
{
await this.RefreshToken();
}
var response = await this.HttpService.POST_ASYNC <ExportDataRequest, T>($"https://{this.Config.ServerName}/ExportDataApi/exportData", ExportDataRequest, headers : new Dictionary <string, string>
{
["Accept"] = "application/json",
["Content-Type"] = "application/json",
["Authorization"] = $"Bearer {this.Config.AccessToken}"
});
var status = this.ParseResponse(response.Content);
// Unauthorized - refresh token and try again
if (!status.Success && status.Error == "Token is not valid")
{
await this.RefreshToken();
response = await this.HttpService.POST_ASYNC <ExportDataRequest, T>($"https://{this.Config.ServerName}/ExportDataApi/exportData", ExportDataRequest, headers : new Dictionary <string, string>
{
["Accept"] = "application/json",
["Content-Type"] = "application/json",
["Authorization"] = $"Bearer {this.Config.AccessToken}"
});
status = this.ParseResponse(response.Content);
}
if (!status.Success)
{
throw new Exception(response.Content);
}
return(response.Model);
}
public JsonExportProvider(ExportDataRequest exportDataRequest)
{
if (exportDataRequest == null)
{
throw new ArgumentNullException(nameof(exportDataRequest));
}
var jsonProviderConfiguration = exportDataRequest.ProviderConfig as JsonProviderConfiguration ?? new JsonProviderConfiguration();
Configuration = jsonProviderConfiguration;
IncludedProperties = exportDataRequest.DataQuery?.IncludedProperties;
var jsonSettings = new JsonSerializerSettings()
{
NullValueHandling = NullValueHandling.Ignore,
DefaultValueHandling = DefaultValueHandling.Ignore,
ReferenceLoopHandling = ReferenceLoopHandling.Ignore,
Formatting = jsonProviderConfiguration.Indented ? Formatting.Indented : Formatting.None
};
_serializer = JsonSerializer.Create(jsonSettings);
_serializer.Converters.Add(new ObjectDiscriminatorJsonConverter(jsonSettings, typeof(Entity)));
}
public async Task <ExportDataResponse> ExportDataAsync(ExportDataRequest request) => await Task.FromResult(new ExportDataResponse());
public void Export(Stream stream, ExportDataRequest request, Action <ExportProgressInfo> progressCallback, ICancellationToken token)
{
if (request == null)
{
throw new ArgumentNullException(nameof(request));
}
token.ThrowIfCancellationRequested();
var exportedTypeDefinition = _exportTypesResolver.ResolveExportedTypeDefinition(request.ExportTypeName);
var pagedDataSource = (exportedTypeDefinition.DataSourceFactory ?? throw new ArgumentNullException(nameof(ExportedTypeDefinition.DataSourceFactory))).Create(request.DataQuery);
var completedMessage = $"Export completed";
var totalCount = pagedDataSource.GetTotalCount();
var exportedCount = 0;
var exportProgress = new ExportProgressInfo()
{
ProcessedCount = 0,
TotalCount = totalCount,
Description = "Export has started",
};
progressCallback(exportProgress);
try
{
exportProgress.Description = "Creating provider…";
progressCallback(exportProgress);
using (var writer = new StreamWriter(stream, Encoding.UTF8, 1024, true)
{
AutoFlush = true
})
using (var exportProvider = _exportProviderFactory.CreateProvider(request))
{
var needTabularData = exportProvider.IsTabular;
if (needTabularData && !exportedTypeDefinition.IsTabularExportSupported)
{
throw new NotSupportedException($"Provider \"{exportProvider.TypeName}\" does not support tabular export.");
}
exportProgress.Description = "Fetching…";
progressCallback(exportProgress);
while (pagedDataSource.Fetch())
{
token.ThrowIfCancellationRequested();
var objectBatch = pagedDataSource.Items;
foreach (var obj in objectBatch)
{
try
{
var preparedObject = obj.Clone() as IExportable;
request.DataQuery.FilterProperties(preparedObject);
if (needTabularData)
{
preparedObject = (preparedObject as ITabularConvertible)?.ToTabular() ??
throw new NotSupportedException($"Object should be {nameof(ITabularConvertible)} to be exported using tabular provider.");
}
exportProvider.WriteRecord(writer, preparedObject);
}
catch (Exception e)
{
exportProgress.Errors.Add(e.Message);
progressCallback(exportProgress);
}
exportedCount++;
}
exportProgress.ProcessedCount = exportedCount;
if (exportedCount != totalCount)
{
exportProgress.Description = $"{exportedCount} out of {totalCount} have been exported.";
progressCallback(exportProgress);
}
}
}
}
catch (Exception e)
{
exportProgress.Errors.Add(e.Message);
}
finally
{
if (exportProgress.Errors.Count > 0)
{
completedMessage = $"Export completed with errors";
}
exportProgress.Description = $"{completedMessage}: {exportedCount} out of {totalCount} have been exported.";
progressCallback(exportProgress);
}
}
/// <summary>
/// 数据导出
/// </summary>
/// <param name="request">请求参数信息</param>
/// <returns>请求结果信息</returns>
public ExportDataResponse ExportData(ExportDataRequest request)
{
return(new ExportDataExecutor().Client(this).Execute <ExportDataResponse, ExportDataResult, ExportDataRequest>(request));
}
/// <summary>
/// 数据导出
/// </summary>
/// <param name="request">请求参数信息</param>
/// <returns>请求结果信息</returns>
public async Task <ExportDataResponse> ExportData(ExportDataRequest request)
{
return(await new ExportDataExecutor().Client(this).Execute <ExportDataResponse, ExportDataResult, ExportDataRequest>(request).ConfigureAwait(false));
}
