private static async Task <Func <MemoryStream, bool, int> > GetDataChunkUploader(this IFileMedia fileMedia, IFileConfiguration fileConfig, Interfaces.ILog log)
{
var getContext = await fileMedia.GetWaveContextFunc(fileConfig.Name, log);
var xmdJson = fileConfig.GetMetadataBuilder();
var chunkNo = 0;
return((stream, isFinalizing) =>
{
var context = getContext();
if (string.IsNullOrWhiteSpace(context.SetId))
{
context.InitiateDatasetUpload(xmdJson(), fileMedia.Operation, log).Wait();
if (string.IsNullOrWhiteSpace(context.SetId))
{
throw new ImporterException(
Localization.GetLocalizationString("Could not get job id from wave, cannot upload chunks"));
}
}
stream.Flush();
var encCSVchunk = Convert.ToBase64String(stream.ToArray());
var payload =
$"{{\"InsightsExternalDataId\":\"{context.SetId}\",\"PartNumber\":{++chunkNo},\"DataFile\":\"{encCSVchunk}\"}}";
var tryCount = 0;
while (true)
{
try
{
var client = new HttpClient();
var content = new StringContent(payload, Encoding.ASCII);
content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
var url = $"{context.EntryPoint}/services/data/v41.0/sobjects/InsightsExternalDataPart";
client.AddAuthorization(context);
log.Debug($"Uploading chunk #{chunkNo}");
var response = client.PostAsync(url, content).Result;
if (response.IsSuccessStatusCode)
{
log.Debug($"Uploaded chunk #{chunkNo}");
if (isFinalizing)
{
context.FinalizeDatasetUpload(log).Wait();
}
return chunkNo;
}
}
catch (Exception ex)
{
log.Error(Localization.GetLocalizationString("Error while uploading chunk #{0} - {1}", chunkNo, ex.Message));
log.Debug(ex.ToString());
}
if (++tryCount > 4)
{
throw new ImporterUploadException(Localization.GetLocalizationString("Failed to upload dataset."));
}
log.Debug(Localization.GetLocalizationString("Retrying to upload chunk#{0}", chunkNo));
}
});
}