public WaveStream(IFileMedia fileMedia, IFileConfiguration fileConfig, Interfaces.ILog log)
{
this.logger = log;
this.fileConfig = fileConfig;
this.ms = new MemoryStream();
this.writeToWaveFunc = fileMedia.GetDataSenderFunc(fileConfig, log).Result;
}
private static async Task <Func <WaveContext> > GetWaveContextFunc(this IFileMedia fileMedia, string fileName, Interfaces.ILog log)
{
var client = new HttpClient();
var url = $"{fileMedia.ConnectionCredentials.EntryPoint}/services/oauth2/token?grant_type=password&client_id={fileMedia.ConnectionCredentials.ClientId}&client_secret={fileMedia.ConnectionCredentials.ClientSecret}&username={fileMedia.ConnectionCredentials.Login}&password={fileMedia.ConnectionCredentials.Password}{fileMedia.ConnectionCredentials.Token}";
var content = new StringContent(string.Empty);
content.Headers.ContentType = new MediaTypeHeaderValue("application/x-www-form-urlencoded");
log.Info(string.Format(
Localization.GetLocalizationString("Obtaining access token and entry point from \"{0}\" for \"{1}\""),
fileMedia.ConnectionCredentials.EntryPoint, fileMedia.ConnectionCredentials.Login));
var response = await client.PostAsync(url, content);
WaveContext context = null;
if (response.IsSuccessStatusCode)
{
log.Info(Localization.GetLocalizationString("Retrieved entry point and access token."));
var responseType = new { access_token = "", instance_url = "" };
var result = JsonConvert.DeserializeAnonymousType(await response.Content.ReadAsStringAsync(), responseType);
context = new WaveContext
{
Token = result.access_token,
EntryPoint = result.instance_url,
Alias = fileName.Replace(" ", "_")
};
}
else
{
log.Fatal($"{response.StatusCode.ToString()}-{response.Content.ReadAsStringAsync()}");
throw new UnauthorizedAccessException(Localization.GetLocalizationString("Could not get access to wave entry point."));
}
return(() => context);
}
private static Func <ISourceFileContext> GetSourceStream(this IFileMedia fileMedia, IFileConfiguration fileConfig, Interfaces.ILog logger)
{
logger?.Debug($"Getting source stream(s) for media \"{fileMedia.MediaType.ToString()}\" - \"{fileMedia.Path}\"");
switch (fileMedia.MediaType)
{
case MediaType.Local:
var directoryName = Path.GetDirectoryName(Path.GetFullPath(fileMedia.Path));
logger?.Debug($"Searching for local file(s) \"{Path.GetFullPath(fileMedia.Path)}\"");
var files = Directory.EnumerateFiles(directoryName, Path.GetFileName(fileMedia.Path),
fileMedia.IncludeSubfolders ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly).OrderBy(x => x).ToList();
logger?.Debug($"Found {files.Count} file(s) matching the pattern.");
foreach (var file in files)
{
logger?.Debug(file);
}
var getFile = files.GetNextFunc();
return(() =>
{
var file = getFile();
return new SourceFileContext
{
SourcePath = file,
Stream = file.GetLocalFileStream(fileMedia.Encoding.GetFilEncoding(), logger),
FileMedia = fileMedia,
FileConfiguration = fileConfig
};
});
}
return(null);
}
private static Stream GetWriterStream(this IFileMedia media, IFileConfiguration fileConfig, Interfaces.ILog log)
{
switch (media.MediaType)
{
case MediaType.Local:
return(media.GetLocalStream(log));
case MediaType.Wave:
return(media.GetWaveStreamWriter(fileConfig, log));
}
return(null);
}
private static Stream GetLocalStream(this IFileMedia media, Interfaces.ILog log)
{
if (File.Exists(media.Path) && media.Operation != DataOperation.Append)
{
try
{
File.Delete(media.Path);
}
catch (Exception ex)
{
log.Error(ex.ToString());
return(null);
}
}
return(File.Open(media.Path, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read));
}
private static async Task <Func <MemoryStream, bool, int> > GetDataSenderFunc(this IFileMedia fileMedia, IFileConfiguration fileConfig, Interfaces.ILog log)
{
var chunkNumber = 0;
var queue = new ConcurrentQueue <MemoryStream>();
var isFinalizing = false;
var uploader = await fileMedia.GetDataChunkUploader(fileConfig, log);
var senderTask = new Task(() =>
{
while (true)
{
while (queue.TryDequeue(out var nextChunk))
{
try
{
chunkNumber = uploader(nextChunk, isFinalizing && queue.IsEmpty);
}
catch
{
log.Fatal(Localization.GetLocalizationString("Upload has terminated"));
return;
}
}
if (isFinalizing && queue.IsEmpty)
{
return;
}
Thread.Sleep(100);
}
});
senderTask.Start();
return((stream, finalize) =>
{
if (stream != null)
{
queue.Enqueue(stream);
while (queue.Count > 3)
{
Thread.Sleep(50);
}
}
isFinalizing = finalize;
if (isFinalizing)
{
log.Info(Localization.GetLocalizationString("Awaiting for upload to finish."));
senderTask.Wait();
}
return chunkNumber;
});
}
private static async Task <Func <MemoryStream, bool, int> > GetDataChunkUploader(this IFileMedia fileMedia, IFileConfiguration fileConfig, Interfaces.ILog log)
{
var getContext = await fileMedia.GetWaveContextFunc(fileConfig.Name, log);
var xmdJson = fileConfig.GetMetadataBuilder();
var chunkNo = 0;
return((stream, isFinalizing) =>
{
var context = getContext();
if (string.IsNullOrWhiteSpace(context.SetId))
{
context.InitiateDatasetUpload(xmdJson(), fileMedia.Operation, log).Wait();
if (string.IsNullOrWhiteSpace(context.SetId))
{
throw new ImporterException(
Localization.GetLocalizationString("Could not get job id from wave, cannot upload chunks"));
}
}
stream.Flush();
var encCSVchunk = Convert.ToBase64String(stream.ToArray());
var payload =
$"{{\"InsightsExternalDataId\":\"{context.SetId}\",\"PartNumber\":{++chunkNo},\"DataFile\":\"{encCSVchunk}\"}}";
var tryCount = 0;
while (true)
{
try
{
var client = new HttpClient();
var content = new StringContent(payload, Encoding.ASCII);
content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
var url = $"{context.EntryPoint}/services/data/v41.0/sobjects/InsightsExternalDataPart";
client.AddAuthorization(context);
log.Debug($"Uploading chunk #{chunkNo}");
var response = client.PostAsync(url, content).Result;
if (response.IsSuccessStatusCode)
{
log.Debug($"Uploaded chunk #{chunkNo}");
if (isFinalizing)
{
context.FinalizeDatasetUpload(log).Wait();
}
return chunkNo;
}
}
catch (Exception ex)
{
log.Error(Localization.GetLocalizationString("Error while uploading chunk #{0} - {1}", chunkNo, ex.Message));
log.Debug(ex.ToString());
}
if (++tryCount > 4)
{
throw new ImporterUploadException(Localization.GetLocalizationString("Failed to upload dataset."));
}
log.Debug(Localization.GetLocalizationString("Retrying to upload chunk#{0}", chunkNo));
}
});
}
public static Stream GetWaveStreamWriter(this IFileMedia fileMedia, IFileConfiguration fileConfig, Interfaces.ILog log)
{
return(new WaveStream(fileMedia, fileConfig, log));
}
