private static Func <ISourceFileContext> GetSourceStream(this IFileMedia fileMedia, IFileConfiguration fileConfig, Interfaces.ILog logger)
{
logger?.Debug($"Getting source stream(s) for media \"{fileMedia.MediaType.ToString()}\" - \"{fileMedia.Path}\"");
switch (fileMedia.MediaType)
{
case MediaType.Local:
var directoryName = Path.GetDirectoryName(Path.GetFullPath(fileMedia.Path));
logger?.Debug($"Searching for local file(s) \"{Path.GetFullPath(fileMedia.Path)}\"");
var files = Directory.EnumerateFiles(directoryName, Path.GetFileName(fileMedia.Path),
fileMedia.IncludeSubfolders ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly).OrderBy(x => x).ToList();
logger?.Debug($"Found {files.Count} file(s) matching the pattern.");
foreach (var file in files)
{
logger?.Debug(file);
}
var getFile = files.GetNextFunc();
return(() =>
{
var file = getFile();
return new SourceFileContext
{
SourcePath = file,
Stream = file.GetLocalFileStream(fileMedia.Encoding.GetFilEncoding(), logger),
FileMedia = fileMedia,
FileConfiguration = fileConfig
};
});
}
return(null);
}
private static async Task <Func <MemoryStream, bool, int> > GetDataChunkUploader(this IFileMedia fileMedia, IFileConfiguration fileConfig, Interfaces.ILog log)
{
var getContext = await fileMedia.GetWaveContextFunc(fileConfig.Name, log);
var xmdJson = fileConfig.GetMetadataBuilder();
var chunkNo = 0;
return((stream, isFinalizing) =>
{
var context = getContext();
if (string.IsNullOrWhiteSpace(context.SetId))
{
context.InitiateDatasetUpload(xmdJson(), fileMedia.Operation, log).Wait();
if (string.IsNullOrWhiteSpace(context.SetId))
{
throw new ImporterException(
Localization.GetLocalizationString("Could not get job id from wave, cannot upload chunks"));
}
}
stream.Flush();
var encCSVchunk = Convert.ToBase64String(stream.ToArray());
var payload =
$"{{\"InsightsExternalDataId\":\"{context.SetId}\",\"PartNumber\":{++chunkNo},\"DataFile\":\"{encCSVchunk}\"}}";
var tryCount = 0;
while (true)
{
try
{
var client = new HttpClient();
var content = new StringContent(payload, Encoding.ASCII);
content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
var url = $"{context.EntryPoint}/services/data/v41.0/sobjects/InsightsExternalDataPart";
client.AddAuthorization(context);
log.Debug($"Uploading chunk #{chunkNo}");
var response = client.PostAsync(url, content).Result;
if (response.IsSuccessStatusCode)
{
log.Debug($"Uploaded chunk #{chunkNo}");
if (isFinalizing)
{
context.FinalizeDatasetUpload(log).Wait();
}
return chunkNo;
}
}
catch (Exception ex)
{
log.Error(Localization.GetLocalizationString("Error while uploading chunk #{0} - {1}", chunkNo, ex.Message));
log.Debug(ex.ToString());
}
if (++tryCount > 4)
{
throw new ImporterUploadException(Localization.GetLocalizationString("Failed to upload dataset."));
}
log.Debug(Localization.GetLocalizationString("Retrying to upload chunk#{0}", chunkNo));
}
});
}
private static async Task InitiateDatasetUpload(this WaveContext context, string xmdJson, DataOperation operation, Interfaces.ILog log)
{
var url = $"{context.EntryPoint}/services/data/v41.0/sobjects/InsightsExternalData";
log.Info(Localization.GetLocalizationString("Initializing data upload"));
var client = new HttpClient();
var encJson = Convert.ToBase64String(Encoding.UTF8.GetBytes(xmdJson));
var payload =
$"{{\"Format\":\"csv\",\"EdgemartAlias\":\"{context.Alias}\",\"Operation\":\"{operation.ToString()}\",\"Action\":\"None\",\"MetadataJson\":\"{encJson}\"}}";
var content = new StringContent(payload, Encoding.ASCII);
content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
client.AddAuthorization(context);
var response = await client.PostAsync(url, content);
var responseType = new { id = "" };
var responseText = await response.Content.ReadAsStringAsync();
log.Debug($"Received {responseText}");
if (response.IsSuccessStatusCode)
{
context.SetId = JsonConvert.DeserializeAnonymousType(responseText, responseType).id;
log.Info(Localization.GetLocalizationString("Received job id {0}", context.SetId));
}
}
private static StreamReader GetLocalFileStream(this string filePath, Encoding encoding, Interfaces.ILog logger)
{
if (string.IsNullOrWhiteSpace(filePath))
{
return(null);
}
logger.Debug(string.Format(Localization.GetLocalizationString("Opening source file \"{0}\""),
filePath));
return(new StreamReader(File.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.Read), encoding));
}
public static Func <int> BufferedRead(this ISourceFileContext r, Interfaces.ILog logger)
{
if (r?.Stream == null)
{
throw new ImporterArgumentOutOfRangeException(Localization.GetLocalizationString("Source stream cannot be null"));
}
var buff = new char[60];
var position = 0;
var length = 0;
var rd = r.Stream;
var readChars = (long)0;
var resultMessage = new Action(() =>
{
if (readChars > 0)
{
logger?.Debug($"Loaded {readChars} characters");
}
});
return(() =>
{
if (position >= length)
{
if (rd == null || rd.EndOfStream)
{
resultMessage();
readChars = 0;
return -1;
}
length = rd.ReadBlock(buff, 0, 5);
readChars += length;
position = 0;
if (length == 0)
{
return -1;
}
}
return buff[position++];
});
}
