private static async Task InitiateDatasetUpload(this WaveContext context, string xmdJson, DataOperation operation, Interfaces.ILog log)
{
var url = $"{context.EntryPoint}/services/data/v41.0/sobjects/InsightsExternalData";
log.Info(Localization.GetLocalizationString("Initializing data upload"));
var client = new HttpClient();
var encJson = Convert.ToBase64String(Encoding.UTF8.GetBytes(xmdJson));
var payload =
$"{{\"Format\":\"csv\",\"EdgemartAlias\":\"{context.Alias}\",\"Operation\":\"{operation.ToString()}\",\"Action\":\"None\",\"MetadataJson\":\"{encJson}\"}}";
var content = new StringContent(payload, Encoding.ASCII);
content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
client.AddAuthorization(context);
var response = await client.PostAsync(url, content);
var responseType = new { id = "" };
var responseText = await response.Content.ReadAsStringAsync();
log.Debug($"Received {responseText}");
if (response.IsSuccessStatusCode)
{
context.SetId = JsonConvert.DeserializeAnonymousType(responseText, responseType).id;
log.Info(Localization.GetLocalizationString("Received job id {0}", context.SetId));
}
}
private static async Task <Func <WaveContext> > GetWaveContextFunc(this IFileMedia fileMedia, string fileName, Interfaces.ILog log)
{
var client = new HttpClient();
var url = $"{fileMedia.ConnectionCredentials.EntryPoint}/services/oauth2/token?grant_type=password&client_id={fileMedia.ConnectionCredentials.ClientId}&client_secret={fileMedia.ConnectionCredentials.ClientSecret}&username={fileMedia.ConnectionCredentials.Login}&password={fileMedia.ConnectionCredentials.Password}{fileMedia.ConnectionCredentials.Token}";
var content = new StringContent(string.Empty);
content.Headers.ContentType = new MediaTypeHeaderValue("application/x-www-form-urlencoded");
log.Info(string.Format(
Localization.GetLocalizationString("Obtaining access token and entry point from \"{0}\" for \"{1}\""),
fileMedia.ConnectionCredentials.EntryPoint, fileMedia.ConnectionCredentials.Login));
var response = await client.PostAsync(url, content);
WaveContext context = null;
if (response.IsSuccessStatusCode)
{
log.Info(Localization.GetLocalizationString("Retrieved entry point and access token."));
var responseType = new { access_token = "", instance_url = "" };
var result = JsonConvert.DeserializeAnonymousType(await response.Content.ReadAsStringAsync(), responseType);
context = new WaveContext
{
Token = result.access_token,
EntryPoint = result.instance_url,
Alias = fileName.Replace(" ", "_")
};
}
else
{
log.Fatal($"{response.StatusCode.ToString()}-{response.Content.ReadAsStringAsync()}");
throw new UnauthorizedAccessException(Localization.GetLocalizationString("Could not get access to wave entry point."));
}
return(() => context);
}
public static Func <IDataRow> ParseData(this Func <ISourceRow> getLineFunc, IFileConfiguration fileConfig, Interfaces.ILog logger)
{
if (fileConfig == null)
{
var msg = Localization.GetLocalizationString("Could not get Source Configuration...");
logger?.Fatal(msg);
throw new ArgumentException(msg);
}
logger?.Info(string.Format(Localization.GetLocalizationString("Parsing data from {0}"), fileConfig.Name));
var parsers = fileConfig.GetRowParsers();
var currentRecord = (long)0;
var parsedRecords = (long)0;
return(() =>
{
var line = getLineFunc();
if (line != null)
{
currentRecord++;
try
{
var row = parsers(line, currentRecord, currentRecord);
if (row != null)
{
parsedRecords++;
return row;
}
}
catch (Exception ex)
{
logger?.Error(Localization.GetLocalizationString("Failed to parse line: \"{0}\"", string.Join(",", line.Fields.Select(x => x.Source))));
throw ex;
}
return new DataRow(
new Dictionary <string, IValue>
{
{
"raw",
new ValueWrapper <string>(string.Join(",", line.Fields.Select(x => x.Source)),
Localization.GetLocalizationString("Parse error"), true, string.Join(",", line.Fields.Select(x => x.Source)))
}
},
Localization.GetLocalizationString("Could not parse line."), currentRecord, line.LineNumber, line.Context.SourcePath, line.Context.FileConfiguration.Name);
}
logger?.Info(string.Format(Localization.GetLocalizationString("Parsed {0}/{1} records from {2}"), parsedRecords, currentRecord, fileConfig.Name));
return null;
});
}
private static Action <IDataRow> GetWriter(this IFile file, Interfaces.ILog log)
{
var stream = file.GetWriterStreams(log);
var rowCount = (long)0;
if (stream != null)
{
switch (file.Format)
{
case FileFormat.CSV:
var writerStream = stream.GetCsvWriter(file, log);
return(row =>
{
if (row == null)
{
log?.Info(string.Format(
Localization.GetLocalizationString("{0} line(s) written to {1}"),
rowCount, file.Name));
stream.Flush();
return;
}
if (row.SourceName == file.Name)
{
rowCount = writerStream(row);
}
});
}
}
return(row => { });
}
public static Func <ISourceRow> FixedWidthReader(this Func <int> readNext, ISourceFileContext context, Interfaces.ILog logger)
{
if (context?.FileConfiguration == null)
{
var msg = Localization.GetLocalizationString("Could not get Source Configuration...");
logger?.Fatal(msg);
throw new ArgumentException(msg);
}
var locker = new object();
var rowCount = 0;
logger?.Info(string.Format(Localization.GetLocalizationString("Loading data from {0}..."), context.FileConfiguration.Name));
var lineReaders = context.FileConfiguration.Rows.Select(x => x.GetLineFunc(context.FileConfiguration)).ToList();
return(() =>
{
lock (locker)
{
var result = new List <ISourceField>();
foreach (var reader in lineReaders)
{
result.AddRange(reader(readNext));
}
return result.Count > 0 ? new SourceRow {
Context = context, Fields = result, LineNumber = ++rowCount
} : null;
}
});
}
private static async Task FinalizeDatasetUpload(this WaveContext context, Interfaces.ILog log)
{
var client = new HttpClient();
var url = $"{context.EntryPoint}/services/data/v36.0/sobjects/InsightsExternalData/{context.SetId}";
log.Info(Localization.GetLocalizationString("Finalizing upload job {0}", context.SetId));
var content = new StringContent("{\"Action\" : \"Process\"}", Encoding.ASCII);
content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
client.AddAuthorization(context);
var response = await client.PatchAsync(url, content);
if (response.IsSuccessStatusCode)
{
log.Info(Localization.GetLocalizationString("Successfully uploaded {0}", context.Alias));
return;
}
var responseText = await response.Content.ReadAsStringAsync();
throw new ImporterUploadException($"{response.StatusCode} -{responseText}");
}
private static async Task <Func <MemoryStream, bool, int> > GetDataSenderFunc(this IFileMedia fileMedia, IFileConfiguration fileConfig, Interfaces.ILog log)
{
var chunkNumber = 0;
var queue = new ConcurrentQueue <MemoryStream>();
var isFinalizing = false;
var uploader = await fileMedia.GetDataChunkUploader(fileConfig, log);
var senderTask = new Task(() =>
{
while (true)
{
while (queue.TryDequeue(out var nextChunk))
{
try
{
chunkNumber = uploader(nextChunk, isFinalizing && queue.IsEmpty);
}
catch
{
log.Fatal(Localization.GetLocalizationString("Upload has terminated"));
return;
}
}
if (isFinalizing && queue.IsEmpty)
{
return;
}
Thread.Sleep(100);
}
});
senderTask.Start();
return((stream, finalize) =>
{
if (stream != null)
{
queue.Enqueue(stream);
while (queue.Count > 3)
{
Thread.Sleep(50);
}
}
isFinalizing = finalize;
if (isFinalizing)
{
log.Info(Localization.GetLocalizationString("Awaiting for upload to finish."));
senderTask.Wait();
}
return chunkNumber;
});
}
public static Func <ISourceRow> CsvReader(this Func <int> readNext, ISourceFileContext context, Interfaces.ILog logger)
{
var fileConfig = context.FileConfiguration;
if (fileConfig == null)
{
var msg = Localization.GetLocalizationString("Could not get Source Configuration...");
logger?.Fatal(msg);
throw new ArgumentException(msg);
}
var nullValue = string.IsNullOrWhiteSpace(fileConfig.NullValue) ? "" : fileConfig.NullValue;
var delimiter = string.IsNullOrWhiteSpace(fileConfig.Delimiter) ? ',' : fileConfig.Delimiter[0];
var qualifier = string.IsNullOrWhiteSpace(fileConfig.Qualifier) ? '\"' : fileConfig.Qualifier[0];
var locker = new object();
var rowCount = 0;
logger?.Info(string.Format(Localization.GetLocalizationString("Loading data from {0}..."), fileConfig.Name));
return(() =>
{
lock (locker)
{
var columns = new List <ISourceField>();
var hadQualifier = false;
Action <StringBuilder> enqueue = clmn =>
{
var value = clmn.ToString();
columns.Add((value.Length > 0 && (value != nullValue || hadQualifier)) ? new SourceField(value) : new SourceField(null));
};
int c;
while ((c = readNext()) >= 0 && (c == '\n' || c == '\r'))
{
;
}
if (c < 0)
{
logger?.Info(Localization.GetLocalizationString("Loaded {0} line(s) from {1}.", rowCount, fileConfig.Name));
return null;
}
var isQualified = false;
var column = new StringBuilder();
while (c >= 0)
{
if (c == qualifier && (column.Length == 0 || hadQualifier))
{
hadQualifier = true;
isQualified = !isQualified;
if (isQualified && column.Length > 0)
{
column.Append((char)c);
}
}
else
if (c == delimiter && !isQualified)
{
enqueue(column);
column = new StringBuilder();
}
else
if ((c == '\n' || c == '\r') && !isQualified)
{
enqueue(column);
column = null;
break;
}
else
{
column.Append((char)c);
}
c = readNext();
}
if (c == -1 && column != null)
{
enqueue(column);
}
rowCount++;
return new SourceRow {
Context = context, Fields = columns, LineNumber = rowCount
};
}
});
}
