private static Action <IDataRow> GetWriter(this IFile file, Interfaces.ILog log)
{
var stream = file.GetWriterStreams(log);
var rowCount = (long)0;
if (stream != null)
{
switch (file.Format)
{
case FileFormat.CSV:
var writerStream = stream.GetCsvWriter(file, log);
return(row =>
{
if (row == null)
{
log?.Info(string.Format(
Localization.GetLocalizationString("{0} line(s) written to {1}"),
rowCount, file.Name));
stream.Flush();
return;
}
if (row.SourceName == file.Name)
{
rowCount = writerStream(row);
}
});
}
}
return(row => { });
}
public static Func <IDataRow> GetRecordGroupingFunc(this Func <IDataRow> getLineFunc, IFileConfiguration fileConfig,
Interfaces.ILog logger)
{
var getRow = getLineFunc.GroupRowsFunc(fileConfig.Rows[0], logger);
return(() => getRow());
}
private static StreamWriter GetWriterStreams(this IFile file, Interfaces.ILog log)
{
var streams = new List <Stream>();
if (file.MultipleMedia?.Count > 0)
{
streams = file.MultipleMedia.Where(x => !x.Disabled).Select(x => x.GetWriterStream(file, log)).ToList();
}
else if (!file.Media.Disabled)
{
streams.Add(file.Media.GetWriterStream(file, log));
}
return(new StreamWriter(new WriteStreamProxy(streams), Encoding.UTF8));
}
private static Stream GetLocalStream(this IFileMedia media, Interfaces.ILog log)
{
if (File.Exists(media.Path) && media.Operation != DataOperation.Append)
{
try
{
File.Delete(media.Path);
}
catch (Exception ex)
{
log.Error(ex.ToString());
return(null);
}
}
return(File.Open(media.Path, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read));
}
public static Func <int> BufferedRead(this ISourceFileContext r, Interfaces.ILog logger)
{
if (r?.Stream == null)
{
throw new ImporterArgumentOutOfRangeException(Localization.GetLocalizationString("Source stream cannot be null"));
}
var buff = new char[60];
var position = 0;
var length = 0;
var rd = r.Stream;
var readChars = (long)0;
var resultMessage = new Action(() =>
{
if (readChars > 0)
{
logger?.Debug($"Loaded {readChars} characters");
}
});
return(() =>
{
if (position >= length)
{
if (rd == null || rd.EndOfStream)
{
resultMessage();
readChars = 0;
return -1;
}
length = rd.ReadBlock(buff, 0, 5);
readChars += length;
position = 0;
if (length == 0)
{
return -1;
}
}
return buff[position++];
});
}
private static async Task FinalizeDatasetUpload(this WaveContext context, Interfaces.ILog log)
{
var client = new HttpClient();
var url = $"{context.EntryPoint}/services/data/v36.0/sobjects/InsightsExternalData/{context.SetId}";
log.Info(Localization.GetLocalizationString("Finalizing upload job {0}", context.SetId));
var content = new StringContent("{\"Action\" : \"Process\"}", Encoding.ASCII);
content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
client.AddAuthorization(context);
var response = await client.PatchAsync(url, content);
if (response.IsSuccessStatusCode)
{
log.Info(Localization.GetLocalizationString("Successfully uploaded {0}", context.Alias));
return;
}
var responseText = await response.Content.ReadAsStringAsync();
throw new ImporterUploadException($"{response.StatusCode} -{responseText}");
}
private static async Task InitiateDatasetUpload(this WaveContext context, string xmdJson, DataOperation operation, Interfaces.ILog log)
{
var url = $"{context.EntryPoint}/services/data/v41.0/sobjects/InsightsExternalData";
log.Info(Localization.GetLocalizationString("Initializing data upload"));
var client = new HttpClient();
var encJson = Convert.ToBase64String(Encoding.UTF8.GetBytes(xmdJson));
var payload =
$"{{\"Format\":\"csv\",\"EdgemartAlias\":\"{context.Alias}\",\"Operation\":\"{operation.ToString()}\",\"Action\":\"None\",\"MetadataJson\":\"{encJson}\"}}";
var content = new StringContent(payload, Encoding.ASCII);
content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
client.AddAuthorization(context);
var response = await client.PostAsync(url, content);
var responseType = new { id = "" };
var responseText = await response.Content.ReadAsStringAsync();
log.Debug($"Received {responseText}");
if (response.IsSuccessStatusCode)
{
context.SetId = JsonConvert.DeserializeAnonymousType(responseText, responseType).id;
log.Info(Localization.GetLocalizationString("Received job id {0}", context.SetId));
}
}
public void AddLogger(Interfaces.ILog logger)
{
loggers.Add(logger);
}
public static Func <ISourceRow> CsvReader(this Func <int> readNext, ISourceFileContext context, Interfaces.ILog logger)
{
var fileConfig = context.FileConfiguration;
if (fileConfig == null)
{
var msg = Localization.GetLocalizationString("Could not get Source Configuration...");
logger?.Fatal(msg);
throw new ArgumentException(msg);
}
var nullValue = string.IsNullOrWhiteSpace(fileConfig.NullValue) ? "" : fileConfig.NullValue;
var delimiter = string.IsNullOrWhiteSpace(fileConfig.Delimiter) ? ',' : fileConfig.Delimiter[0];
var qualifier = string.IsNullOrWhiteSpace(fileConfig.Qualifier) ? '\"' : fileConfig.Qualifier[0];
var locker = new object();
var rowCount = 0;
logger?.Info(string.Format(Localization.GetLocalizationString("Loading data from {0}..."), fileConfig.Name));
return(() =>
{
lock (locker)
{
var columns = new List <ISourceField>();
var hadQualifier = false;
Action <StringBuilder> enqueue = clmn =>
{
var value = clmn.ToString();
columns.Add((value.Length > 0 && (value != nullValue || hadQualifier)) ? new SourceField(value) : new SourceField(null));
};
int c;
while ((c = readNext()) >= 0 && (c == '\n' || c == '\r'))
{
;
}
if (c < 0)
{
logger?.Info(Localization.GetLocalizationString("Loaded {0} line(s) from {1}.", rowCount, fileConfig.Name));
return null;
}
var isQualified = false;
var column = new StringBuilder();
while (c >= 0)
{
if (c == qualifier && (column.Length == 0 || hadQualifier))
{
hadQualifier = true;
isQualified = !isQualified;
if (isQualified && column.Length > 0)
{
column.Append((char)c);
}
}
else
if (c == delimiter && !isQualified)
{
enqueue(column);
column = new StringBuilder();
}
else
if ((c == '\n' || c == '\r') && !isQualified)
{
enqueue(column);
column = null;
break;
}
else
{
column.Append((char)c);
}
c = readNext();
}
if (c == -1 && column != null)
{
enqueue(column);
}
rowCount++;
return new SourceRow {
Context = context, Fields = columns, LineNumber = rowCount
};
}
});
}
private static async Task <Func <MemoryStream, bool, int> > GetDataSenderFunc(this IFileMedia fileMedia, IFileConfiguration fileConfig, Interfaces.ILog log)
{
var chunkNumber = 0;
var queue = new ConcurrentQueue <MemoryStream>();
var isFinalizing = false;
var uploader = await fileMedia.GetDataChunkUploader(fileConfig, log);
var senderTask = new Task(() =>
{
while (true)
{
while (queue.TryDequeue(out var nextChunk))
{
try
{
chunkNumber = uploader(nextChunk, isFinalizing && queue.IsEmpty);
}
catch
{
log.Fatal(Localization.GetLocalizationString("Upload has terminated"));
return;
}
}
if (isFinalizing && queue.IsEmpty)
{
return;
}
Thread.Sleep(100);
}
});
senderTask.Start();
return((stream, finalize) =>
{
if (stream != null)
{
queue.Enqueue(stream);
while (queue.Count > 3)
{
Thread.Sleep(50);
}
}
isFinalizing = finalize;
if (isFinalizing)
{
log.Info(Localization.GetLocalizationString("Awaiting for upload to finish."));
senderTask.Wait();
}
return chunkNumber;
});
}
private static async Task <Func <MemoryStream, bool, int> > GetDataChunkUploader(this IFileMedia fileMedia, IFileConfiguration fileConfig, Interfaces.ILog log)
{
var getContext = await fileMedia.GetWaveContextFunc(fileConfig.Name, log);
var xmdJson = fileConfig.GetMetadataBuilder();
var chunkNo = 0;
return((stream, isFinalizing) =>
{
var context = getContext();
if (string.IsNullOrWhiteSpace(context.SetId))
{
context.InitiateDatasetUpload(xmdJson(), fileMedia.Operation, log).Wait();
if (string.IsNullOrWhiteSpace(context.SetId))
{
throw new ImporterException(
Localization.GetLocalizationString("Could not get job id from wave, cannot upload chunks"));
}
}
stream.Flush();
var encCSVchunk = Convert.ToBase64String(stream.ToArray());
var payload =
$"{{\"InsightsExternalDataId\":\"{context.SetId}\",\"PartNumber\":{++chunkNo},\"DataFile\":\"{encCSVchunk}\"}}";
var tryCount = 0;
while (true)
{
try
{
var client = new HttpClient();
var content = new StringContent(payload, Encoding.ASCII);
content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
var url = $"{context.EntryPoint}/services/data/v41.0/sobjects/InsightsExternalDataPart";
client.AddAuthorization(context);
log.Debug($"Uploading chunk #{chunkNo}");
var response = client.PostAsync(url, content).Result;
if (response.IsSuccessStatusCode)
{
log.Debug($"Uploaded chunk #{chunkNo}");
if (isFinalizing)
{
context.FinalizeDatasetUpload(log).Wait();
}
return chunkNo;
}
}
catch (Exception ex)
{
log.Error(Localization.GetLocalizationString("Error while uploading chunk #{0} - {1}", chunkNo, ex.Message));
log.Debug(ex.ToString());
}
if (++tryCount > 4)
{
throw new ImporterUploadException(Localization.GetLocalizationString("Failed to upload dataset."));
}
log.Debug(Localization.GetLocalizationString("Retrying to upload chunk#{0}", chunkNo));
}
});
}
public static Func <IDataRow> ParseData(this Func <ISourceRow> getLineFunc, IFileConfiguration fileConfig, Interfaces.ILog logger)
{
if (fileConfig == null)
{
var msg = Localization.GetLocalizationString("Could not get Source Configuration...");
logger?.Fatal(msg);
throw new ArgumentException(msg);
}
logger?.Info(string.Format(Localization.GetLocalizationString("Parsing data from {0}"), fileConfig.Name));
var parsers = fileConfig.GetRowParsers();
var currentRecord = (long)0;
var parsedRecords = (long)0;
return(() =>
{
var line = getLineFunc();
if (line != null)
{
currentRecord++;
try
{
var row = parsers(line, currentRecord, currentRecord);
if (row != null)
{
parsedRecords++;
return row;
}
}
catch (Exception ex)
{
logger?.Error(Localization.GetLocalizationString("Failed to parse line: \"{0}\"", string.Join(",", line.Fields.Select(x => x.Source))));
throw ex;
}
return new DataRow(
new Dictionary <string, IValue>
{
{
"raw",
new ValueWrapper <string>(string.Join(",", line.Fields.Select(x => x.Source)),
Localization.GetLocalizationString("Parse error"), true, string.Join(",", line.Fields.Select(x => x.Source)))
}
},
Localization.GetLocalizationString("Could not parse line."), currentRecord, line.LineNumber, line.Context.SourcePath, line.Context.FileConfiguration.Name);
}
logger?.Info(string.Format(Localization.GetLocalizationString("Parsed {0}/{1} records from {2}"), parsedRecords, currentRecord, fileConfig.Name));
return null;
});
}
private static Func <ISourceFileContext, Func <IDataRow> > GetStreamReader(this IFileConfiguration fileConfig, Interfaces.ILog logger)
{
switch (fileConfig.Format)
{
case FileFormat.Fixed:
return(context => context?.BufferedRead(logger).FixedWidthReader(context, logger).ParseData(context.FileConfiguration, logger));
case FileFormat.CSV:
return(context => context?.BufferedRead(logger).CsvReader(context, logger).ParseData(context.FileConfiguration, logger));
default:
return(null);
}
}
private static StreamReader GetLocalFileStream(this string filePath, Encoding encoding, Interfaces.ILog logger)
{
if (string.IsNullOrWhiteSpace(filePath))
{
return(null);
}
logger.Debug(string.Format(Localization.GetLocalizationString("Opening source file \"{0}\""),
filePath));
return(new StreamReader(File.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.Read), encoding));
}
private static Func <ISourceFileContext> GetSourceStream(this IFileMedia fileMedia, IFileConfiguration fileConfig, Interfaces.ILog logger)
{
logger?.Debug($"Getting source stream(s) for media \"{fileMedia.MediaType.ToString()}\" - \"{fileMedia.Path}\"");
switch (fileMedia.MediaType)
{
case MediaType.Local:
var directoryName = Path.GetDirectoryName(Path.GetFullPath(fileMedia.Path));
logger?.Debug($"Searching for local file(s) \"{Path.GetFullPath(fileMedia.Path)}\"");
var files = Directory.EnumerateFiles(directoryName, Path.GetFileName(fileMedia.Path),
fileMedia.IncludeSubfolders ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly).OrderBy(x => x).ToList();
logger?.Debug($"Found {files.Count} file(s) matching the pattern.");
foreach (var file in files)
{
logger?.Debug(file);
}
var getFile = files.GetNextFunc();
return(() =>
{
var file = getFile();
return new SourceFileContext
{
SourcePath = file,
Stream = file.GetLocalFileStream(fileMedia.Encoding.GetFilEncoding(), logger),
FileMedia = fileMedia,
FileConfiguration = fileConfig
};
});
}
return(null);
}
public static Stream GetWaveStreamWriter(this IFileMedia fileMedia, IFileConfiguration fileConfig, Interfaces.ILog log)
{
return(new WaveStream(fileMedia, fileConfig, log));
}
private static Stream GetWriterStream(this IFileMedia media, IFileConfiguration fileConfig, Interfaces.ILog log)
{
switch (media.MediaType)
{
case MediaType.Local:
return(media.GetLocalStream(log));
case MediaType.Wave:
return(media.GetWaveStreamWriter(fileConfig, log));
}
return(null);
}
private static async Task <Func <WaveContext> > GetWaveContextFunc(this IFileMedia fileMedia, string fileName, Interfaces.ILog log)
{
var client = new HttpClient();
var url = $"{fileMedia.ConnectionCredentials.EntryPoint}/services/oauth2/token?grant_type=password&client_id={fileMedia.ConnectionCredentials.ClientId}&client_secret={fileMedia.ConnectionCredentials.ClientSecret}&username={fileMedia.ConnectionCredentials.Login}&password={fileMedia.ConnectionCredentials.Password}{fileMedia.ConnectionCredentials.Token}";
var content = new StringContent(string.Empty);
content.Headers.ContentType = new MediaTypeHeaderValue("application/x-www-form-urlencoded");
log.Info(string.Format(
Localization.GetLocalizationString("Obtaining access token and entry point from \"{0}\" for \"{1}\""),
fileMedia.ConnectionCredentials.EntryPoint, fileMedia.ConnectionCredentials.Login));
var response = await client.PostAsync(url, content);
WaveContext context = null;
if (response.IsSuccessStatusCode)
{
log.Info(Localization.GetLocalizationString("Retrieved entry point and access token."));
var responseType = new { access_token = "", instance_url = "" };
var result = JsonConvert.DeserializeAnonymousType(await response.Content.ReadAsStringAsync(), responseType);
context = new WaveContext
{
Token = result.access_token,
EntryPoint = result.instance_url,
Alias = fileName.Replace(" ", "_")
};
}
else
{
log.Fatal($"{response.StatusCode.ToString()}-{response.Content.ReadAsStringAsync()}");
throw new UnauthorizedAccessException(Localization.GetLocalizationString("Could not get access to wave entry point."));
}
return(() => context);
}
private static IList <Action <IDataRow> > ConfigureWriter(this Func <IFile> file, Interfaces.ILog log)
{
var writers = new List <Action <IDataRow> >();
IFile fileConfig;
while ((fileConfig = file()) != null)
{
if (!fileConfig.Disabled)
{
var writer = fileConfig.GetWriter(log).GroupRecords(fileConfig, log);
if (writer != null)
{
writers.Add(writer);
}
}
}
return(writers);
}
public static Func <IDataRow, long> GetCsvWriter(this StreamWriter stream, IFile fileConfig, Interfaces.ILog log)
{
var nullValue = string.IsNullOrWhiteSpace(fileConfig.NullValue) ? "" : fileConfig.NullValue;
var delimiter = string.IsNullOrWhiteSpace(fileConfig.Delimiter)?DEFAULT_DELIMITER[0]:fileConfig.Delimiter[0];
var filters = fileConfig.Rows.Select(x => x.PrepareTargetFilter()).ToList();
var rowCount = (long)0;
var getQualifiedString = fileConfig.GetQualifiedStrFunc(log);
bool isHeadersWritten = false;
return(row =>
{
if (!isHeadersWritten && fileConfig.HasHeaders)
{
for (var r = 0; r < fileConfig.Rows.Count; r++)
{
for (var c = 0; c < fileConfig.Rows[r].Columns.Count; c++)
{
if (c > 0)
{
stream.Write(delimiter);
}
stream.Write(fileConfig.Rows[r].Columns[c].Alias ?? fileConfig.Rows[r].Columns[c].Name);
}
stream.WriteLine();
}
isHeadersWritten = true;
}
if (!string.IsNullOrWhiteSpace(row.Error))
{
log.Error(Localization.GetLocalizationString("Error in file {0}, line {1}", row.SourcePath, row.RawLineNumber) + " - " +
(row.Error ?? "") + "\n\r\t" +
string.Join(",", row.Columns.Values.Select(x => x.Source)));
return rowCount;
}
for (var r = 0; r < fileConfig.Rows.Count; r++)
{
if (!filters[r](row))
{
continue;
}
var columns = fileConfig.Rows[r].Columns;
var rowString = new StringBuilder();
for (var c = 0; c < columns.Count; c++)
{
var column = columns[c];
var value = row[column.Alias ?? column.Name];
if (value == null)
{
//log.Warning(Localization.GetLocalizationString("Could not find column {0}", columns[c].Name));
}
var valueString = (value != null && !value.IsNull) ? getQualifiedString(value.ToString(column.Format), column.Type) : nullValue;
rowString.Append(valueString);
if (columns.Count - 1 > c)
{
rowString.Append(delimiter);
}
}
stream.WriteLine(rowString.ToString());
rowCount++;
}
return rowCount;
});
}
public static Func <ISourceRow> FixedWidthReader(this Func <int> readNext, ISourceFileContext context, Interfaces.ILog logger)
{
if (context?.FileConfiguration == null)
{
var msg = Localization.GetLocalizationString("Could not get Source Configuration...");
logger?.Fatal(msg);
throw new ArgumentException(msg);
}
var locker = new object();
var rowCount = 0;
logger?.Info(string.Format(Localization.GetLocalizationString("Loading data from {0}..."), context.FileConfiguration.Name));
var lineReaders = context.FileConfiguration.Rows.Select(x => x.GetLineFunc(context.FileConfiguration)).ToList();
return(() =>
{
lock (locker)
{
var result = new List <ISourceField>();
foreach (var reader in lineReaders)
{
result.AddRange(reader(readNext));
}
return result.Count > 0 ? new SourceRow {
Context = context, Fields = result, LineNumber = ++rowCount
} : null;
}
});
}