private static async Task <Func <WaveContext> > GetWaveContextFunc(this IFileMedia fileMedia, string fileName, Interfaces.ILog log)
{
var client = new HttpClient();
var url = $"{fileMedia.ConnectionCredentials.EntryPoint}/services/oauth2/token?grant_type=password&client_id={fileMedia.ConnectionCredentials.ClientId}&client_secret={fileMedia.ConnectionCredentials.ClientSecret}&username={fileMedia.ConnectionCredentials.Login}&password={fileMedia.ConnectionCredentials.Password}{fileMedia.ConnectionCredentials.Token}";
var content = new StringContent(string.Empty);
content.Headers.ContentType = new MediaTypeHeaderValue("application/x-www-form-urlencoded");
log.Info(string.Format(
Localization.GetLocalizationString("Obtaining access token and entry point from \"{0}\" for \"{1}\""),
fileMedia.ConnectionCredentials.EntryPoint, fileMedia.ConnectionCredentials.Login));
var response = await client.PostAsync(url, content);
WaveContext context = null;
if (response.IsSuccessStatusCode)
{
log.Info(Localization.GetLocalizationString("Retrieved entry point and access token."));
var responseType = new { access_token = "", instance_url = "" };
var result = JsonConvert.DeserializeAnonymousType(await response.Content.ReadAsStringAsync(), responseType);
context = new WaveContext
{
Token = result.access_token,
EntryPoint = result.instance_url,
Alias = fileName.Replace(" ", "_")
};
}
else
{
log.Fatal($"{response.StatusCode.ToString()}-{response.Content.ReadAsStringAsync()}");
throw new UnauthorizedAccessException(Localization.GetLocalizationString("Could not get access to wave entry point."));
}
return(() => context);
}
public static Func <ISourceRow> FixedWidthReader(this Func <int> readNext, ISourceFileContext context, Interfaces.ILog logger)
{
if (context?.FileConfiguration == null)
{
var msg = Localization.GetLocalizationString("Could not get Source Configuration...");
logger?.Fatal(msg);
throw new ArgumentException(msg);
}
var locker = new object();
var rowCount = 0;
logger?.Info(string.Format(Localization.GetLocalizationString("Loading data from {0}..."), context.FileConfiguration.Name));
var lineReaders = context.FileConfiguration.Rows.Select(x => x.GetLineFunc(context.FileConfiguration)).ToList();
return(() =>
{
lock (locker)
{
var result = new List <ISourceField>();
foreach (var reader in lineReaders)
{
result.AddRange(reader(readNext));
}
return result.Count > 0 ? new SourceRow {
Context = context, Fields = result, LineNumber = ++rowCount
} : null;
}
});
}
private static async Task <Func <MemoryStream, bool, int> > GetDataSenderFunc(this IFileMedia fileMedia, IFileConfiguration fileConfig, Interfaces.ILog log)
{
var chunkNumber = 0;
var queue = new ConcurrentQueue <MemoryStream>();
var isFinalizing = false;
var uploader = await fileMedia.GetDataChunkUploader(fileConfig, log);
var senderTask = new Task(() =>
{
while (true)
{
while (queue.TryDequeue(out var nextChunk))
{
try
{
chunkNumber = uploader(nextChunk, isFinalizing && queue.IsEmpty);
}
catch
{
log.Fatal(Localization.GetLocalizationString("Upload has terminated"));
return;
}
}
if (isFinalizing && queue.IsEmpty)
{
return;
}
Thread.Sleep(100);
}
});
senderTask.Start();
return((stream, finalize) =>
{
if (stream != null)
{
queue.Enqueue(stream);
while (queue.Count > 3)
{
Thread.Sleep(50);
}
}
isFinalizing = finalize;
if (isFinalizing)
{
log.Info(Localization.GetLocalizationString("Awaiting for upload to finish."));
senderTask.Wait();
}
return chunkNumber;
});
}
public static Func <IDataRow> ParseData(this Func <ISourceRow> getLineFunc, IFileConfiguration fileConfig, Interfaces.ILog logger)
{
if (fileConfig == null)
{
var msg = Localization.GetLocalizationString("Could not get Source Configuration...");
logger?.Fatal(msg);
throw new ArgumentException(msg);
}
logger?.Info(string.Format(Localization.GetLocalizationString("Parsing data from {0}"), fileConfig.Name));
var parsers = fileConfig.GetRowParsers();
var currentRecord = (long)0;
var parsedRecords = (long)0;
return(() =>
{
var line = getLineFunc();
if (line != null)
{
currentRecord++;
try
{
var row = parsers(line, currentRecord, currentRecord);
if (row != null)
{
parsedRecords++;
return row;
}
}
catch (Exception ex)
{
logger?.Error(Localization.GetLocalizationString("Failed to parse line: \"{0}\"", string.Join(",", line.Fields.Select(x => x.Source))));
throw ex;
}
return new DataRow(
new Dictionary <string, IValue>
{
{
"raw",
new ValueWrapper <string>(string.Join(",", line.Fields.Select(x => x.Source)),
Localization.GetLocalizationString("Parse error"), true, string.Join(",", line.Fields.Select(x => x.Source)))
}
},
Localization.GetLocalizationString("Could not parse line."), currentRecord, line.LineNumber, line.Context.SourcePath, line.Context.FileConfiguration.Name);
}
logger?.Info(string.Format(Localization.GetLocalizationString("Parsed {0}/{1} records from {2}"), parsedRecords, currentRecord, fileConfig.Name));
return null;
});
}
public static Func <ISourceRow> CsvReader(this Func <int> readNext, ISourceFileContext context, Interfaces.ILog logger)
{
var fileConfig = context.FileConfiguration;
if (fileConfig == null)
{
var msg = Localization.GetLocalizationString("Could not get Source Configuration...");
logger?.Fatal(msg);
throw new ArgumentException(msg);
}
var nullValue = string.IsNullOrWhiteSpace(fileConfig.NullValue) ? "" : fileConfig.NullValue;
var delimiter = string.IsNullOrWhiteSpace(fileConfig.Delimiter) ? ',' : fileConfig.Delimiter[0];
var qualifier = string.IsNullOrWhiteSpace(fileConfig.Qualifier) ? '\"' : fileConfig.Qualifier[0];
var locker = new object();
var rowCount = 0;
logger?.Info(string.Format(Localization.GetLocalizationString("Loading data from {0}..."), fileConfig.Name));
return(() =>
{
lock (locker)
{
var columns = new List <ISourceField>();
var hadQualifier = false;
Action <StringBuilder> enqueue = clmn =>
{
var value = clmn.ToString();
columns.Add((value.Length > 0 && (value != nullValue || hadQualifier)) ? new SourceField(value) : new SourceField(null));
};
int c;
while ((c = readNext()) >= 0 && (c == '\n' || c == '\r'))
{
;
}
if (c < 0)
{
logger?.Info(Localization.GetLocalizationString("Loaded {0} line(s) from {1}.", rowCount, fileConfig.Name));
return null;
}
var isQualified = false;
var column = new StringBuilder();
while (c >= 0)
{
if (c == qualifier && (column.Length == 0 || hadQualifier))
{
hadQualifier = true;
isQualified = !isQualified;
if (isQualified && column.Length > 0)
{
column.Append((char)c);
}
}
else
if (c == delimiter && !isQualified)
{
enqueue(column);
column = new StringBuilder();
}
else
if ((c == '\n' || c == '\r') && !isQualified)
{
enqueue(column);
column = null;
break;
}
else
{
column.Append((char)c);
}
c = readNext();
}
if (c == -1 && column != null)
{
enqueue(column);
}
rowCount++;
return new SourceRow {
Context = context, Fields = columns, LineNumber = rowCount
};
}
});
}