private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
{
TraceSwitch = traceSwitch;
TextReader sr = source as TextReader;
ChoGuard.ArgumentNotNull(sr, "TextReader");
if (sr is StreamReader)
{
((StreamReader)sr).Seek(0, SeekOrigin.Begin);
}
if (!RaiseBeginLoad(sr))
{
yield break;
}
string[] commentTokens = Configuration.Comments;
bool? skip = false;
bool isRecordStartFound = false;
bool isRecordEndFound = false;
long seekOriginPos = sr is StreamReader ? ((StreamReader)sr).BaseStream.Position : 0;
List <string> headers = new List <string>();
Tuple <long, string> lastLine = null;
List <Tuple <long, string> > recLines = new List <Tuple <long, string> >();
long recNo = 0;
int loopCount = Configuration.AutoDiscoverColumns && Configuration.KVPRecordFieldConfigurations.Count == 0 ? 2 : 1;
bool isHeaderFound = loopCount == 1;
bool IsHeaderLoaded = false;
Tuple <long, string> pairIn;
bool abortRequested = false;
for (int i = 0; i < loopCount; i++)
{
if (i == 1)
{
if (sr is StreamReader)
{
((StreamReader)sr).Seek(seekOriginPos, SeekOrigin.Begin);
}
TraceSwitch = traceSwitch;
}
else
{
TraceSwitch = ChoETLFramework.TraceSwitchOff;
}
lastLine = null;
recLines.Clear();
isRecordEndFound = false;
isRecordStartFound = false;
using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >(
new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, Configuration.MayContainEOLInData)).ToEnumerable(),
(pair) =>
{
//bool isStateAvail = IsStateAvail();
skip = false;
//if (isStateAvail)
//{
// if (!IsStateMatches(item))
// {
// skip = filterFunc != null ? filterFunc(item) : false;
// }
// else
// skip = true;
//}
//else
// skip = filterFunc != null ? filterFunc(item) : false;
if (skip == null)
{
return(null);
}
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine);
if (!skip.Value)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1));
}
}
if (skip.Value)
{
return(skip);
}
//if (!(sr.BaseStream is MemoryStream))
// ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1));
//if (Task != null)
// return !IsStateNOTExistsOrNOTMatch(item);
if (pair.Item2.IsNullOrWhiteSpace())
{
if (!Configuration.IgnoreEmptyLine)
{
throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1));
}
else
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1));
}
return(true);
}
}
if (commentTokens != null && commentTokens.Length > 0)
{
foreach (string comment in commentTokens)
{
if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal)) //, true, Configuration.Culture))
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1));
}
return(true);
}
}
}
if (!_configCheckDone)
{
Configuration.Validate(null);
var dict = Configuration.KVPRecordFieldConfigurations.ToDictionary(i1 => i1.Name, i1 => i1.FieldType == null ? null : i1.FieldType);
RaiseMembersDiscovered(dict);
Configuration.UpdateFieldTypesIfAny(dict);
_configCheckDone = true;
}
return(false);
}))
{
while (true)
{
pairIn = e.Peek;
if (!isRecordStartFound)
{
if (pairIn == null)
{
break;
}
lastLine = null;
recLines.Clear();
isRecordEndFound = false;
isRecordStartFound = true;
if (!Configuration.RecordStart.IsNullOrWhiteSpace())
{
//Move to record start
while (!(Configuration.IsRecordStartMatch(e.Peek.Item2)))
{
e.MoveNext();
if (e.Peek == null)
{
break;
}
}
}
if (e.Peek != null)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Record start found at [{0}] line...".FormatString(e.Peek.Item1));
}
e.MoveNext();
continue;
}
else
{
string recordEnd = !Configuration.RecordEnd.IsNullOrWhiteSpace() ? Configuration.RecordEnd : Configuration.RecordStart;
if (!recordEnd.IsNullOrWhiteSpace())
{
if (e.Peek == null)
{
if (Configuration.RecordEnd.IsNullOrWhiteSpace())
{
isRecordEndFound = true;
}
else
{
break;
}
}
else
{
//Move to record start
if (Configuration.IsRecordEndMatch(e.Peek.Item2))
{
isRecordEndFound = true;
isRecordStartFound = false;
if (e.Peek != null)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Record end found at [{0}] line...".FormatString(e.Peek.Item1));
}
}
}
}
else if (e.Peek == null)
{
isRecordEndFound = true;
}
if (!isHeaderFound)
{
//if (isRecordEndFound && headers.Count == 0)
//{
// //throw new ChoParserException("Unexpected EOF found.");
//}
if (!isRecordEndFound)
{
e.MoveNext();
if (e.Peek != null)
{
//If line empty or line continuation, skip
if (pairIn.Item2.IsNullOrWhiteSpace() || IsLineContinuationCharFound(pairIn.Item2)) //.Item2[0] == ' ' || pairIn.Item2[0] == '\t')
{
}
else
{
string header = ToKVP(pairIn.Item1, pairIn.Item2).Key;
if (!header.IsNullOrWhiteSpace())
{
headers.Add(header);
}
}
}
}
else
{
Configuration.Validate(headers.ToArray());
isHeaderFound = true;
isRecordEndFound = false;
IsHeaderLoaded = true;
break;
}
}
else
{
if (!IsHeaderLoaded)
{
Configuration.Validate(new string[] { });
IsHeaderLoaded = true;
}
if (isRecordEndFound && recLines.Count == 0)
{
//throw new ChoParserException("Unexpected EOF found.");
}
else if (!isRecordEndFound)
{
e.MoveNext();
if (e.Peek != null)
{
//If line empty or line continuation, skip
if (pairIn.Item2.IsNullOrWhiteSpace())
{
if (!Configuration.IgnoreEmptyLine)
{
throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pairIn.Item1));
}
else
{
Tuple <long, string> t = new Tuple <long, string>(lastLine.Item1, lastLine.Item2 + Configuration.EOLDelimiter);
recLines.RemoveAt(recLines.Count - 1);
recLines.Add(t);
}
}
else if (IsLineContinuationCharFound(pairIn.Item2)) //pairIn.Item2[0] == ' ' || pairIn.Item2[0] == '\t')
{
if (lastLine == null)
{
throw new ChoParserException("Unexpected line continuation found at {0} location.".FormatString(pairIn.Item1));
}
else
{
Tuple <long, string> t = new Tuple <long, string>(lastLine.Item1, lastLine.Item2 + Configuration.EOLDelimiter + pairIn.Item2);
recLines.RemoveAt(recLines.Count - 1);
recLines.Add(t);
}
}
else
{
lastLine = pairIn;
recLines.Add(pairIn);
}
}
}
else
{
object rec = Configuration.IsDynamicObject ? new ChoDynamicObject(new Dictionary <string, object>(Configuration.FileHeaderConfiguration.StringComparer))
{
ThrowExceptionIfPropNotExists = true,
AlternativeKeys = Configuration.AlternativeKeys
} : Activator.CreateInstance(RecordType);
if (!LoadLines(new Tuple <long, List <Tuple <long, string> > >(++recNo, recLines), ref rec))
{
yield break;
}
isRecordStartFound = false;
//StoreState(e.Current, rec != null);
if (!Configuration.RecordEnd.IsNullOrWhiteSpace())
{
e.MoveNext();
}
if (rec == null)
{
continue;
}
yield return(rec);
if (Configuration.NotifyAfter > 0 && recNo % Configuration.NotifyAfter == 0)
{
if (RaisedRowsLoaded(recNo))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
abortRequested = true;
yield break;
}
}
if (e.Peek == null)
{
break;
}
}
}
}
}
}
}
if (!abortRequested)
{
RaisedRowsLoaded(recNo, true);
}
RaiseEndLoad(sr);
}
private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
{
TraceSwitch = traceSwitch;
TextReader sr = source as TextReader;
ChoGuard.ArgumentNotNull(sr, "TextReader");
if (sr is StreamReader)
{
((StreamReader)sr).Seek(0, SeekOrigin.Begin);
}
if (Configuration.RecordSelector == null)
{
throw new ChoRecordConfigurationException("Missing record selector.");
}
if (!RaiseBeginLoad(sr))
{
yield break;
}
string[] commentTokens = Configuration.Comments;
bool? skip = false;
bool _headerFound = false;
bool? skipUntil = true;
bool? doWhile = true;
using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >(
new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, Configuration.MayContainEOLInData, Configuration.MaxLineSize)).ToEnumerable(),
(pair) =>
{
//bool isStateAvail = IsStateAvail();
skip = false;
if (skipUntil != null)
{
if (skipUntil.Value)
{
skipUntil = RaiseSkipUntil(pair);
if (skipUntil == null)
{
}
else
{
skip = skipUntil.Value;
}
}
}
//if (isStateAvail)
//{
// if (!IsStateMatches(item))
// {
// skip = filterFunc != null ? filterFunc(item) : false;
// }
// else
// skip = true;
//}
//else
// skip = filterFunc != null ? filterFunc(item) : false;
if (skip == null)
{
return(null);
}
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine);
if (!skip.Value)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1));
}
}
if (skip.Value)
{
return(skip);
}
//if (!(sr.BaseStream is MemoryStream))
// ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1));
//if (Task != null)
// return !IsStateNOTExistsOrNOTMatch(item);
if (pair.Item2.IsNullOrWhiteSpace())
{
if (!Configuration.IgnoreEmptyLine)
{
throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1));
}
else
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1));
}
return(true);
}
}
if (commentTokens != null && commentTokens.Length > 0)
{
foreach (string comment in commentTokens)
{
if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal)) //, true, Configuration.Culture))
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1));
}
return(true);
}
}
}
if (!_configCheckDone)
{
Configuration.Validate(pair); // GetHeaders(pair.Item2));
_configCheckDone = true;
}
//Ignore Header if any
if (Configuration.FileHeaderConfiguration.HasHeaderRecord &&
!_headerFound)
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring header line at [{0}]...".FormatString(pair.Item1));
}
_headerFound = true;
return(true);
}
return(false);
}))
{
while (true)
{
Tuple <long, string> pair = e.Peek;
if (pair == null)
{
RaiseEndLoad(sr);
yield break;
}
Type recType = Configuration.RecordSelector(pair.Item2);
if (recType == null)
{
if (Configuration.IgnoreIfNoRecordParserExists)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, $"No record type found for [{pair.Item1}] line to parse.");
}
else
{
throw new ChoParserException($"No record type found for [{pair.Item1}] line to parse.");
}
}
object rec = ChoActivator.CreateInstance(recType);
if (!LoadLine(pair, ref rec))
{
yield break;
}
//StoreState(e.Current, rec != null);
e.MoveNext();
if (rec == null)
{
continue;
}
yield return(rec);
if (Configuration.NotifyAfter > 0 && pair.Item1 % Configuration.NotifyAfter == 0)
{
if (RaisedRowsLoaded(pair.Item1))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
yield break;
}
}
if (doWhile != null)
{
doWhile = RaiseDoWhile(pair);
if (doWhile != null && doWhile.Value)
{
break;
}
}
}
}
}
private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
{
TraceSwitch = traceSwitch;
TextReader sr = source as TextReader;
ChoGuard.ArgumentNotNull(sr, "TextReader");
if (sr is StreamReader)
{
((StreamReader)sr).Seek(0, SeekOrigin.Begin);
}
if (!RaiseBeginLoad(sr))
{
yield break;
}
string[] commentTokens = Configuration.Comments;
bool? skip = false;
bool abortRequested = false;
long runningCount = 0;
long recCount = 0;
bool headerLineLoaded = false;
List <object> buffer = new List <object>();
IDictionary <string, Type> recFieldTypes = null;
bool?skipUntil = true;
bool?doWhile = true;
using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >(
new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, false /*Configuration.MayContainEOLInData*/)).ToEnumerable(),
(pair) =>
{
//bool isStateAvail = IsStateAvail();
skip = false;
if (skipUntil != null)
{
if (skipUntil.Value)
{
skipUntil = RaiseSkipUntil(pair);
if (skipUntil == null)
{
}
else
{
if (skipUntil.Value)
{
skip = skipUntil;
}
else
{
skip = true;
}
}
}
}
//if (isStateAvail)
//{
// if (!IsStateMatches(item))
// {
// skip = filterFunc != null ? filterFunc(item) : false;
// }
// else
// skip = true;
//}
//else
// skip = filterFunc != null ? filterFunc(item) : false;
if (skip == null)
{
return(null);
}
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine);
if (!skip.Value)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1));
}
}
if (skip.Value)
{
return(skip);
}
//if (!(sr.BaseStream is MemoryStream))
// ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1));
//if (Task != null)
// return !IsStateNOTExistsOrNOTMatch(item);
if (pair.Item2.IsNullOrWhiteSpace())
{
if (!Configuration.IgnoreEmptyLine)
{
throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1));
}
else
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1));
}
return(true);
}
}
if (commentTokens != null && commentTokens.Length > 0)
{
foreach (string comment in commentTokens)
{
if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal)) //, true, Configuration.Culture))
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1));
}
return(true);
}
}
}
if (Configuration.FileHeaderConfiguration.HeaderLineAt > 0)
{
if (pair.Item1 < Configuration.FileHeaderConfiguration.HeaderLineAt)
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Header line at {1}. Skipping [{0}] line...".FormatString(pair.Item1, Configuration.FileHeaderConfiguration.HeaderLineAt));
}
return(true);
}
}
if (!_configCheckDone)
{
if (Configuration.SupportsMultiRecordTypes && Configuration.RecordSelector != null && !Configuration.RecordTypeMapped)
{
}
else
{
Configuration.Validate(pair); // GetHeaders(pair.Item2));
}
var dict = recFieldTypes = Configuration.FixedLengthRecordFieldConfigurations.ToDictionary(i => i.Name, i => i.FieldType == null ? null : i.FieldType);
RaiseMembersDiscovered(dict);
Configuration.UpdateFieldTypesIfAny(dict);
_configCheckDone = true;
}
//LoadHeader if any
if ((Configuration.FileHeaderConfiguration.HasHeaderRecord ||
Configuration.FileHeaderConfiguration.HeaderLineAt > 0) &&
!_headerFound)
{
if (Configuration.FileHeaderConfiguration.IgnoreHeader)
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring header line at [{0}]...".FormatString(pair.Item1));
}
}
else
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading header line at [{0}]...".FormatString(pair.Item1));
}
headerLineLoaded = true;
LoadHeaderLine(pair);
}
_headerFound = true;
return(true);
}
return(false);
}))
{
while (true)
{
recCount++;
Tuple <long, string> pair = e.Peek;
if (pair == null)
{
if (!abortRequested)
{
RaisedRowsLoaded(runningCount);
}
RaiseEndLoad(sr);
yield break;
}
runningCount = pair.Item1;
object rec = null;
if (Configuration.SupportsMultiRecordTypes && Configuration.RecordSelector != null)
{
Type recType = Configuration.RecordSelector(pair);
if (recType == null)
{
if (Configuration.IgnoreIfNoRecordTypeFound)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, $"No record type found for [{pair.Item1}] line to parse.");
e.MoveNext();
continue;
}
else
{
throw new ChoParserException($"No record type found for [{pair.Item1}] line to parse.");
}
}
if (!Configuration.RecordTypeMapped)
{
Configuration.MapRecordFields(recType);
Configuration.Validate(null);
}
rec = recType.IsDynamicType() ? new ChoDynamicObject(new Dictionary <string, object>(Configuration.FileHeaderConfiguration.StringComparer))
{
ThrowExceptionIfPropNotExists = true,
AlternativeKeys = Configuration.AlternativeKeys
} : ChoActivator.CreateInstance(recType);
}
else
{
rec = Configuration.IsDynamicObject ? new ChoDynamicObject(new Dictionary <string, object>(Configuration.FileHeaderConfiguration.StringComparer))
{
ThrowExceptionIfPropNotExists = true,
AlternativeKeys = Configuration.AlternativeKeys
} : ChoActivator.CreateInstance(RecordType);
}
if (!LoadLine(pair, ref rec))
{
yield break;
}
//StoreState(e.Current, rec != null);
e.MoveNext();
if (rec == null)
{
continue;
}
if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject)
{
if (Configuration.AreAllFieldTypesNull && Configuration.AutoDiscoverFieldTypes && Configuration.MaxScanRows > 0 && recCount <= Configuration.MaxScanRows)
{
buffer.Add(rec);
RaiseRecordFieldTypeAssessment(recFieldTypes, (IDictionary <string, object>)rec, recCount == Configuration.MaxScanRows);
if (recCount == Configuration.MaxScanRows || e.Peek == null)
{
Configuration.UpdateFieldTypesIfAny(recFieldTypes);
var dict = recFieldTypes = Configuration.FixedLengthRecordFieldConfigurations.ToDictionary(i => i.Name, i => i.FieldType == null ? null : i.FieldType);
RaiseMembersDiscovered(dict);
foreach (object rec1 in buffer)
{
yield return(ConvertToNestedObjectIfApplicable(new ChoDynamicObject(MigrateToNewSchema(rec1 as IDictionary <string, object>, recFieldTypes)) as object, headerLineLoaded));
}
}
}
else
{
yield return(ConvertToNestedObjectIfApplicable(rec, headerLineLoaded));
}
}
else
{
yield return(rec);
}
if (Configuration.NotifyAfter > 0 && pair.Item1 % Configuration.NotifyAfter == 0)
{
if (RaisedRowsLoaded(pair.Item1))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
yield break;
}
}
if (doWhile != null)
{
doWhile = RaiseDoWhile(pair);
if (doWhile != null && doWhile.Value)
{
break;
}
}
}
}
}
private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
{
TraceSwitch = traceSwitch;
StreamReader sr = source as StreamReader;
ChoGuard.ArgumentNotNull(sr, "StreamReader");
if (Configuration.RecordSelector == null)
{
throw new ChoRecordConfigurationException("Missing record selector.");
}
sr.Seek(0, SeekOrigin.Begin);
if (!RaiseBeginLoad(sr))
{
yield break;
}
string[] commentTokens = Configuration.Comments;
bool _headerFound = false;
using (ChoPeekEnumerator <Tuple <int, string> > e = new ChoPeekEnumerator <Tuple <int, string> >(
new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar)).ToEnumerable(),
(pair) =>
{
//bool isStateAvail = IsStateAvail();
bool?skip = false;
//if (isStateAvail)
//{
// if (!IsStateMatches(item))
// {
// skip = filterFunc != null ? filterFunc(item) : false;
// }
// else
// skip = true;
//}
//else
// skip = filterFunc != null ? filterFunc(item) : false;
if (skip == null)
{
return(null);
}
//if (!(sr.BaseStream is MemoryStream))
//{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine);
if (!skip.Value)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1));
}
//}
if (skip.Value)
{
return(skip);
}
//if (!(sr.BaseStream is MemoryStream))
// ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1));
//if (Task != null)
// return !IsStateNOTExistsOrNOTMatch(item);
if (pair.Item2.IsNullOrWhiteSpace())
{
if (!Configuration.IgnoreEmptyLine)
{
throw new ChoParserException("Empty line found at {0} location.".FormatString(e.Peek.Item1));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Empty line found at [{0}]...".FormatString(pair.Item1));
return(true);
}
}
if (commentTokens == null)
{
return(false);
}
else
{
var x = (from comment in commentTokens
where !pair.Item2.IsNull() && pair.Item2.StartsWith(comment, true, Configuration.Culture)
select comment).FirstOrDefault();
if (x != null)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1));
return(true);
}
}
if (!_configCheckDone)
{
Configuration.Validate(pair); // GetHeaders(pair.Item2));
_configCheckDone = true;
}
//Ignore Header if any
if (Configuration.FileHeaderConfiguration.HasHeaderRecord &&
!_headerFound)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring header line at [{0}]...".FormatString(pair.Item1));
_headerFound = true;
return(true);
}
return(false);
}))
{
while (true)
{
Tuple <int, string> pair = e.Peek;
if (pair == null)
{
RaiseEndLoad(sr);
yield break;
}
Type recType = Configuration.RecordSelector(pair.Item2);
if (recType == null)
{
if (Configuration.IgnoreIfNoRecordParserExists)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, $"No record type found for [{pair.Item1}] line to parse.");
}
else
{
throw new ChoParserException($"No record type found for [{pair.Item1}] line to parse.");
}
}
object rec = ChoActivator.CreateInstance(recType);
if (!LoadLine(pair, ref rec))
{
yield break;
}
//StoreState(e.Current, rec != null);
e.MoveNext();
if (rec == null)
{
continue;
}
yield return(rec);
}
}
}
private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
{
TraceSwitch = traceSwitch;
TextReader sr = source as TextReader;
ChoGuard.ArgumentNotNull(sr, "TextReader");
if (sr is StreamReader)
{
((StreamReader)sr).Seek(0, SeekOrigin.Begin);
}
if (!RaiseBeginLoad(sr))
{
yield break;
}
string[] commentTokens = Configuration.Comments;
bool? skip = false;
bool abortRequested = false;
long runningCount = 0;
using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >(
new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, false /*Configuration.MayContainEOLInData*/)).ToEnumerable(),
(pair) =>
{
//bool isStateAvail = IsStateAvail();
skip = false;
//if (isStateAvail)
//{
// if (!IsStateMatches(item))
// {
// skip = filterFunc != null ? filterFunc(item) : false;
// }
// else
// skip = true;
//}
//else
// skip = filterFunc != null ? filterFunc(item) : false;
if (skip == null)
{
return(null);
}
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine);
if (!skip.Value)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1));
}
}
if (skip.Value)
{
return(skip);
}
//if (!(sr.BaseStream is MemoryStream))
// ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1));
//if (Task != null)
// return !IsStateNOTExistsOrNOTMatch(item);
if (pair.Item2.IsNullOrWhiteSpace())
{
if (!Configuration.IgnoreEmptyLine)
{
throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1));
}
else
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1));
}
return(true);
}
}
if (commentTokens != null && commentTokens.Length > 0)
{
foreach (string comment in commentTokens)
{
if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal)) //, true, Configuration.Culture))
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1));
}
return(true);
}
}
}
if (!_configCheckDone)
{
Configuration.Validate(pair); // GetHeaders(pair.Item2));
RaiseMembersDiscovered(Configuration.FixedLengthRecordFieldConfigurations.Select(i => new KeyValuePair <string, Type>(i.Name, i.FieldType == null ? typeof(string) : i.FieldType)).ToArray());
_configCheckDone = true;
}
//LoadHeader if any
if (Configuration.FileHeaderConfiguration.HasHeaderRecord &&
!_headerFound)
{
if (Configuration.FileHeaderConfiguration.IgnoreHeader)
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring header line at [{0}]...".FormatString(pair.Item1));
}
}
else
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading header line at [{0}]...".FormatString(pair.Item1));
}
LoadHeaderLine(pair);
}
_headerFound = true;
return(true);
}
return(false);
}))
{
while (true)
{
Tuple <long, string> pair = e.Peek;
if (pair == null)
{
if (!abortRequested)
{
RaisedRowsLoaded(runningCount);
}
RaiseEndLoad(sr);
yield break;
}
runningCount = pair.Item1;
object rec = Configuration.IsDynamicObject ? new ChoDynamicObject()
{
ThrowExceptionIfPropNotExists = true
} : Activator.CreateInstance(RecordType);
if (!LoadLine(pair, ref rec))
{
yield break;
}
//StoreState(e.Current, rec != null);
e.MoveNext();
if (rec == null)
{
continue;
}
yield return(rec);
if (Configuration.NotifyAfter > 0 && pair.Item1 % Configuration.NotifyAfter == 0)
{
if (RaisedRowsLoaded(pair.Item1))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
yield break;
}
}
}
}
}
public override IEnumerable <object> AsEnumerable(object source, Func <object, bool?> filterFunc = null)
{
StreamReader sr = source as StreamReader;
ChoGuard.ArgumentNotNull(sr, "StreamReader");
sr.Seek(0, SeekOrigin.Begin);
if (!RaiseBeginLoad(sr))
{
yield break;
}
string[] commentTokens = Configuration.Comments;
using (ChoPeekEnumerator <Tuple <int, string> > e = new ChoPeekEnumerator <Tuple <int, string> >(
new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar)).ToEnumerable(),
(pair) =>
{
//bool isStateAvail = IsStateAvail();
bool?skip = false;
//if (isStateAvail)
//{
// if (!IsStateMatches(item))
// {
// skip = filterFunc != null ? filterFunc(item) : false;
// }
// else
// skip = true;
//}
//else
// skip = filterFunc != null ? filterFunc(item) : false;
if (skip == null)
{
return(null);
}
//if (!(sr.BaseStream is MemoryStream))
//{
ChoETLFramework.WriteLog(Environment.NewLine);
if (!skip.Value)
{
ChoETLFramework.WriteLog("Loading line [{0}]...".FormatString(pair.Item1));
}
else
{
ChoETLFramework.WriteLog("Skipping line [{0}]...".FormatString(pair.Item1));
}
//}
if (skip.Value)
{
return(skip);
}
//if (!(sr.BaseStream is MemoryStream))
// ChoETLFramework.WriteLog(ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1));
//if (Task != null)
// return !IsStateNOTExistsOrNOTMatch(item);
if (pair.Item2.IsNullOrWhiteSpace())
{
if (!Configuration.IgnoreEmptyLine)
{
throw new ChoParserException("Empty line found at {0} location.".FormatString(e.Peek.Item1));
}
else
{
ChoETLFramework.WriteLog("Empty line found at [{0}]...".FormatString(pair.Item1));
return(true);
}
}
//LoadExcelSeparator if any
if (pair.Item1 == 1 &&
!_excelSeparatorFound)
{
ChoETLFramework.WriteLog("Inspecting for excel separator at [{0}]...".FormatString(pair.Item1));
bool retVal = LoadExcelSeperatorIfAny(pair);
_excelSeparatorFound = true;
if (Configuration.HasExcelSeparator.HasValue &&
Configuration.HasExcelSeparator.Value &&
!retVal)
{
throw new ChoParserException("Missing excel separator header line in the file.");
}
if (retVal)
{
return(true);
}
}
if (commentTokens == null)
{
return(false);
}
else
{
var x = (from comment in commentTokens
where !pair.Item2.IsNull() && pair.Item2.StartsWith(comment, true, Configuration.Culture)
select comment).FirstOrDefault();
if (x != null)
{
ChoETLFramework.WriteLog("Comment line found at [{0}]...".FormatString(pair.Item1));
return(true);
}
}
//LoadHeader if any
if (Configuration.CSVFileHeaderConfiguration.HasHeaderRecord &&
!_headerFound)
{
ChoETLFramework.WriteLog("Loading header line at [{0}]...".FormatString(pair.Item1));
LoadHeaderLine(pair);
_headerFound = true;
return(true);
}
return(false);
}))
{
while (true)
{
Tuple <int, string> pair = e.Peek;
if (pair == null)
{
RaiseEndLoad(sr);
yield break;
}
object rec = ChoActivator.CreateInstance(RecordType);
if (!LoadLine(pair, ref rec))
{
yield break;
}
//StoreState(e.Current, rec != null);
e.MoveNext();
if (rec == null)
{
continue;
}
yield return(rec);
}
}
}