public static void Initialize(this object target)
{
if (target == null)
{
return;
}
object defaultValue = null;
foreach (PropertyDescriptor pd in ChoTypeDescriptor.GetProperties <DefaultValueAttribute>(target.GetType()))
{
try
{
defaultValue = ChoTypeDescriptor.GetPropetyAttribute <DefaultValueAttribute>(pd).Value;
if (defaultValue != null)
{
ChoType.ConvertNSetPropertyValue(target, pd.Name, defaultValue);
}
}
catch (Exception ex)
{
ChoETLFramework.WriteLog(ChoETLFramework.TraceSwitch.TraceError, "Error while assigning default value '{0}' to '{1}' member. {2}".FormatString(defaultValue, ChoType.GetMemberName(pd), ex.Message));
}
}
foreach (PropertyDescriptor pd in ChoTypeDescriptor.GetProperties <ChoDefaultValueAttribute>(target.GetType()))
{
if (ChoTypeDescriptor.GetPropetyAttribute <ChoDefaultValueAttribute>(pd) == null)
{
continue;
}
try
{
defaultValue = ChoTypeDescriptor.GetPropetyAttribute <ChoDefaultValueAttribute>(pd).Value;
if (defaultValue != null)
{
ChoType.ConvertNSetPropertyValue(target, pd.Name, defaultValue);
}
}
catch (Exception ex)
{
ChoETLFramework.WriteLog(ChoETLFramework.TraceSwitch.TraceError, "Error while assigning default value '{0}' to '{1}' member. {2}".FormatString(defaultValue, ChoType.GetMemberName(pd), ex.Message));
}
}
//ChoETLFramework.InitializeObject(target);
if (target is IChoInitializable)
{
((IChoInitializable)target).Initialize();
}
}
private void LoadFixedLengthRecordFieldOptions(ChoIniFile iniFile1)
{
ChoIniFile iniFile = iniFile1.GetSection("FIXED_LENGTH_RECORD_FIELD");
var dict = new OrderedDictionary();
ChoFixedLengthRecordFieldAttribute value;
Dictionary <string, string> iniKeyValuesDict = iniFile.KeyValuesDict;
if (iniKeyValuesDict.Count == 0)
{
throw new ApplicationException("Missing fixed length field names in the '{0}' ini file.".FormatString(iniFile1.FilePath));
}
foreach (string fieldName in iniKeyValuesDict.Keys) //GetFieldNames(iniFile1))
{
if (fieldName.IsNullOrWhiteSpace())
{
continue;
}
if (!iniKeyValuesDict.ContainsKey(fieldName))
{
continue;
}
try
{
value = new ChoFixedLengthRecordFieldAttribute();
value.FieldValueTrimOption = ChoFieldValueTrimOption.Trim;
value.FieldValueJustification = ChoFieldValueJustification.Left;
foreach (KeyValuePair <string, string> kvp in iniKeyValuesDict[fieldName].ToKeyValuePairs())
{
try
{
ChoType.ConvertNSetMemberValue(value, kvp.Key, kvp.Value);
}
catch (Exception ex)
{
ChoETLFramework.WriteLog(ChoETLFramework.Switch.TraceError, "Error while loading record field option '{0}' for '{1}' field. {2}".FormatString(iniKeyValuesDict[fieldName], fieldName, ex.Message));
}
}
value.Validate();
dict.Add(fieldName, value);
}
catch (Exception ex)
{
throw new ApplicationException("[FieldName: {0}] - {1}".FormatString(fieldName, ex.Message));
}
}
_flRecordFieldOptionCache.Add(iniFile1.Key, dict);
}
private DateTime GetNowTime()
{
DateTime now = DateTime.Now;
string nowText = ChoETLFramework.GetConfigValue("NOW");
if (!nowText.IsNullOrWhiteSpace())
{
try
{
now = Convert.ToDateTime(nowText);
}
catch { }
}
return(now);
}
private DateTime GetTodaysDate()
{
DateTime today = DateTime.Today;
string todayText = ChoETLFramework.GetConfigValue("TODAY");
if (!todayText.IsNullOrWhiteSpace())
{
try
{
today = Convert.ToDateTime(todayText);
}
catch { }
}
return(today);
}
private bool ValidateOnWrite(ref object value)
{
var crs = Writer.ContractResolverState;
var fc = crs.FieldConfig;
crs.Name = _fc == null?_mi.GetFullName() : crs.Name;
var rec = ChoType.GetMemberObjectMatchingType(crs.Name, crs.Record);
var name = ChoType.GetFieldName(crs.Name);
try
{
Validate(value);
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ref ex);
if (fc != null && fc.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
if (fc != null)
{
if (fc.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
return(false);
}
else
{
if (!RaiseRecordFieldWriteError(rec, crs.Index, name, ref value, ex))
{
throw new ChoWriterException($"Failed to write '{value}' value of '{crs.Name}' member.", ex);
}
}
}
else
{
throw new ChoWriterException($"Failed to write '{value}' value of '{crs.Name}' member.", ex);
}
}
return(true);
}
private bool LoadExcelSeperatorIfAny(Tuple <int, string> pair)
{
string line = pair.Item2.NTrim();
if (!line.IsNullOrWhiteSpace() && line.StartsWith("sep=", true, Configuration.Culture))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Excel separator specified at [{0}]...".FormatString(pair.Item1));
string delimiter = line.Substring(4);
if (!delimiter.IsNullOrWhiteSpace())
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Excel separator [{0}] found.".FormatString(delimiter));
Configuration.Delimiter = delimiter;
}
return(true);
}
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Excel separator NOT found. Default separator [{0}] used.".FormatString(Configuration.Delimiter));
return(false);
}
private void LoadConverterParams(Type declaringType)
{
ChoIniFile iniFile = GetIniSection(declaringType, "FORMATTER");
var dict = new Dictionary <string, object[]>();
string parameters = null;
ChoMemberAttribute memberAttribute = null;
foreach (MemberInfo memberInfo in ChoType.GetMembers(declaringType))
{
memberAttribute = ChoType.GetAttribute <ChoMemberAttribute>(memberInfo);
if (memberAttribute == null)
{
continue;
}
try
{
if (_turnOnMetaDataCache)
{
parameters = iniFile.GetValue(memberInfo.Name);
}
List <object> p = new List <object>();
if (!parameters.IsNullOrWhiteSpace())
{
foreach (string kv in parameters.SplitNTrim(';'))
{
p.Add(kv.SplitNTrim(','));
}
}
dict.Add(memberInfo.Name, parameters.IsNullOrWhiteSpace() ? null : p.ToArray());
}
catch (Exception ex)
{
ChoETLFramework.WriteLog(ChoETLFramework.Switch.TraceError, "Failed to retrieve converter params for '{0}' member type from INI file. {1}".FormatString(ChoType.GetMemberName(memberInfo), ex.Message));
}
}
_converterParams.Add(declaringType, dict);
}
private void Start()
{
lock (typeof(ChoTextWriterTraceListener))
{
if (_fileWriterThread != null && _fileWriterThread.IsAlive)
{
return;
}
try
{
_fileWriterThread = new Thread(new ThreadStart(SyncFileWriter));
_fileWriterThread.IsBackground = true;
_fileWriterThread.Start();
}
catch (Exception ex)
{
ChoETLFramework.WriteLog(ChoETLFramework.TraceSwitch.TraceError, ex.ToString());
}
}
}
private void LoadIsRequireds(Type declaringType)
{
ChoIniFile iniFile = GetIniSection(declaringType, "REQUIRED");
var dict = new Dictionary <string, bool>();
bool isRequired = false;
ChoMemberAttribute memberAttribute = null;
foreach (MemberInfo memberInfo in ChoType.GetMembers(declaringType))
{
memberAttribute = ChoType.GetAttribute <ChoMemberAttribute>(memberInfo);
if (memberAttribute == null)
{
continue;
}
try
{
if (_turnOnMetaDataCache)
{
isRequired = iniFile.GetValue(memberInfo.Name, memberAttribute.IsRequired, _isLockedCache[declaringType]);
}
else
{
isRequired = memberAttribute.IsRequired;
}
dict.Add(memberInfo.Name, isRequired);
}
catch (Exception ex)
{
ChoETLFramework.WriteLog(ChoETLFramework.Switch.TraceError, "Incorrect IsRequired value specified for '{0}' member type in INI file. Defaulted to false. {1}".FormatString(ChoType.GetMemberName(memberInfo), ex.Message));
}
}
_isReqCache.Add(declaringType, dict);
}
private void LoadDbRecordFieldOptions(ChoIniFile iniFile1)
{
ChoIniFile iniFile = iniFile1.GetSection("Db_RECORD_FIELD");
var dict = new Dictionary <string, ChoDbRecordFieldAttribute>();
ChoDbRecordFieldAttribute value;
Dictionary <string, string> iniKeyValuesDict = iniFile.KeyValuesDict;
foreach (string fieldName in iniKeyValuesDict.Keys)
{
if (fieldName.IsNullOrWhiteSpace())
{
continue;
}
if (!iniKeyValuesDict.ContainsKey(fieldName))
{
continue;
}
value = new ChoDbRecordFieldAttribute();
foreach (KeyValuePair <string, string> kvp in iniKeyValuesDict[fieldName].ToKeyValuePairs())
{
try
{
ChoType.ConvertNSetMemberValue(value, kvp.Key, kvp.Value);
}
catch (Exception ex)
{
ChoETLFramework.WriteLog(ChoETLFramework.Switch.TraceError, "Error while loading record field '{0}' for '{1}' field. {2}".FormatString(iniKeyValuesDict[fieldName], fieldName, ex.Message));
}
}
value.Validate();
dict.Add(fieldName, value);
}
_dbRecordFieldOptionCache.Add(iniFile1.Key, dict);
}
private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
{
TraceSwitch = traceSwitch;
TextReader sr = source as TextReader;
ChoGuard.ArgumentNotNull(sr, "TextReader");
if (sr is StreamReader)
{
((StreamReader)sr).Seek(0, SeekOrigin.Begin);
}
if (Configuration.RecordSelector == null)
{
throw new ChoRecordConfigurationException("Missing record selector.");
}
if (!RaiseBeginLoad(sr))
{
yield break;
}
string[] commentTokens = Configuration.Comments;
bool? skip = false;
bool _headerFound = false;
bool? skipUntil = true;
bool? doWhile = true;
using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >(
new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, Configuration.MayContainEOLInData, Configuration.MaxLineSize)).ToEnumerable(),
(pair) =>
{
//bool isStateAvail = IsStateAvail();
skip = false;
if (skipUntil != null)
{
if (skipUntil.Value)
{
skipUntil = RaiseSkipUntil(pair);
if (skipUntil == null)
{
}
else
{
skip = skipUntil.Value;
}
}
}
//if (isStateAvail)
//{
// if (!IsStateMatches(item))
// {
// skip = filterFunc != null ? filterFunc(item) : false;
// }
// else
// skip = true;
//}
//else
// skip = filterFunc != null ? filterFunc(item) : false;
if (skip == null)
{
return(null);
}
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine);
if (!skip.Value)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1));
}
}
if (skip.Value)
{
return(skip);
}
//if (!(sr.BaseStream is MemoryStream))
// ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1));
//if (Task != null)
// return !IsStateNOTExistsOrNOTMatch(item);
if (pair.Item2.IsNullOrWhiteSpace())
{
if (!Configuration.IgnoreEmptyLine)
{
throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1));
}
else
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1));
}
return(true);
}
}
if (commentTokens != null && commentTokens.Length > 0)
{
foreach (string comment in commentTokens)
{
if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal)) //, true, Configuration.Culture))
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1));
}
return(true);
}
}
}
if (!_configCheckDone)
{
Configuration.Validate(pair); // GetHeaders(pair.Item2));
_configCheckDone = true;
}
//Ignore Header if any
if (Configuration.FileHeaderConfiguration.HasHeaderRecord &&
!_headerFound)
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring header line at [{0}]...".FormatString(pair.Item1));
}
_headerFound = true;
return(true);
}
return(false);
}))
{
while (true)
{
Tuple <long, string> pair = e.Peek;
if (pair == null)
{
RaiseEndLoad(sr);
yield break;
}
Type recType = Configuration.RecordSelector(pair.Item2);
if (recType == null)
{
if (Configuration.IgnoreIfNoRecordParserExists)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, $"No record type found for [{pair.Item1}] line to parse.");
}
else
{
throw new ChoParserException($"No record type found for [{pair.Item1}] line to parse.");
}
}
object rec = ChoActivator.CreateInstance(recType);
if (!LoadLine(pair, ref rec))
{
yield break;
}
//StoreState(e.Current, rec != null);
e.MoveNext();
if (rec == null)
{
continue;
}
yield return(rec);
if (Configuration.NotifyAfter > 0 && pair.Item1 % Configuration.NotifyAfter == 0)
{
if (RaisedRowsLoaded(pair.Item1))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
yield break;
}
}
if (doWhile != null)
{
doWhile = RaiseDoWhile(pair);
if (doWhile != null && doWhile.Value)
{
break;
}
}
}
}
}
private bool LoadLine(Tuple <long, string> pair, ref object rec)
{
Type recType = rec.GetType();
try
{
if (!RaiseBeforeRecordLoad(rec, ref pair))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping...");
rec = null;
return(true);
}
if (pair.Item2 == null)
{
rec = null;
return(true);
}
else if (pair.Item2 == String.Empty)
{
return(true);
}
if (!pair.Item2.IsNullOrWhiteSpace())
{
var config = GetConfiguration(rec.GetType());
if (config == null)
{
throw new ChoParserException("No parser found to parse record line.");
}
if (config.GetType() == typeof(ChoCSVRecordConfiguration))
{
var r = ChoCSVReader.LoadText(rec.GetType(), pair.Item2, config as ChoCSVRecordConfiguration, Configuration.Encoding, Configuration.BufferSize, TraceSwitch, Reader);
rec = r.FirstOrDefault <object>();
}
else if (config.GetType() == typeof(ChoFixedLengthRecordConfiguration))
{
var r = ChoFixedLengthReader.LoadText(rec.GetType(), pair.Item2, config as ChoFixedLengthRecordConfiguration, Configuration.Encoding, Configuration.BufferSize, TraceSwitch, Reader);
rec = r.FirstOrDefault <object>();
}
else
{
throw new ChoParserException("Unsupported record line found to parse.");
}
}
bool skip = false;
if (!RaiseAfterRecordLoad(rec, pair, ref skip))
{
return(false);
}
else if (skip)
{
rec = null;
return(true);
}
}
catch (ChoParserException pEx)
{
throw new ChoParserException($"Failed to parse line to '{recType}' object.", pEx);
}
catch (ChoMissingRecordFieldException mEx)
{
throw new ChoParserException($"Failed to parse line to '{recType}' object.", mEx);
}
catch (Exception ex)
{
Reader.IsValid = false;
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw;
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
}
else
{
throw;
}
//if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
//{
// rec = null;
//}
//else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
//{
// if (!RaiseRecordLoadError(rec, pair, ex))
// throw new ChoReaderException($"Failed to parse line to '{recType}' object.", ex);
//}
//else
// throw new ChoReaderException($"Failed to parse line to '{recType}' object.", ex);
return(true);
}
return(true);
}
private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
{
TraceSwitch = traceSwitch;
TextReader sr = source as TextReader;
ChoGuard.ArgumentNotNull(sr, "TextReader");
if (sr is StreamReader)
{
((StreamReader)sr).Seek(0, SeekOrigin.Begin);
}
if (!RaiseBeginLoad(sr))
{
yield break;
}
string[] commentTokens = Configuration.Comments;
bool? skip = false;
bool isRecordStartFound = false;
bool isRecordEndFound = false;
long seekOriginPos = sr is StreamReader ? ((StreamReader)sr).BaseStream.Position : 0;
List <string> headers = new List <string>();
Tuple <long, string> lastLine = null;
List <Tuple <long, string> > recLines = new List <Tuple <long, string> >();
long recNo = 0;
int loopCount = Configuration.AutoDiscoverColumns && Configuration.KVPRecordFieldConfigurations.Count == 0 ? 2 : 1;
bool isHeaderFound = loopCount == 1;
bool IsHeaderLoaded = false;
Tuple <long, string> pairIn;
bool abortRequested = false;
for (int i = 0; i < loopCount; i++)
{
if (i == 1)
{
if (sr is StreamReader)
{
((StreamReader)sr).Seek(seekOriginPos, SeekOrigin.Begin);
}
TraceSwitch = traceSwitch;
}
else
{
TraceSwitch = ChoETLFramework.TraceSwitchOff;
}
lastLine = null;
recLines.Clear();
isRecordEndFound = false;
isRecordStartFound = false;
using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >(
new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, Configuration.MayContainEOLInData)).ToEnumerable(),
(pair) =>
{
//bool isStateAvail = IsStateAvail();
skip = false;
//if (isStateAvail)
//{
// if (!IsStateMatches(item))
// {
// skip = filterFunc != null ? filterFunc(item) : false;
// }
// else
// skip = true;
//}
//else
// skip = filterFunc != null ? filterFunc(item) : false;
if (skip == null)
{
return(null);
}
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine);
if (!skip.Value)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1));
}
}
if (skip.Value)
{
return(skip);
}
//if (!(sr.BaseStream is MemoryStream))
// ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1));
//if (Task != null)
// return !IsStateNOTExistsOrNOTMatch(item);
if (pair.Item2.IsNullOrWhiteSpace())
{
if (!Configuration.IgnoreEmptyLine)
{
throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1));
}
else
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1));
}
return(true);
}
}
if (commentTokens != null && commentTokens.Length > 0)
{
foreach (string comment in commentTokens)
{
if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal)) //, true, Configuration.Culture))
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1));
}
return(true);
}
}
}
if (!_configCheckDone)
{
Configuration.Validate(null);
var dict = Configuration.KVPRecordFieldConfigurations.ToDictionary(i1 => i1.Name, i1 => i1.FieldType == null ? null : i1.FieldType);
RaiseMembersDiscovered(dict);
Configuration.UpdateFieldTypesIfAny(dict);
_configCheckDone = true;
}
return(false);
}))
{
while (true)
{
pairIn = e.Peek;
if (!isRecordStartFound)
{
if (pairIn == null)
{
break;
}
lastLine = null;
recLines.Clear();
isRecordEndFound = false;
isRecordStartFound = true;
if (!Configuration.RecordStart.IsNullOrWhiteSpace())
{
//Move to record start
while (!(Configuration.IsRecordStartMatch(e.Peek.Item2)))
{
e.MoveNext();
if (e.Peek == null)
{
break;
}
}
}
if (e.Peek != null)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Record start found at [{0}] line...".FormatString(e.Peek.Item1));
}
e.MoveNext();
continue;
}
else
{
string recordEnd = !Configuration.RecordEnd.IsNullOrWhiteSpace() ? Configuration.RecordEnd : Configuration.RecordStart;
if (!recordEnd.IsNullOrWhiteSpace())
{
if (e.Peek == null)
{
if (Configuration.RecordEnd.IsNullOrWhiteSpace())
{
isRecordEndFound = true;
}
else
{
break;
}
}
else
{
//Move to record start
if (Configuration.IsRecordEndMatch(e.Peek.Item2))
{
isRecordEndFound = true;
isRecordStartFound = false;
if (e.Peek != null)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Record end found at [{0}] line...".FormatString(e.Peek.Item1));
}
}
}
}
else if (e.Peek == null)
{
isRecordEndFound = true;
}
if (!isHeaderFound)
{
//if (isRecordEndFound && headers.Count == 0)
//{
// //throw new ChoParserException("Unexpected EOF found.");
//}
if (!isRecordEndFound)
{
e.MoveNext();
if (e.Peek != null)
{
//If line empty or line continuation, skip
if (pairIn.Item2.IsNullOrWhiteSpace() || IsLineContinuationCharFound(pairIn.Item2)) //.Item2[0] == ' ' || pairIn.Item2[0] == '\t')
{
}
else
{
string header = ToKVP(pairIn.Item1, pairIn.Item2).Key;
if (!header.IsNullOrWhiteSpace())
{
headers.Add(header);
}
}
}
}
else
{
Configuration.Validate(headers.ToArray());
isHeaderFound = true;
isRecordEndFound = false;
IsHeaderLoaded = true;
break;
}
}
else
{
if (!IsHeaderLoaded)
{
Configuration.Validate(new string[] { });
IsHeaderLoaded = true;
}
if (isRecordEndFound && recLines.Count == 0)
{
//throw new ChoParserException("Unexpected EOF found.");
}
else if (!isRecordEndFound)
{
e.MoveNext();
if (e.Peek != null)
{
//If line empty or line continuation, skip
if (pairIn.Item2.IsNullOrWhiteSpace())
{
if (!Configuration.IgnoreEmptyLine)
{
throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pairIn.Item1));
}
else
{
Tuple <long, string> t = new Tuple <long, string>(lastLine.Item1, lastLine.Item2 + Configuration.EOLDelimiter);
recLines.RemoveAt(recLines.Count - 1);
recLines.Add(t);
}
}
else if (IsLineContinuationCharFound(pairIn.Item2)) //pairIn.Item2[0] == ' ' || pairIn.Item2[0] == '\t')
{
if (lastLine == null)
{
throw new ChoParserException("Unexpected line continuation found at {0} location.".FormatString(pairIn.Item1));
}
else
{
Tuple <long, string> t = new Tuple <long, string>(lastLine.Item1, lastLine.Item2 + Configuration.EOLDelimiter + pairIn.Item2);
recLines.RemoveAt(recLines.Count - 1);
recLines.Add(t);
}
}
else
{
lastLine = pairIn;
recLines.Add(pairIn);
}
}
}
else
{
object rec = Configuration.IsDynamicObject ? new ChoDynamicObject(new Dictionary <string, object>(Configuration.FileHeaderConfiguration.StringComparer))
{
ThrowExceptionIfPropNotExists = true,
AlternativeKeys = Configuration.AlternativeKeys
} : Activator.CreateInstance(RecordType);
if (!LoadLines(new Tuple <long, List <Tuple <long, string> > >(++recNo, recLines), ref rec))
{
yield break;
}
isRecordStartFound = false;
//StoreState(e.Current, rec != null);
if (!Configuration.RecordEnd.IsNullOrWhiteSpace())
{
e.MoveNext();
}
if (rec == null)
{
continue;
}
yield return(rec);
if (Configuration.NotifyAfter > 0 && recNo % Configuration.NotifyAfter == 0)
{
if (RaisedRowsLoaded(recNo))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
abortRequested = true;
yield break;
}
}
if (e.Peek == null)
{
break;
}
}
}
}
}
}
}
if (!abortRequested)
{
RaisedRowsLoaded(recNo, true);
}
RaiseEndLoad(sr);
}
private bool ToText(long index, object rec, out string recText)
{
if (typeof(IChoScalarObject).IsAssignableFrom(Configuration.RecordType))
{
rec = ChoActivator.CreateInstance(Configuration.RecordType, rec);
}
recText = null;
StringBuilder msg = new StringBuilder();
object fieldValue = null;
string fieldText = null;
ChoFixedLengthRecordFieldConfiguration fieldConfig = null;
if (Configuration.ColumnCountStrict)
{
CheckColumnsStrict(rec);
}
//bool firstColumn = true;
PropertyInfo pi = null;
object rootRec = rec;
IDictionary <string, Object> dict = null;
foreach (KeyValuePair <string, ChoFixedLengthRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
{
fieldConfig = kvp.Value;
fieldValue = null;
fieldText = String.Empty;
if (Configuration.PIDict != null)
{
Configuration.PIDict.TryGetValue(kvp.Key, out pi);
}
rec = GetDeclaringRecord(kvp.Value.DeclaringMember, rootRec);
dict = rec.ToDynamicObject() as IDictionary <string, Object>;
if (Configuration.IsDynamicObject)
{
dict = dict.Flatten().ToDictionary();
}
if (Configuration.ThrowAndStopOnMissingField)
{
if (Configuration.IsDynamicObject)
{
if (!dict.ContainsKey(kvp.Key))
{
throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' FixedLength column.".FormatString(fieldConfig.FieldName));
}
}
else
{
if (pi == null)
{
throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' FixedLength column.".FormatString(fieldConfig.FieldName));
}
}
}
try
{
if (Configuration.IsDynamicObject)
{
fieldValue = dict[kvp.Key]; // dict.GetValue(kvp.Key, Configuration.FileHeaderConfiguration.IgnoreCase, Configuration.Culture);
if (kvp.Value.FieldType == null)
{
if (fieldValue == null)
{
kvp.Value.FieldType = typeof(string);
}
else
{
kvp.Value.FieldType = fieldValue.GetType();
}
}
}
else
{
if (pi != null)
{
fieldValue = ChoType.GetPropertyValue(rec, pi);
if (kvp.Value.FieldType == null)
{
kvp.Value.FieldType = pi.PropertyType;
}
}
else
{
kvp.Value.FieldType = typeof(string);
}
}
//Discover default value, use it if null
if (fieldValue == null)
{
if (fieldConfig.IsDefaultValueSpecified)
{
fieldValue = fieldConfig.DefaultValue;
}
}
if (!RaiseBeforeRecordFieldWrite(rec, index, kvp.Key, ref fieldValue))
{
return(false);
}
if (fieldConfig.ValueConverter != null)
{
fieldValue = fieldConfig.ValueConverter(fieldValue);
}
else
{
rec.GetNConvertMemberValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue);
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.MemberLevel)
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
if (!RaiseAfterRecordFieldWrite(rec, index, kvp.Key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
throw;
}
catch (ChoMissingRecordFieldException)
{
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ref ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
if (Configuration.IsDynamicObject)
{
if (dict.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else if (dict.GetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else
{
var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
fieldValue = null;
throw ex1;
}
}
else if (pi != null)
{
if (rec.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (rec.GetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else
{
var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
fieldValue = null;
throw ex1;
}
}
else
{
var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
fieldValue = null;
throw ex1;
}
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
continue;
}
else
{
if (!RaiseRecordFieldWriteError(rec, index, kvp.Key, fieldText, ex))
{
throw new ChoWriterException($"Failed to write '{fieldValue}' value of '{kvp.Key}' member.", ex);
}
}
}
else
{
throw new ChoWriterException("Failed to use '{0}' fallback value for '{1}' member.".FormatString(fieldValue, kvp.Key), innerEx);
}
}
}
if (fieldValue == null)
{
fieldText = String.Empty;
}
else
{
fieldText = fieldValue.ToString();
}
msg.Append(NormalizeFieldValue(kvp.Key, fieldText, kvp.Value.Size, kvp.Value.Truncate, kvp.Value.QuoteField,
GetFieldValueJustification(kvp.Value.FieldValueJustification, kvp.Value.FieldType),
GetFillChar(kvp.Value.FillChar, kvp.Value.FieldType), false, kvp.Value.NullValue,
kvp.Value.GetFieldValueTrimOption(kvp.Value.FieldType)));
}
recText = msg.ToString();
return(true);
}
static ChoRecordWriter()
{
ChoETLFramework.Initialize();
}
public static void Write(string msg)
{
ChoETLFramework.WriteLog(msg);
}
public static void Warning(string msg)
{
ChoETLFramework.WriteLog(ChoETLFramework.TraceSwitch.TraceWarning, msg);
}
public static void Verbose(string msg)
{
ChoETLFramework.WriteLog(ChoETLFramework.TraceSwitch.TraceVerbose, msg);
}
public static void Info(string msg)
{
ChoETLFramework.WriteLog(ChoETLFramework.TraceSwitch.TraceInfo, msg);
}
private void LoadDefaults(Type declaringType)
{
ChoIniFile iniFile = GetIniSection(declaringType, "DEFAULT_VALUE");
var dict = new Dictionary <string, object>();
object defaultValue = null;
string memberName = null;
foreach (PropertyDescriptor pd in TypeDescriptor.GetProperties(declaringType).AsTypedEnumerable <PropertyDescriptor>().Where(pd => !pd.Attributes.OfType <ChoIgnoreMemberAttribute>().Any()))
{
try
{
memberName = pd.Name;
if (_turnOnMetaDataCache)
{
defaultValue = iniFile.GetValue(pd.Name, ChoType.GetDefaultValue(pd), _isLockedCache[declaringType]);
}
else
{
defaultValue = ChoType.GetDefaultValue(pd);
}
if (defaultValue is string)
{
defaultValue = ((string)defaultValue).ExpandProperties();
}
defaultValue = ChoConvert.ConvertFrom(defaultValue, pd.PropertyType);
dict.Add(pd.Name, defaultValue);
}
catch (Exception ex)
{
ChoETLFramework.WriteLog(ChoETLFramework.Switch.TraceError, "Error while converting default value '{0}' to '{1}' member type. {2}".FormatString(defaultValue, ChoType.GetMemberName(pd), ex.Message));
}
}
foreach (FieldInfo memberInfo in ChoType.GetFields(declaringType))
{
try
{
memberName = memberInfo.Name;
if (_turnOnMetaDataCache)
{
defaultValue = iniFile.GetValue(memberInfo.Name, ChoType.GetDefaultValue(memberInfo), _isLockedCache[declaringType]);
}
else
{
defaultValue = ChoType.GetDefaultValue(memberInfo);
}
if (defaultValue is string)
{
defaultValue = ((string)defaultValue).ExpandProperties();
}
defaultValue = ChoConvert.ConvertFrom(defaultValue, memberInfo.FieldType);
dict.Add(memberInfo.Name, defaultValue);
}
catch (Exception ex)
{
ChoETLFramework.WriteLog(ChoETLFramework.Switch.TraceError, "Error while converting default value '{0}' to '{1}' member type. {2}".FormatString(defaultValue, ChoType.GetMemberName(memberInfo), ex.Message));
}
}
_defaultsCache.Add(declaringType, dict);
}
public static void Error(string msg)
{
ChoETLFramework.WriteLog(ChoETLFramework.TraceSwitch.TraceError, msg);
}
private bool LoadLine(Tuple <int, string> pair, ref object rec)
{
Type recType = rec.GetType();
try
{
if (!RaiseBeforeRecordLoad(rec, ref pair))
{
return(false);
}
if (pair.Item2 == null)
{
rec = null;
return(true);
}
else if (pair.Item2 == String.Empty)
{
return(true);
}
if (!pair.Item2.IsNullOrWhiteSpace())
{
var config = GetConfiguration(rec.GetType());
if (config == null)
{
throw new ChoParserException("No parser found to parse record line.");
}
if (config.GetType() == typeof(ChoCSVRecordConfiguration))
{
var r = ChoCSVReader.LoadText(rec.GetType(), pair.Item2, config as ChoCSVRecordConfiguration, Configuration.Encoding, Configuration.BufferSize);
rec = r.FirstOrDefault <object>();
}
else if (config.GetType() == typeof(ChoFixedLengthRecordConfiguration))
{
var r = ChoFixedLengthReader.LoadText(rec.GetType(), pair.Item2, config as ChoFixedLengthRecordConfiguration, Configuration.Encoding, Configuration.BufferSize);
rec = r.FirstOrDefault <object>();
}
else
{
throw new ChoParserException("Unsupported record line found to parse.");
}
}
if (!RaiseAfterRecordLoad(rec, pair))
{
return(false);
}
}
catch (ChoParserException pEx)
{
throw new ChoParserException($"Failed to parse line to '{recType}' object.", pEx);
}
catch (ChoMissingRecordFieldException mEx)
{
throw new ChoParserException($"Failed to parse line to '{recType}' object.", mEx);
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
rec = null;
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw new ChoParserException($"Failed to parse line to '{recType}' object.", ex);
}
}
else
{
throw new ChoParserException($"Failed to parse line to '{recType}' object.", ex);
}
return(true);
}
return(true);
}
public static void Write(bool condition, string msg)
{
ChoETLFramework.WriteLog(condition, msg);
}
private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
{
TraceSwitch = traceSwitch;
StreamReader sr = source as StreamReader;
ChoGuard.ArgumentNotNull(sr, "StreamReader");
if (Configuration.RecordSelector == null)
{
throw new ChoRecordConfigurationException("Missing record selector.");
}
sr.Seek(0, SeekOrigin.Begin);
if (!RaiseBeginLoad(sr))
{
yield break;
}
string[] commentTokens = Configuration.Comments;
bool _headerFound = false;
using (ChoPeekEnumerator <Tuple <int, string> > e = new ChoPeekEnumerator <Tuple <int, string> >(
new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar)).ToEnumerable(),
(pair) =>
{
//bool isStateAvail = IsStateAvail();
bool?skip = false;
//if (isStateAvail)
//{
// if (!IsStateMatches(item))
// {
// skip = filterFunc != null ? filterFunc(item) : false;
// }
// else
// skip = true;
//}
//else
// skip = filterFunc != null ? filterFunc(item) : false;
if (skip == null)
{
return(null);
}
//if (!(sr.BaseStream is MemoryStream))
//{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine);
if (!skip.Value)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1));
}
//}
if (skip.Value)
{
return(skip);
}
//if (!(sr.BaseStream is MemoryStream))
// ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1));
//if (Task != null)
// return !IsStateNOTExistsOrNOTMatch(item);
if (pair.Item2.IsNullOrWhiteSpace())
{
if (!Configuration.IgnoreEmptyLine)
{
throw new ChoParserException("Empty line found at {0} location.".FormatString(e.Peek.Item1));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Empty line found at [{0}]...".FormatString(pair.Item1));
return(true);
}
}
if (commentTokens == null)
{
return(false);
}
else
{
var x = (from comment in commentTokens
where !pair.Item2.IsNull() && pair.Item2.StartsWith(comment, true, Configuration.Culture)
select comment).FirstOrDefault();
if (x != null)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1));
return(true);
}
}
if (!_configCheckDone)
{
Configuration.Validate(pair); // GetHeaders(pair.Item2));
_configCheckDone = true;
}
//Ignore Header if any
if (Configuration.FileHeaderConfiguration.HasHeaderRecord &&
!_headerFound)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring header line at [{0}]...".FormatString(pair.Item1));
_headerFound = true;
return(true);
}
return(false);
}))
{
while (true)
{
Tuple <int, string> pair = e.Peek;
if (pair == null)
{
RaiseEndLoad(sr);
yield break;
}
Type recType = Configuration.RecordSelector(pair.Item2);
if (recType == null)
{
if (Configuration.IgnoreIfNoRecordParserExists)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, $"No record type found for [{pair.Item1}] line to parse.");
}
else
{
throw new ChoParserException($"No record type found for [{pair.Item1}] line to parse.");
}
}
object rec = ChoActivator.CreateInstance(recType);
if (!LoadLine(pair, ref rec))
{
yield break;
}
//StoreState(e.Current, rec != null);
e.MoveNext();
if (rec == null)
{
continue;
}
yield return(rec);
}
}
}
public override IEnumerable <object> WriteTo(object writer, IEnumerable <object> records, Func <object, bool> predicate = null)
{
TextWriter sw = writer as TextWriter;
ChoGuard.ArgumentNotNull(sw, "TextWriter");
if (records == null)
{
yield break;
}
if (!RaiseBeginWrite(sw))
{
yield break;
}
CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture;
System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture;
string recText = String.Empty;
long recCount = 0;
var recEnum = records.GetEnumerator();
try
{
object notNullRecord = GetFirstNotNullRecord(recEnum);
if (notNullRecord == null)
{
yield break;
}
if (Configuration.IsDynamicObject)
{
if (Configuration.MaxScanRows > 0)
{
List <string> fns = new List <string>();
foreach (object record1 in GetRecords(recEnum))
{
recCount++;
if (record1 != null)
{
if (recCount <= Configuration.MaxScanRows)
{
if (!record1.GetType().IsDynamicType())
{
throw new ChoParserException("Invalid record found.");
}
_recBuffer.Value.Add(record1);
fns = fns.Union(GetFields(record1)).ToList();
if (recCount == Configuration.MaxScanRows)
{
Configuration.Validate(fns.ToArray());
WriteHeaderLine(sw);
_configCheckDone = true;
break;
}
}
}
}
}
}
foreach (object record in GetRecords(recEnum))
{
_index++;
if (TraceSwitch.TraceVerbose)
{
if (record is IChoETLNameableObject)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index));
}
}
recText = String.Empty;
if (record != null)
{
if (predicate == null || predicate(record))
{
//Discover and load FixedLength columns from first record
if (!_configCheckDone)
{
if (notNullRecord != null)
{
string[] fieldNames = GetFields(notNullRecord);
Configuration.Validate(fieldNames);
WriteHeaderLine(sw);
_configCheckDone = true;
}
}
//Check record
if (record != null)
{
Type rt = record.GetType().ResolveType();
if (Configuration.IsDynamicObject)
{
if (ElementType != null)
{
}
else if (!rt.IsDynamicType())
{
throw new ChoWriterException("Invalid record found.");
}
}
else
{
if (rt != Configuration.RecordType)
{
throw new ChoWriterException("Invalid record found.");
}
}
}
if (!RaiseBeforeRecordWrite(record, _index, ref recText))
{
yield break;
}
if (recText == null)
{
continue;
}
else if (recText.Length > 0)
{
sw.Write("{1}{0}", recText, _hadHeaderWritten ? Configuration.EOLDelimiter : "");
continue;
}
try
{
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
record.DoObjectLevelValidation(Configuration, Configuration.FixedLengthRecordFieldConfigurations);
}
if (ToText(_index, record, out recText))
{
if (_index == 1)
{
sw.Write("{1}{0}", recText, _hadHeaderWritten ? Configuration.EOLDelimiter : "");
}
else
{
sw.Write("{1}{0}", recText, Configuration.EOLDelimiter);
}
if (!RaiseAfterRecordWrite(record, _index, recText))
{
yield break;
}
}
}
//catch (ChoParserException)
//{
// throw;
//}
catch (Exception ex)
{
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordWriteError(record, _index, recText, ex))
{
throw;
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
}
else
{
throw;
}
}
}
}
yield return(record);
if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0)
{
if (RaisedRowsWritten(_index))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
yield break;
}
}
}
}
finally
{
System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo;
}
RaiseEndWrite(sw);
}
private bool LoadLine(Tuple <long, string> pair, ref object rec)
{
try
{
if (!RaiseBeforeRecordLoad(rec, ref pair))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping...");
rec = null;
return(true);
}
if (pair.Item2 == null)
{
rec = null;
return(true);
}
else if (pair.Item2 == String.Empty)
{
return(true);
}
if (!pair.Item2.IsNullOrWhiteSpace())
{
if (!FillRecord(rec, pair))
{
return(false);
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
rec.DoObjectLevelValidation(Configuration, Configuration.FixedLengthRecordFieldConfigurations);
}
}
bool skip = false;
if (!RaiseAfterRecordLoad(rec, pair, ref skip))
{
return(false);
}
else if (skip)
{
rec = null;
return(true);
}
}
//catch (ChoParserException)
//{
// throw;
//}
//catch (ChoMissingRecordFieldException)
//{
// throw;
//}
catch (Exception ex)
{
Reader.IsValid = false;
if (ex is ChoMissingRecordFieldException && Configuration.ThrowAndStopOnMissingField)
{
throw;
}
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw;
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
}
else
{
throw;
}
return(true);
}
return(true);
}
private bool LoadLines(Tuple <long, List <Tuple <long, string> > > pairs, ref object rec)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading [{0}] record...".FormatString(pairs.Item1));
Tuple <long, string> pair = null;
long recNo = pairs.Item1;
if (!Configuration.AutoDiscoveredColumns)
{
if (Configuration.ColumnCountStrict)
{
if (pairs.Item2.Count != Configuration.KVPRecordFieldConfigurations.Count)
{
throw new ChoParserException("Incorrect number of field values found at record [{2}]. Expected [{0}] field values. Found [{1}] field values.".FormatString(Configuration.KVPRecordFieldConfigurations.Count, pairs.Item2.Count, recNo));
}
List <string> keys = (from p in pairs.Item2
select ToKVP(p.Item1, p.Item2).Key).ToList();
if (!Enumerable.SequenceEqual(keys.OrderBy(t => t), Configuration.FCArray.Select(a => a.Value.FieldName).OrderBy(t => t), Configuration.FileHeaderConfiguration.StringComparer))
{
throw new ChoParserException("Column count mismatch detected at [{0}] record.".FormatString(recNo));
}
}
if (Configuration.ColumnOrderStrict)
{
List <string> keys = (from p in pairs.Item2
select ToKVP(p.Item1, p.Item2).Key).ToList();
int runnngIndex = -1;
foreach (var k in keys)
{
runnngIndex++;
if (runnngIndex < Configuration.FCArray.Length)
{
if (Configuration.FileHeaderConfiguration.IsEqual(Configuration.FCArray[runnngIndex].Value.FieldName, k))
{
continue;
}
}
throw new ChoParserException("Found incorrect order on '{1}' column at [{0}] record.".FormatString(recNo, k));
}
}
}
object fieldValue = String.Empty;
PropertyInfo pi = null;
//Set default values
foreach (KeyValuePair <string, ChoKVPRecordFieldConfiguration> kvp in Configuration.FCArray)
{
if (Configuration.PIDict != null)
{
Configuration.PIDict.TryGetValue(kvp.Key, out pi);
}
try
{
if (kvp.Value.IsDefaultValueSpecified)
{
fieldValue = kvp.Value.DefaultValue;
}
if (Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
dict.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
}
else
{
if (pi != null)
{
rec.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
}
}
}
catch
{
}
}
_propInit.Clear();
foreach (var pair1 in pairs.Item2)
{
pair = pair1;
try
{
if (!RaiseBeforeRecordLoad(rec, ref pair))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping...");
rec = null;
return(true);
}
if (pair.Item2 == null)
{
rec = null;
return(true);
}
else if (pair.Item2 == String.Empty)
{
return(true);
}
if (!pair.Item2.IsNullOrWhiteSpace())
{
if (!FillRecord(rec, pair))
{
return(false);
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
rec.DoObjectLevelValidation(Configuration, Configuration.KVPRecordFieldConfigurations);
}
}
bool skip = false;
if (!RaiseAfterRecordLoad(rec, pair, ref skip))
{
return(false);
}
else if (skip)
{
rec = null;
return(true);
}
}
//catch (ChoParserException)
//{
// throw;
//}
//catch (ChoMissingRecordFieldException)
//{
// throw;
//}
catch (Exception ex)
{
if (ex is ChoMissingRecordFieldException && Configuration.ThrowAndStopOnMissingField)
{
throw;
}
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw;
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
}
else
{
throw;
}
return(true);
}
}
return(true);
}
private bool FillRecord(object rec, Tuple <long, string> pair)
{
long lineNo;
string line;
lineNo = pair.Item1;
line = pair.Item2;
if (line.Length != Configuration.RecordLength)
{
throw new ChoParserException("Incorrect record length [Length: {0}] found. Expected record length: {1}".FormatString(line.Length, Configuration.RecordLength));
}
object fieldValue = null;
ChoFixedLengthRecordFieldConfiguration fieldConfig = null;
PropertyInfo pi = null;
object rootRec = rec;
foreach (KeyValuePair <string, ChoFixedLengthRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
{
fieldValue = null;
fieldConfig = kvp.Value;
if (Configuration.PIDict != null)
{
Configuration.PIDict.TryGetValue(kvp.Key, out pi);
}
rec = GetDeclaringRecord(kvp.Value.DeclaringMember, rootRec);
try
{
if (fieldConfig.StartIndex + fieldConfig.Size > line.Length)
{
if (Configuration.ColumnCountStrict)
{
throw new ChoParserException("Missing '{0}' field value.".FormatString(kvp.Key));
}
}
else
{
fieldValue = line.Substring(fieldConfig.StartIndex, fieldConfig.Size.Value);
}
if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject)
{
if (kvp.Value.FieldType == null)
{
kvp.Value.FieldType = Configuration.MaxScanRows == -1 ? DiscoverFieldType(fieldValue as string, Configuration) : typeof(string);
}
}
else
{
if (pi != null)
{
kvp.Value.FieldType = pi.PropertyType;
}
else
{
kvp.Value.FieldType = typeof(string);
}
}
fieldValue = CleanFieldValue(fieldConfig, kvp.Value.FieldType, fieldValue as string);
if (!RaiseBeforeRecordFieldLoad(rec, pair.Item1, kvp.Key, ref fieldValue))
{
continue;
}
bool ignoreFieldValue = fieldConfig.IgnoreFieldValue(fieldValue);
if (ignoreFieldValue)
{
fieldValue = fieldConfig.IsDefaultValueSpecified ? fieldConfig.DefaultValue : null;
}
if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
dict.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
}
else
{
if (Configuration.SupportsMultiRecordTypes)
{
ChoType.TryGetProperty(rec.GetType(), kvp.Key, out pi);
fieldConfig.PI = pi;
fieldConfig.PropConverters = ChoTypeDescriptor.GetTypeConverters(fieldConfig.PI);
fieldConfig.PropConverterParams = ChoTypeDescriptor.GetTypeConverterParams(fieldConfig.PI);
}
if (pi != null)
{
rec.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
}
else if (!Configuration.SupportsMultiRecordTypes)
{
throw new ChoMissingRecordFieldException("Missing '{0}' property in {1} type.".FormatString(kvp.Key, ChoType.GetTypeName(rec)));
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
}
if (!RaiseAfterRecordFieldLoad(rec, pair.Item1, kvp.Key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
Reader.IsValid = false;
throw;
}
catch (ChoMissingRecordFieldException)
{
Reader.IsValid = false;
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
Reader.IsValid = false;
ChoETLFramework.HandleException(ref ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
if (Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
if (dict.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (dict.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (ex is ValidationException)
{
throw;
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else if (pi != null)
{
if (rec.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (rec.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (ex is ValidationException)
{
throw;
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException || ex is ValidationException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
continue;
}
else
{
if (!RaiseRecordFieldLoadError(rec, pair.Item1, kvp.Key, fieldValue, ex))
{
if (ex is ValidationException)
{
throw;
}
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
}
else
{
throw new ChoReaderException("Failed to assign '{0}' fallback value to '{1}' field.".FormatString(fieldValue, fieldConfig.FieldName), innerEx);
}
}
}
}
return(true);
}
private bool FillRecord(object rec, Tuple <long, string> pair)
{
long lineNo;
string line;
lineNo = pair.Item1;
line = pair.Item2;
var tokens = ToKVP(pair.Item1, pair.Item2);
//ValidateLine(pair.Item1, fieldValues);
object fieldValue = tokens.Value;
string key = tokens.Key;
if (!Configuration.RecordFieldConfigurationsDict2.ContainsKey(key))
{
return(true);
}
key = Configuration.RecordFieldConfigurationsDict2[key].Name;
ChoKVPRecordFieldConfiguration fieldConfig = Configuration.RecordFieldConfigurationsDict[key];
PropertyInfo pi = null;
if (_propInit.ContainsKey(key))
{
return(true);
}
_propInit.Add(key, true);
fieldValue = CleanFieldValue(fieldConfig, fieldConfig.FieldType, fieldValue as string);
if (Configuration.IsDynamicObject)
{
if (fieldConfig.FieldType == null)
{
fieldConfig.FieldType = typeof(string);
}
}
else
{
if (Configuration.PIDict != null)
{
Configuration.PIDict.TryGetValue(key, out pi);
}
if (pi != null)
{
fieldConfig.FieldType = pi.PropertyType;
}
else
{
fieldConfig.FieldType = typeof(string);
}
}
fieldValue = CleanFieldValue(fieldConfig, fieldConfig.FieldType, fieldValue as string);
if (!RaiseBeforeRecordFieldLoad(rec, pair.Item1, key, ref fieldValue))
{
return(true);
}
try
{
bool ignoreFieldValue = fieldConfig.IgnoreFieldValue(fieldValue);
if (ignoreFieldValue)
{
fieldValue = fieldConfig.IsDefaultValueSpecified ? fieldConfig.DefaultValue : null;
}
if (Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
dict.ConvertNSetMemberValue(key, fieldConfig, ref fieldValue, Configuration.Culture);
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
{
dict.DoMemberLevelValidation(key, fieldConfig, Configuration.ObjectValidationMode);
}
}
else
{
if (pi != null)
{
rec.ConvertNSetMemberValue(key, fieldConfig, ref fieldValue, Configuration.Culture);
}
else
{
throw new ChoMissingRecordFieldException("Missing '{0}' property in {1} type.".FormatString(key, ChoType.GetTypeName(rec)));
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
{
rec.DoMemberLevelValidation(key, fieldConfig, Configuration.ObjectValidationMode);
}
}
if (!RaiseAfterRecordFieldLoad(rec, pair.Item1, key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
Reader.IsValid = false;
throw;
}
catch (ChoMissingRecordFieldException)
{
Reader.IsValid = false;
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
Reader.IsValid = false;
ChoETLFramework.HandleException(ref ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
if (Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
if (dict.SetFallbackValue(key, fieldConfig, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(key, fieldConfig, Configuration.ObjectValidationMode);
}
else if (dict.SetDefaultValue(key, fieldConfig, Configuration.Culture))
{
dict.DoMemberLevelValidation(key, fieldConfig, Configuration.ObjectValidationMode);
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else if (pi != null)
{
if (rec.SetFallbackValue(key, fieldConfig, Configuration.Culture))
{
rec.DoMemberLevelValidation(key, fieldConfig, Configuration.ObjectValidationMode);
}
else if (rec.SetDefaultValue(key, fieldConfig, Configuration.Culture))
{
rec.DoMemberLevelValidation(key, fieldConfig, Configuration.ObjectValidationMode);
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
}
else
{
if (!RaiseRecordFieldLoadError(rec, pair.Item1, key, fieldValue, ex))
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
}
else
{
throw new ChoReaderException("Failed to assign '{0}' fallback value to '{1}' field.".FormatString(fieldValue, fieldConfig.FieldName), innerEx);
}
}
}
return(true);
}
private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
{
TraceSwitch = traceSwitch;
TextReader sr = source as TextReader;
ChoGuard.ArgumentNotNull(sr, "TextReader");
if (sr is StreamReader)
{
((StreamReader)sr).Seek(0, SeekOrigin.Begin);
}
if (!RaiseBeginLoad(sr))
{
yield break;
}
string[] commentTokens = Configuration.Comments;
bool? skip = false;
bool abortRequested = false;
long runningCount = 0;
long recCount = 0;
bool headerLineLoaded = false;
List <object> buffer = new List <object>();
IDictionary <string, Type> recFieldTypes = null;
bool?skipUntil = true;
bool?doWhile = true;
using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >(
new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, false /*Configuration.MayContainEOLInData*/)).ToEnumerable(),
(pair) =>
{
//bool isStateAvail = IsStateAvail();
skip = false;
if (skipUntil != null)
{
if (skipUntil.Value)
{
skipUntil = RaiseSkipUntil(pair);
if (skipUntil == null)
{
}
else
{
if (skipUntil.Value)
{
skip = skipUntil;
}
else
{
skip = true;
}
}
}
}
//if (isStateAvail)
//{
// if (!IsStateMatches(item))
// {
// skip = filterFunc != null ? filterFunc(item) : false;
// }
// else
// skip = true;
//}
//else
// skip = filterFunc != null ? filterFunc(item) : false;
if (skip == null)
{
return(null);
}
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine);
if (!skip.Value)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1));
}
}
if (skip.Value)
{
return(skip);
}
//if (!(sr.BaseStream is MemoryStream))
// ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1));
//if (Task != null)
// return !IsStateNOTExistsOrNOTMatch(item);
if (pair.Item2.IsNullOrWhiteSpace())
{
if (!Configuration.IgnoreEmptyLine)
{
throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1));
}
else
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1));
}
return(true);
}
}
if (commentTokens != null && commentTokens.Length > 0)
{
foreach (string comment in commentTokens)
{
if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal)) //, true, Configuration.Culture))
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1));
}
return(true);
}
}
}
if (Configuration.FileHeaderConfiguration.HeaderLineAt > 0)
{
if (pair.Item1 < Configuration.FileHeaderConfiguration.HeaderLineAt)
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Header line at {1}. Skipping [{0}] line...".FormatString(pair.Item1, Configuration.FileHeaderConfiguration.HeaderLineAt));
}
return(true);
}
}
if (!_configCheckDone)
{
if (Configuration.SupportsMultiRecordTypes && Configuration.RecordSelector != null && !Configuration.RecordTypeMapped)
{
}
else
{
Configuration.Validate(pair); // GetHeaders(pair.Item2));
}
var dict = recFieldTypes = Configuration.FixedLengthRecordFieldConfigurations.ToDictionary(i => i.Name, i => i.FieldType == null ? null : i.FieldType);
RaiseMembersDiscovered(dict);
Configuration.UpdateFieldTypesIfAny(dict);
_configCheckDone = true;
}
//LoadHeader if any
if ((Configuration.FileHeaderConfiguration.HasHeaderRecord ||
Configuration.FileHeaderConfiguration.HeaderLineAt > 0) &&
!_headerFound)
{
if (Configuration.FileHeaderConfiguration.IgnoreHeader)
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring header line at [{0}]...".FormatString(pair.Item1));
}
}
else
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading header line at [{0}]...".FormatString(pair.Item1));
}
headerLineLoaded = true;
LoadHeaderLine(pair);
}
_headerFound = true;
return(true);
}
return(false);
}))
{
while (true)
{
recCount++;
Tuple <long, string> pair = e.Peek;
if (pair == null)
{
if (!abortRequested)
{
RaisedRowsLoaded(runningCount);
}
RaiseEndLoad(sr);
yield break;
}
runningCount = pair.Item1;
object rec = null;
if (Configuration.SupportsMultiRecordTypes && Configuration.RecordSelector != null)
{
Type recType = Configuration.RecordSelector(pair);
if (recType == null)
{
if (Configuration.IgnoreIfNoRecordTypeFound)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, $"No record type found for [{pair.Item1}] line to parse.");
e.MoveNext();
continue;
}
else
{
throw new ChoParserException($"No record type found for [{pair.Item1}] line to parse.");
}
}
if (!Configuration.RecordTypeMapped)
{
Configuration.MapRecordFields(recType);
Configuration.Validate(null);
}
rec = recType.IsDynamicType() ? new ChoDynamicObject(new Dictionary <string, object>(Configuration.FileHeaderConfiguration.StringComparer))
{
ThrowExceptionIfPropNotExists = true,
AlternativeKeys = Configuration.AlternativeKeys
} : ChoActivator.CreateInstance(recType);
}
else
{
rec = Configuration.IsDynamicObject ? new ChoDynamicObject(new Dictionary <string, object>(Configuration.FileHeaderConfiguration.StringComparer))
{
ThrowExceptionIfPropNotExists = true,
AlternativeKeys = Configuration.AlternativeKeys
} : ChoActivator.CreateInstance(RecordType);
}
if (!LoadLine(pair, ref rec))
{
yield break;
}
//StoreState(e.Current, rec != null);
e.MoveNext();
if (rec == null)
{
continue;
}
if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject)
{
if (Configuration.AreAllFieldTypesNull && Configuration.AutoDiscoverFieldTypes && Configuration.MaxScanRows > 0 && recCount <= Configuration.MaxScanRows)
{
buffer.Add(rec);
RaiseRecordFieldTypeAssessment(recFieldTypes, (IDictionary <string, object>)rec, recCount == Configuration.MaxScanRows);
if (recCount == Configuration.MaxScanRows || e.Peek == null)
{
Configuration.UpdateFieldTypesIfAny(recFieldTypes);
var dict = recFieldTypes = Configuration.FixedLengthRecordFieldConfigurations.ToDictionary(i => i.Name, i => i.FieldType == null ? null : i.FieldType);
RaiseMembersDiscovered(dict);
foreach (object rec1 in buffer)
{
yield return(ConvertToNestedObjectIfApplicable(new ChoDynamicObject(MigrateToNewSchema(rec1 as IDictionary <string, object>, recFieldTypes)) as object, headerLineLoaded));
}
}
}
else
{
yield return(ConvertToNestedObjectIfApplicable(rec, headerLineLoaded));
}
}
else
{
yield return(rec);
}
if (Configuration.NotifyAfter > 0 && pair.Item1 % Configuration.NotifyAfter == 0)
{
if (RaisedRowsLoaded(pair.Item1))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
yield break;
}
}
if (doWhile != null)
{
doWhile = RaiseDoWhile(pair);
if (doWhile != null && doWhile.Value)
{
break;
}
}
}
}
}