private IEnumerable <object> AsEnumerable(IEnumerable <JObject> jObjects, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null) { TraceSwitch = traceSwitch; long counter = 0; Tuple <long, JObject> pair = null; bool?skip = false; bool?skipUntil = true; bool?doWhile = true; bool abortRequested = false; _se = new Lazy <JsonSerializer>(() => Configuration.JsonSerializerSettings != null ? JsonSerializer.Create(Configuration.JsonSerializerSettings) : null); foreach (var obj in jObjects) { pair = new Tuple <long, JObject>(++counter, obj); skip = false; if (skipUntil != null) { if (skipUntil.Value) { skipUntil = RaiseSkipUntil(pair); if (skipUntil == null) { } else { if (skipUntil.Value) { skip = skipUntil; } else { skip = true; } } } } if (skip == null) { break; } if (skip.Value) { continue; } if (!_configCheckDone) { Configuration.Validate(pair); var dict = Configuration.JSONRecordFieldConfigurations.ToDictionary(i => i.Name, i => i.FieldType == null ? null : i.FieldType); RaiseMembersDiscovered(dict); _configCheckDone = true; } object rec = null; if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading node [{0}]...".FormatString(pair.Item1)); } if (!LoadNode(pair, ref rec)) { yield break; } if (rec == null) { continue; } yield return(rec); if (Configuration.NotifyAfter > 0 && pair.Item1 % Configuration.NotifyAfter == 0) { if (RaisedRowsLoaded(pair.Item1)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested."); abortRequested = true; yield break; } } if (doWhile != null) { doWhile = RaiseDoWhile(pair); if (doWhile != null && doWhile.Value) { break; } } } if (!abortRequested && pair != null) { RaisedRowsLoaded(pair.Item1); } }
private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null) { TraceSwitch = traceSwitch; TextReader sr = source as TextReader; ChoGuard.ArgumentNotNull(sr, "TextReader"); if (sr is StreamReader) { ((StreamReader)sr).Seek(0, SeekOrigin.Begin); } if (!RaiseBeginLoad(sr)) { yield break; } string[] commentTokens = Configuration.Comments; bool? skip = false; bool isRecordStartFound = false; bool isRecordEndFound = false; long seekOriginPos = sr is StreamReader ? ((StreamReader)sr).BaseStream.Position : 0; List <string> headers = new List <string>(); Tuple <long, string> lastLine = null; List <Tuple <long, string> > recLines = new List <Tuple <long, string> >(); long recNo = 0; int loopCount = Configuration.AutoDiscoverColumns && Configuration.KVPRecordFieldConfigurations.Count == 0 ? 2 : 1; bool isHeaderFound = loopCount == 1; bool IsHeaderLoaded = false; Tuple <long, string> pairIn; bool abortRequested = false; bool?skipUntil = true; bool?doWhile = true; for (int i = 0; i < loopCount; i++) { if (i == 1) { if (sr is StreamReader) { ((StreamReader)sr).Seek(seekOriginPos, SeekOrigin.Begin); } TraceSwitch = traceSwitch; } else { TraceSwitch = ChoETLFramework.TraceSwitchOff; } lastLine = null; recLines.Clear(); isRecordEndFound = false; isRecordStartFound = false; using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >( new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, Configuration.MayContainEOLInData)).ToEnumerable(), (pair) => { //bool isStateAvail = IsStateAvail(); skip = false; if (skipUntil != null) { if (skipUntil.Value) { skipUntil = RaiseSkipUntil(pair); if (skipUntil == null) { } else { skip = skipUntil.Value; } } } //if (isStateAvail) //{ // if (!IsStateMatches(item)) // { // skip = filterFunc != null ? filterFunc(item) : false; // } // else // skip = true; //} //else // skip = filterFunc != null ? filterFunc(item) : false; if (skip == null) { return(null); } if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine); if (!skip.Value) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1)); } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1)); } } if (skip.Value) { return(skip); } //if (!(sr.BaseStream is MemoryStream)) // ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1)); //if (Task != null) // return !IsStateNOTExistsOrNOTMatch(item); if (pair.Item2.IsNullOrWhiteSpace()) { if (!Configuration.IgnoreEmptyLine) { throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1)); } else { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1)); } return(true); } } if (commentTokens != null && commentTokens.Length > 0) { foreach (string comment in commentTokens) { if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal)) //, true, Configuration.Culture)) { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1)); } return(true); } } } if (!_configCheckDone) { Configuration.Validate(null); var dict = Configuration.KVPRecordFieldConfigurations.ToDictionary(i1 => i1.Name, i1 => i1.FieldType == null ? null : i1.FieldType); RaiseMembersDiscovered(dict); Configuration.UpdateFieldTypesIfAny(dict); _configCheckDone = true; } return(false); })) { while (true) { pairIn = e.Peek; if (!isRecordStartFound) { if (pairIn == null) { break; } lastLine = null; recLines.Clear(); isRecordEndFound = false; isRecordStartFound = true; if (!Configuration.RecordStart.IsNullOrWhiteSpace()) { //Move to record start while (!(Configuration.IsRecordStartMatch(e.Peek.Item2))) { e.MoveNext(); if (e.Peek == null) { break; } } } if (e.Peek != null) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Record start found at [{0}] line...".FormatString(e.Peek.Item1)); } e.MoveNext(); continue; } else { string recordEnd = !Configuration.RecordEnd.IsNullOrWhiteSpace() ? Configuration.RecordEnd : Configuration.RecordStart; if (!recordEnd.IsNullOrWhiteSpace()) { if (e.Peek == null) { if (Configuration.RecordEnd.IsNullOrWhiteSpace()) { isRecordEndFound = true; } else { break; } } else { //Move to record start if (Configuration.IsRecordEndMatch(e.Peek.Item2)) { isRecordEndFound = true; isRecordStartFound = false; if (e.Peek != null) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Record end found at [{0}] line...".FormatString(e.Peek.Item1)); } } } } else if (e.Peek == null) { isRecordEndFound = true; } if (!isHeaderFound) { //if (isRecordEndFound && headers.Count == 0) //{ // //throw new ChoParserException("Unexpected EOF found."); //} if (!isRecordEndFound) { e.MoveNext(); if (e.Peek != null) { //If line empty or line continuation, skip if (pairIn.Item2.IsNullOrWhiteSpace() || IsLineContinuationCharFound(pairIn.Item2)) //.Item2[0] == ' ' || pairIn.Item2[0] == '\t') { } else { string header = ToKVP(pairIn.Item1, pairIn.Item2).Key; if (!header.IsNullOrWhiteSpace()) { headers.Add(header); } } } } else { Configuration.Validate(headers.ToArray()); isHeaderFound = true; isRecordEndFound = false; IsHeaderLoaded = true; break; } } else { if (!IsHeaderLoaded) { Configuration.Validate(new string[] { }); IsHeaderLoaded = true; } if (isRecordEndFound && recLines.Count == 0) { //throw new ChoParserException("Unexpected EOF found."); } else if (!isRecordEndFound) { e.MoveNext(); if (e.Peek != null) { //If line empty or line continuation, skip if (pairIn.Item2.IsNullOrWhiteSpace()) { if (!Configuration.IgnoreEmptyLine) { throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pairIn.Item1)); } else { Tuple <long, string> t = new Tuple <long, string>(lastLine.Item1, lastLine.Item2 + Configuration.EOLDelimiter); recLines.RemoveAt(recLines.Count - 1); recLines.Add(t); } } else if (IsLineContinuationCharFound(pairIn.Item2)) //pairIn.Item2[0] == ' ' || pairIn.Item2[0] == '\t') { if (lastLine == null) { throw new ChoParserException("Unexpected line continuation found at {0} location.".FormatString(pairIn.Item1)); } else { Tuple <long, string> t = new Tuple <long, string>(lastLine.Item1, lastLine.Item2 + Configuration.EOLDelimiter + pairIn.Item2); recLines.RemoveAt(recLines.Count - 1); recLines.Add(t); } } else { lastLine = pairIn; recLines.Add(pairIn); } } } else { object rec = Configuration.IsDynamicObject ? new ChoDynamicObject(new Dictionary <string, object>(Configuration.FileHeaderConfiguration.StringComparer)) { ThrowExceptionIfPropNotExists = true, AlternativeKeys = Configuration.AlternativeKeys } : ChoActivator.CreateInstance(RecordType); if (!LoadLines(new Tuple <long, List <Tuple <long, string> > >(++recNo, recLines), ref rec)) { yield break; } isRecordStartFound = false; //StoreState(e.Current, rec != null); if (!Configuration.RecordEnd.IsNullOrWhiteSpace()) { e.MoveNext(); } if (rec == null) { continue; } yield return(rec); if (Configuration.NotifyAfter > 0 && recNo % Configuration.NotifyAfter == 0) { if (RaisedRowsLoaded(recNo)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested."); abortRequested = true; yield break; } } if (e.Peek == null) { break; } } } } if (doWhile != null) { doWhile = RaiseDoWhile(pairIn); if (doWhile != null && doWhile.Value) { break; } } } } } if (!abortRequested) { RaisedRowsLoaded(recNo, true); } RaiseEndLoad(sr); }
private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null) { TraceSwitch = traceSwitch; TextReader sr = source as TextReader; ChoGuard.ArgumentNotNull(sr, "TextReader"); if (sr is StreamReader) { ((StreamReader)sr).Seek(0, SeekOrigin.Begin); } if (!RaiseBeginLoad(sr)) { yield break; } string[] commentTokens = Configuration.Comments; bool? skip = false; bool abortRequested = false; long runningCount = 0; using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >( new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, Configuration.MayContainEOLInData)).ToEnumerable(), (pair) => { //bool isStateAvail = IsStateAvail(); skip = false; //if (isStateAvail) //{ // if (!IsStateMatches(item)) // { // skip = filterFunc != null ? filterFunc(item) : false; // } // else // skip = true; //} //else // skip = filterFunc != null ? filterFunc(item) : false; if (skip == null) { return(null); } if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine); if (!skip.Value) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1)); } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1)); } } if (skip.Value) { return(skip); } //if (!(sr.BaseStream is MemoryStream)) // ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1)); //if (Task != null) // return !IsStateNOTExistsOrNOTMatch(item); if (pair.Item2.IsNullOrWhiteSpace()) { if (!Configuration.IgnoreEmptyLine) { throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1)); } else { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1)); } return(true); } } if (commentTokens != null && commentTokens.Length > 0) { foreach (string comment in commentTokens) { if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal)) //, true, Configuration.Culture)) { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1)); } return(true); } } } if (!_configCheckDone) { Configuration.Validate(pair); // GetHeaders(pair.Item2)); RaiseMembersDiscovered(Configuration.FixedLengthRecordFieldConfigurations.Select(i => new KeyValuePair <string, Type>(i.Name, i.FieldType == null ? typeof(string) : i.FieldType)).ToArray()); _configCheckDone = true; } //LoadHeader if any if (Configuration.FileHeaderConfiguration.HasHeaderRecord && !_headerFound) { if (Configuration.FileHeaderConfiguration.IgnoreHeader) { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring header line at [{0}]...".FormatString(pair.Item1)); } } else { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading header line at [{0}]...".FormatString(pair.Item1)); } LoadHeaderLine(pair); } _headerFound = true; return(true); } return(false); })) { while (true) { Tuple <long, string> pair = e.Peek; if (pair == null) { if (!abortRequested) { RaisedRowsLoaded(runningCount); } RaiseEndLoad(sr); yield break; } runningCount = pair.Item1; object rec = Activator.CreateInstance(RecordType); if (!LoadLine(pair, ref rec)) { yield break; } //StoreState(e.Current, rec != null); e.MoveNext(); if (rec == null) { continue; } yield return(rec); if (Configuration.NotifyAfter > 0 && pair.Item1 % Configuration.NotifyAfter == 0) { if (RaisedRowsLoaded(pair.Item1)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested."); yield break; } } } } }
private void LoadValueConverters(ChoIniFile iniFile1) { ChoIniFile iniFile = iniFile1.GetSection("CONVERTER"); var dict = new Dictionary <string, List <Tuple <IValueConverter, string> > >(); IValueConverter value = null; Dictionary <string, string> iniKeyValuesDict = iniFile.KeyValuesDict; foreach (string fieldName in iniKeyValuesDict.Keys) //GetFieldNames(iniFile1)) { if (fieldName.IsNullOrWhiteSpace()) { continue; } if (!iniKeyValuesDict.ContainsKey(fieldName)) { continue; } if (dict.ContainsKey(fieldName)) { continue; } dict.Add(fieldName, new List <Tuple <IValueConverter, string> >()); if (!iniKeyValuesDict[fieldName].IsNullOrWhiteSpace()) { foreach (string val in iniKeyValuesDict[fieldName].SplitNTrim(";")) { if (val.IsNullOrWhiteSpace()) { continue; } string vcType = null; string vcParam = null; foreach (KeyValuePair <string, string> convParams in val.ToKeyValuePairs('&')) { if (convParams.Key == "Type") { vcType = convParams.Value; } else if (convParams.Key == "Parameter") { vcParam = convParams.Value; } } if (!vcType.IsNullOrWhiteSpace()) { try { value = Activator.CreateInstance(ChoType.GetType(vcType)) as IValueConverter; if (value != null) { dict[fieldName].Add(new Tuple <IValueConverter, string>(value, vcParam)); } } catch (Exception ex) { ChoETLFramework.WriteLog(ChoETLFramework.Switch.TraceError, "Error while loading format value '{0}' for '{1}' member. {2}".FormatString(value, fieldName, ex.Message)); } } } } } _valueConverterCache.AddOrUpdate(iniFile1.Key, dict); }
public override IEnumerable <object> WriteTo(object writer, IEnumerable <object> records, Func <object, bool> predicate = null) { TextWriter sw = writer as TextWriter; ChoGuard.ArgumentNotNull(sw, "TextWriter"); if (records == null) { yield break; } if (!RaiseBeginWrite(sw)) { yield break; } CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture; System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture; string recText = String.Empty; long recCount = 0; var recEnum = records.GetEnumerator(); try { object notNullRecord = GetFirstNotNullRecord(recEnum); if (notNullRecord == null) { yield break; } if (Configuration.IsDynamicObject) { if (Configuration.MaxScanRows > 0) { List <string> fns = new List <string>(); foreach (object record1 in GetRecords(recEnum)) { recCount++; if (record1 != null) { if (recCount <= Configuration.MaxScanRows) { if (!record1.GetType().IsDynamicType()) { throw new ChoParserException("Invalid record found."); } _recBuffer.Value.Add(record1); fns = fns.Union(GetFields(record1)).ToList(); if (recCount == Configuration.MaxScanRows) { Configuration.Validate(fns.ToArray()); WriteHeaderLine(sw); _configCheckDone = true; break; } } } } } } foreach (object record in GetRecords(recEnum)) { _index++; if (TraceSwitch.TraceVerbose) { if (record is IChoETLNameableObject) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name)); } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index)); } } recText = String.Empty; if (record != null) { if (predicate == null || predicate(record)) { //Discover and load CSV columns from first record if (!_configCheckDone) { if (notNullRecord != null) { string[] fieldNames = GetFields(notNullRecord); Configuration.Validate(fieldNames); WriteHeaderLine(sw); _configCheckDone = true; } } //Check record if (record != null) { Type rt = record.GetType().ResolveType(); if (Configuration.IsDynamicObject) { if (ElementType != null) { } else if (!rt.IsDynamicType()) { throw new ChoWriterException("Invalid record found."); } } else { if (rt != Configuration.RecordType) { throw new ChoWriterException("Invalid record found."); } } } if (!RaiseBeforeRecordWrite(record, _index, ref recText)) { yield break; } if (recText == null) { continue; } else if (recText.Length > 0) { sw.Write("{1}{0}", recText, _hadHeaderWritten || HasExcelSeparator ? Configuration.EOLDelimiter : ""); continue; } try { if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel) { record.DoObjectLevelValidation(Configuration, Configuration.CSVRecordFieldConfigurations); } if (ToText(_index, record, out recText)) { if (_index == 1) { sw.Write("{1}{0}", recText, _hadHeaderWritten || HasExcelSeparator ? Configuration.EOLDelimiter : ""); } else { sw.Write("{1}{0}", recText, Configuration.EOLDelimiter); } if (!RaiseAfterRecordWrite(record, _index, recText)) { yield break; } } } //catch (ChoParserException) //{ // throw; //} catch (Exception ex) { ChoETLFramework.HandleException(ref ex); if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); } else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue) { if (!RaiseRecordWriteError(record, _index, recText, ex)) { throw; } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); } } else { throw; } } } } yield return(record); if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0) { if (RaisedRowsWritten(_index)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested."); yield break; } } } } finally { System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo; } RaiseEndWrite(sw); }
public static void Warning(string msg) { ChoETLFramework.WriteLog(ChoETLFramework.Switch.TraceWarning, msg); }
public static void Write(string msg) { ChoETLFramework.WriteLog(msg); }
private bool LoadNode(Tuple <long, JObject> pair, ref object rec) { if (!Configuration.UseJSONSerialization) { rec = Configuration.IsDynamicObject ? new ChoDynamicObject() { ThrowExceptionIfPropNotExists = true } } : Activator.CreateInstance(RecordType); try { if (!RaiseBeforeRecordLoad(rec, ref pair)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping..."); rec = null; return(true); } if (pair.Item2 == null) { rec = null; return(true); } if (!Configuration.UseJSONSerialization) { if (!FillRecord(rec, pair)) { return(false); } if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel) { rec.DoObjectLevelValidation(Configuration, Configuration.JSONRecordFieldConfigurations); } } else { //rec = _se.Value != null ? pair.Item2.ToObject(RecordType, _se.Value) : pair.Item2.ToObject(RecordType); if (Configuration.IsDynamicObject) { rec = JsonConvert.DeserializeObject <ExpandoObject>(pair.Item2.ToString(), new ExpandoObjectConverter()); if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.Off) != ChoObjectValidationMode.Off) { rec.DoObjectLevelValidation(Configuration, Configuration.JSONRecordFieldConfigurations); } } else { rec = JsonConvert.DeserializeObject(pair.Item2.ToString(), RecordType); if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.Off) != ChoObjectValidationMode.Off) { rec.DoObjectLevelValidation(Configuration, Configuration.JSONRecordFieldConfigurations); } } } bool skip = false; if (!RaiseAfterRecordLoad(rec, pair, ref skip)) { return(false); } else if (skip) { rec = null; return(true); } } //catch (ChoParserException) //{ // throw; //} catch (ChoMissingRecordFieldException) { throw; } catch (Exception ex) { ChoETLFramework.HandleException(ex); if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); rec = null; } else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue) { if (!RaiseRecordLoadError(rec, pair, ex)) { throw; } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); rec = null; } } else { throw; } return(true); } return(true); }
private IEnumerable <object> AsEnumerable(IEnumerable <object> objs, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null) { TraceSwitch = traceSwitch; long counter = 0; Tuple <long, object> pair = null; bool? skip = false; bool? skipUntil = true; bool? doWhile = true; bool abortRequested = false; List <object> buffer = new List <object>(); IDictionary <string, Type> recFieldTypes = null; foreach (var obj in objs) { pair = new Tuple <long, object>(++counter, obj); skip = false; if (skipUntil != null) { if (skipUntil.Value) { skipUntil = RaiseSkipUntil(pair); if (skipUntil == null) { } else { skip = skipUntil.Value; } } } if (skip == null) { break; } if (skip.Value) { continue; } if (!_configCheckDone) { Configuration.Validate(pair); _configCheckDone = true; } object rec = null; if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading node [{0}]...".FormatString(pair.Item1)); } if (!LoadNode(pair, ref rec)) { yield break; } if (rec == null) { continue; } yield return(rec); if (Configuration.NotifyAfter > 0 && pair.Item1 % Configuration.NotifyAfter == 0) { if (RaisedRowsLoaded(pair.Item1)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested."); abortRequested = true; yield break; } } if (doWhile != null) { doWhile = RaiseDoWhile(pair); if (doWhile != null && doWhile.Value) { break; } } pair = null; } if (!abortRequested && pair != null) { RaisedRowsLoaded(pair.Item1, true); } }
public override IEnumerable <object> WriteTo(object writer, IEnumerable <object> records, Func <object, bool> predicate = null) { _sw = writer; TextWriter sw = writer as TextWriter; ChoGuard.ArgumentNotNull(sw, "TextWriter"); if (Configuration.JsonSerializerSettings.ContractResolver is ChoPropertyRenameAndIgnoreSerializerContractResolver) { ChoPropertyRenameAndIgnoreSerializerContractResolver cr = Configuration.JsonSerializerSettings.ContractResolver as ChoPropertyRenameAndIgnoreSerializerContractResolver; cr.CallbackRecordFieldWrite = _callbackRecordFieldWrite; cr.Writer = Writer; } if (records == null) { yield break; } if (Configuration.SingleDocument == null) { Configuration.SingleDocument = false; } CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture; System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture; string recText = String.Empty; bool recordIgnored = false; try { foreach (object record in records) { _index++; //if (!isFirstRec) //{ // if (!recordIgnored) // sw.Write(","); // else // recordIgnored = false; //} if (TraceSwitch.TraceVerbose) { if (record is IChoETLNameableObject) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name)); } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index)); } } recText = String.Empty; if (predicate == null || predicate(record)) { //Discover and load Xml columns from first record if (!Configuration.IsInitialized) { if (record == null) { continue; } string[] fieldNames = null; if (Configuration.RecordType == typeof(object)) { Type recordType = ElementType == null?record.GetType() : ElementType; RecordType = Configuration.RecordType = recordType.GetUnderlyingType(); //.ResolveType(); Configuration.IsDynamicObject = recordType.IsDynamicType(); } if (typeof(IDictionary).IsAssignableFrom(Configuration.RecordType) || typeof(IList).IsAssignableFrom(Configuration.RecordType)) { Configuration.UseYamlSerialization = true; } if (!Configuration.IsDynamicObject) { if (Configuration.YamlRecordFieldConfigurations.Count == 0) { Configuration.MapRecordFields(Configuration.RecordType); } } if (Configuration.IsDynamicObject) { var dict = record.ToDynamicObject() as IDictionary <string, Object>; fieldNames = dict.Keys.ToArray(); } else { fieldNames = ChoTypeDescriptor.GetProperties <ChoYamlRecordFieldAttribute>(Configuration.RecordType).Select(pd => pd.Name).ToArray(); if (fieldNames.Length == 0) { fieldNames = ChoType.GetProperties(Configuration.RecordType).Select(p => p.Name).ToArray(); } } Configuration.Validate(fieldNames); Configuration.IsInitialized = true; if (!BeginWrite.Value) { yield break; } } if (!RaiseBeforeRecordWrite(record, _index, ref recText)) { yield break; } if (recText == null) { continue; } try { if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel) { record.DoObjectLevelValidation(Configuration, Configuration.YamlRecordFieldConfigurations); } if (ToText(_index, record, out recText)) { if (recText.EndsWith("...")) { recText = recText.Left(recText.Length - 3); } if (recText.EndsWith($"...{Environment.NewLine}")) { recText = recText.Left(recText.Length - $"...{Environment.NewLine}".Length); } if (!Configuration.SingleDocument.Value) { sw.Write($"---{EOLDelimiter}"); } sw.Write("{0}", recText); if (!Configuration.SingleDocument.Value) { sw.Write($"...{EOLDelimiter}"); } if (!RaiseAfterRecordWrite(record, _index, recText)) { yield break; } } } //catch (ChoParserException) //{ // throw; //} catch (Exception ex) { ChoETLFramework.HandleException(ref ex); if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue) { recordIgnored = true; ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); } else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue) { if (!RaiseRecordWriteError(record, _index, recText, ex)) { throw; } else { recordIgnored = true; //ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); } } else { throw; } } } yield return(record); if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0) { if (RaisedRowsWritten(_index)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested."); yield break; } } } } finally { System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo; } }
private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null) { TraceSwitch = traceSwitch; TextReader sr = source as TextReader; ChoGuard.ArgumentNotNull(sr, "TextReader"); if (sr is StreamReader) { ((StreamReader)sr).Seek(0, SeekOrigin.Begin); } if (!RaiseBeginLoad(sr)) { yield break; } string[] commentTokens = Configuration.Comments; bool? skip = false; bool abortRequested = false; long runningCount = 0; using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >( new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, Configuration.MayContainEOLInData)).ToEnumerable(), (pair) => { //bool isStateAvail = IsStateAvail(); skip = false; //if (isStateAvail) //{ // if (!IsStateMatches(item)) // { // skip = filterFunc != null ? filterFunc(item) : false; // } // else // skip = true; //} //else // skip = filterFunc != null ? filterFunc(item) : false; if (skip == null) { return(null); } if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine); if (!skip.Value) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1)); } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1)); } } if (skip.Value) { return(skip); } //if (!(sr.BaseStream is MemoryStream)) // ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1)); //if (Task != null) // return !IsStateNOTExistsOrNOTMatch(item); if (pair.Item2.IsNullOrWhiteSpace()) { if (!Configuration.IgnoreEmptyLine) { throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1)); } else { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1)); } return(true); } } //LoadExcelSeparator if any if (pair.Item1 == 1 && !_excelSeparatorFound) { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Inspecting for excel separator at [{0}]...".FormatString(pair.Item1)); } bool retVal = LoadExcelSeperatorIfAny(pair); _excelSeparatorFound = true; if (Configuration.HasExcelSeparator != null && Configuration.HasExcelSeparator.Value && !retVal) { throw new ChoParserException("Missing excel separator header line in the file."); } if (retVal) { return(true); } } if (commentTokens != null && commentTokens.Length > 0) { foreach (string comment in commentTokens) { if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal)) //, true, Configuration.Culture)) { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1)); } return(true); } } } if (Configuration.FileHeaderConfiguration.HeaderLineAt > 0) { if (pair.Item1 < Configuration.FileHeaderConfiguration.HeaderLineAt) { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Header line at {1}. Skipping [{0}] line...".FormatString(pair.Item1, Configuration.FileHeaderConfiguration.HeaderLineAt)); } return(true); } } if (!_configCheckDone) { Configuration.Validate(GetHeaders(pair.Item2)); var dict = Configuration.CSVRecordFieldConfigurations.ToDictionary(i => i.Name, i => i.FieldType == null ? null : i.FieldType); RaiseMembersDiscovered(ref dict); Configuration.UpdateFieldTypesIfAny(dict); _configCheckDone = true; } //LoadHeader if any if ((Configuration.FileHeaderConfiguration.HasHeaderRecord || Configuration.FileHeaderConfiguration.HeaderLineAt > 0) && !_headerFound) { if (Configuration.FileHeaderConfiguration.IgnoreHeader) { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring header line at [{0}]...".FormatString(pair.Item1)); } } else { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading header line at [{0}]...".FormatString(pair.Item1)); } LoadHeaderLine(pair); } _headerFound = true; return(true); } return(false); })) { while (true) { Tuple <long, string> pair = e.Peek; if (pair == null) { if (!abortRequested) { RaisedRowsLoaded(runningCount); } RaiseEndLoad(sr); yield break; } runningCount = pair.Item1; object rec = Configuration.IsDynamicObject ? new ChoDynamicObject(new Dictionary <string, object>(Configuration.FileHeaderConfiguration.StringComparer)) { ThrowExceptionIfPropNotExists = true, KeyResolver = (name) => { if (Configuration.RecordFieldConfigurationsDict2.ContainsKey(name)) { return(Configuration.RecordFieldConfigurationsDict2[name].Name); } else { return(name); } } } : Activator.CreateInstance(RecordType); if (!LoadLine(pair, ref rec)) { yield break; } //StoreState(e.Current, rec != null); e.MoveNext(); if (rec == null) { continue; } yield return(rec); if (Configuration.NotifyAfter > 0 && pair.Item1 % Configuration.NotifyAfter == 0) { if (RaisedRowsLoaded(pair.Item1)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested."); abortRequested = true; yield break; } } } } }
public override IEnumerable <object> WriteTo(object writer, IEnumerable <object> records, Func <object, bool> predicate = null) { TextWriter sw = writer as TextWriter; ChoGuard.ArgumentNotNull(sw, "TextWriter"); if (records == null) { yield break; } CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture; System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture; string recText = String.Empty; bool recordIgnored = false; try { foreach (object record in records) { _index++; if (!isFirstRec) { if (!recordIgnored) { sw.Write(","); } else { recordIgnored = false; } } if (TraceSwitch.TraceVerbose) { if (record is IChoETLNameableObject) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name)); } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index)); } } recText = String.Empty; if (predicate == null || predicate(record)) { //Discover and load Xml columns from first record if (!_configCheckDone) { if (record == null) { continue; } string[] fieldNames = null; Type recordType = ElementType == null?record.GetType() : ElementType; if (typeof(ICollection).IsAssignableFrom(recordType)) { recordType = recordType.GetEnumerableItemType().GetUnderlyingType(); } else { recordType = recordType.GetUnderlyingType(); } Configuration.IsDynamicObject = recordType.IsDynamicType(); if (!Configuration.IsDynamicObject) { if (recordType.IsSimple()) { Configuration.RecordType = typeof(ChoScalarObject <>).MakeGenericType(recordType); } else { Configuration.RecordType = recordType; } } if (Configuration.IsDynamicObject) { var dict = record.ToDynamicObject() as IDictionary <string, Object>; fieldNames = dict.Keys.ToArray(); } else { fieldNames = ChoTypeDescriptor.GetProperties <ChoJSONRecordFieldAttribute>(Configuration.RecordType).Select(pd => pd.Name).ToArray(); if (fieldNames.Length == 0) { fieldNames = ChoType.GetProperties(Configuration.RecordType).Select(p => p.Name).ToArray(); } } Configuration.Validate(fieldNames); _configCheckDone = true; if (!RaiseBeginWrite(sw)) { yield break; } if (!SupportMultipleContent) { sw.Write("["); } } if (!RaiseBeforeRecordWrite(record, _index, ref recText)) { yield break; } if (recText == null) { continue; } try { if (!Configuration.UseJSONSerialization) { if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel) { record.DoObjectLevelValidation(Configuration, Configuration.JSONRecordFieldConfigurations); } if (ToText(_index, record, out recText)) { if (!recText.IsNullOrEmpty()) { if (!SupportMultipleContent) { sw.Write("{1}{0}", Configuration.Formatting == Formatting.Indented ? recText.Indent(1, " ") : recText, Configuration.EOLDelimiter); } else { if (_index == 1) { sw.Write("{0}", recText); } else { sw.Write("{1}{0}", recText, Configuration.EOLDelimiter); } } if (!RaiseAfterRecordWrite(record, _index, recText)) { yield break; } } } } else { if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.Off) != ChoObjectValidationMode.Off) { record.DoObjectLevelValidation(Configuration, Configuration.JSONRecordFieldConfigurations); } recText = JsonConvert.SerializeObject(record, Configuration.Formatting); if (!SupportMultipleContent) { sw.Write("{1}{0}", Configuration.Formatting == Formatting.Indented ? recText.Indent(1, " ") : recText, Configuration.EOLDelimiter); } else { if (_index == 1) { sw.Write("{0}", recText); } else { sw.Write("{1}{0}", recText, Configuration.EOLDelimiter); } } if (!RaiseAfterRecordWrite(record, _index, recText)) { yield break; } } } //catch (ChoParserException) //{ // throw; //} catch (Exception ex) { ChoETLFramework.HandleException(ref ex); if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue) { recordIgnored = true; ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); } else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue) { if (!RaiseRecordWriteError(record, _index, recText, ex)) { throw; } else { recordIgnored = true; ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); } } else { throw; } } } yield return(record); if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0) { if (RaisedRowsWritten(_index)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested."); yield break; } } isFirstRec = false; } } finally { System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo; } }
private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null) { TraceSwitch = traceSwitch; StreamReader sr = source as StreamReader; ChoGuard.ArgumentNotNull(sr, "StreamReader"); sr.Seek(0, SeekOrigin.Begin); if (!RaiseBeginLoad(sr)) { yield break; } string[] commentTokens = Configuration.Comments; using (ChoPeekEnumerator <Tuple <int, string> > e = new ChoPeekEnumerator <Tuple <int, string> >( new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar)).ToEnumerable(), (pair) => { //bool isStateAvail = IsStateAvail(); bool?skip = false; //if (isStateAvail) //{ // if (!IsStateMatches(item)) // { // skip = filterFunc != null ? filterFunc(item) : false; // } // else // skip = true; //} //else // skip = filterFunc != null ? filterFunc(item) : false; if (skip == null) { return(null); } //if (!(sr.BaseStream is MemoryStream)) //{ ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine); if (!skip.Value) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1)); } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1)); } //} if (skip.Value) { return(skip); } //if (!(sr.BaseStream is MemoryStream)) // ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1)); //if (Task != null) // return !IsStateNOTExistsOrNOTMatch(item); if (pair.Item2.IsNullOrWhiteSpace()) { if (!Configuration.IgnoreEmptyLine) { throw new ChoParserException("Empty line found at {0} location.".FormatString(pair.Item1)); } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1)); return(true); } } if (commentTokens == null) { return(false); } else { var x = (from comment in commentTokens where !pair.Item2.IsNull() && pair.Item2.StartsWith(comment, true, Configuration.Culture) select comment).FirstOrDefault(); if (x != null) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1)); return(true); } } if (!_configCheckDone) { Configuration.Validate(pair); // GetHeaders(pair.Item2)); _configCheckDone = true; } //LoadHeader if any if (Configuration.FileHeaderConfiguration.HasHeaderRecord && !_headerFound) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading header line at [{0}]...".FormatString(pair.Item1)); LoadHeaderLine(pair); _headerFound = true; return(true); } return(false); })) { while (true) { Tuple <int, string> pair = e.Peek; if (pair == null) { RaiseEndLoad(sr); yield break; } object rec = ChoActivator.CreateInstance(RecordType); if (!LoadLine(pair, ref rec)) { yield break; } //StoreState(e.Current, rec != null); e.MoveNext(); if (rec == null) { continue; } yield return(rec); } } }
private bool FillRecord(ref object rec, Tuple <long, IDictionary <string, object> > pair) { long lineNo; lineNo = pair.Item1; var node = pair.Item2; fieldValue = null; fieldConfig = null; pi = null; object rootRec = rec; foreach (KeyValuePair <string, ChoParquetRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict) { if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject) { if (Configuration.IgnoredFields.Contains(kvp.Key)) { continue; } } fieldValue = null; fieldConfig = kvp.Value; if (Configuration.PIDict != null) { // if FieldName is set if (!string.IsNullOrEmpty(fieldConfig.FieldName)) { // match using FieldName Configuration.PIDict.TryGetValue(fieldConfig.FieldName, out pi); } if (pi == null) { // otherwise match usign the property name Configuration.PIDict.TryGetValue(kvp.Key, out pi); } } rec = GetDeclaringRecord(kvp.Value.DeclaringMember, rootRec); if (!node.ContainsKey(kvp.Value.FieldName)) { if (Configuration.ColumnCountStrict) { throw new ChoParserException("No matching '{0}' field found.".FormatString(fieldConfig.FieldName)); } } fieldValue = node[kvp.Value.FieldName]; if (!RaiseBeforeRecordFieldLoad(rec, pair.Item1, kvp.Key, ref fieldValue)) { continue; } try { if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject) { } else { if (pi != null) { kvp.Value.FieldType = pi.PropertyType; } else { kvp.Value.FieldType = typeof(string); } } object v1 = node; if (fieldConfig.CustomSerializer != null) { fieldValue = fieldConfig.CustomSerializer(v1); } else if (RaiseRecordFieldDeserialize(rec, pair.Item1, kvp.Key, ref v1)) { fieldValue = v1; } else if (fieldConfig.PropCustomSerializer != null) { fieldValue = ChoCustomSerializer.Deserialize(v1, fieldConfig.FieldType, fieldConfig.PropCustomSerializer, fieldConfig.PropCustomSerializerParams, Configuration.Culture, fieldConfig.Name); } else { if (!node.ContainsKey(kvp.Value.FieldName)) { if (Configuration.ColumnCountStrict) { throw new ChoParserException("No matching '{0}' field found.".FormatString(fieldConfig.FieldName)); } } else { fieldValue = node[kvp.Value.FieldName]; } } bool ignoreFieldValue = fieldValue.IgnoreFieldValue(fieldConfig.IgnoreFieldValueMode); if (ignoreFieldValue && fieldConfig.IsDefaultValueSpecified) { fieldValue = fieldConfig.DefaultValue; } ignoreFieldValue = fieldValue.IgnoreFieldValue(fieldConfig.IgnoreFieldValueMode); if (ignoreFieldValue) { continue; } if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject) { var dict = rec as IDictionary <string, Object>; dict.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture); if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel) { dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode); } } else { if (Configuration.SupportsMultiRecordTypes) { ChoType.TryGetProperty(rec.GetType(), kvp.Key, out pi); //*** TODO //fieldConfig.PI = pi; //fieldConfig.PropConverters = ChoTypeDescriptor.GetTypeConverters(fieldConfig.PI); //fieldConfig.PropConverterParams = ChoTypeDescriptor.GetTypeConverterParams(fieldConfig.PI); } if (pi != null) { rec.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture); } else if (RecordType.IsSimple()) { rec = ChoConvert.ConvertTo(fieldValue, RecordType, Configuration.Culture); } else { throw new ChoMissingRecordFieldException("Missing '{0}' property in {1} type.".FormatString(kvp.Key, ChoType.GetTypeName(rec))); } if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel) { rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode); } } if (!RaiseAfterRecordFieldLoad(rec, pair.Item1, kvp.Key, fieldValue)) { return(false); } } catch (ChoParserException) { Reader.IsValid = false; throw; } catch (ChoMissingRecordFieldException) { Reader.IsValid = false; if (Configuration.ThrowAndStopOnMissingField) { throw; } } catch (Exception ex) { Reader.IsValid = false; ChoETLFramework.HandleException(ref ex); if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop) { throw; } try { if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject) { var dict = rec as IDictionary <string, Object>; if (dict.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue)) { dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode); } else if (dict.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture)) { dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode); } else if (ex is ValidationException) { throw; } else { throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex); } } else if (pi != null) { if (rec.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture)) { rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode); } else if (rec.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture)) { rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode); } else if (ex is ValidationException) { throw; } else { throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex); } } else { throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex); } } catch (Exception innerEx) { if (ex == innerEx.InnerException || ex is ValidationException) { if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue) { ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring field...".FormatString(ex.Message)); continue; } else { if (!RaiseRecordFieldLoadError(rec, pair.Item1, kvp.Key, ref fieldValue, ex)) { if (ex is ValidationException) { throw; } throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex); } else { try { if (Configuration.IsDynamicObject) { var dict = rec as IDictionary <string, Object>; dict.ConvertNSetMemberValue(kvp.Key, fieldConfig, ref fieldValue, Configuration.Culture); } else { if (pi != null) { rec.ConvertNSetMemberValue(kvp.Key, fieldConfig, ref fieldValue, Configuration.Culture); } else { throw new ChoMissingRecordFieldException("Missing '{0}' property in {1} type.".FormatString(kvp.Key, ChoType.GetTypeName(rec))); } } } catch { } } } } else { throw new ChoReaderException("Failed to assign '{0}' fallback value to '{1}' field.".FormatString(fieldValue, fieldConfig.FieldName), innerEx); } } } } rec = rootRec; return(true); }
public static void Verbose(string msg) { ChoETLFramework.WriteLog(ChoETLFramework.Switch.TraceVerbose, msg); }
private bool LoadNode(Tuple <long, object> pair, ref object rec) { bool ignoreFieldValue = pair.Item2.IgnoreFieldValue(Configuration.IgnoreFieldValueMode); if (ignoreFieldValue) { return(false); } else if (pair.Item2 == null && !Configuration.IsDynamicObject) { rec = RecordType.CreateInstanceAndDefaultToMembers(Configuration.RecordFieldConfigurationsDict.ToDictionary(kvp => kvp.Key, kvp => kvp.Value as ChoRecordFieldConfiguration)); return(true); } try { if (!RaiseBeforeRecordLoad(rec, ref pair)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping..."); rec = null; return(true); } //if (Configuration.CustomNodeSelecter != null) //{ // pair = new Tuple<long, object>(pair.Item1, Configuration.CustomNodeSelecter(pair.Item2)); //} if (pair.Item2 == null) { rec = null; return(true); } rec = pair.Item2; if (rec is AvroRecord) { rec = ToDynamicObject(rec as AvroRecord); } if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel) { rec.DoObjectLevelValidation(Configuration, Configuration.AvroRecordFieldConfigurations); } bool skip = false; if (!RaiseAfterRecordLoad(rec, pair, ref skip)) { return(false); } else if (skip) { rec = null; return(true); } } catch (Exception ex) { Reader.IsValid = false; if (ex is ChoMissingRecordFieldException && Configuration.ThrowAndStopOnMissingField) { if (!RaiseRecordLoadError(rec, pair, ex)) { throw; } else { //ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); //rec = null; } } else { ChoETLFramework.HandleException(ref ex); if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue) { ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); rec = null; } else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue) { if (!RaiseRecordLoadError(rec, pair, ex)) { throw; } else { //ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); //rec = null; } } else { throw; } } return(true); } return(true); }
public static void Info(string msg) { ChoETLFramework.WriteLog(ChoETLFramework.Switch.TraceInfo, msg); }
private IEnumerable <object> AsEnumerable(IEnumerable <DataColumn[]> dataColumns, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null) { TraceSwitch = traceSwitch; long counter = 0; Tuple <long, IDictionary <string, object> > pair = null; bool? skip = false; bool? skipUntil = true; bool? doWhile = true; bool abortRequested = false; List <object> buffer = new List <object>(); IDictionary <string, Type> recFieldTypes = null; foreach (var dc in dataColumns) { foreach (var obj in Unpack(dc)) { pair = new Tuple <long, IDictionary <string, object> >(++counter, obj); skip = false; if (skipUntil != null) { if (skipUntil.Value) { skipUntil = RaiseSkipUntil(pair); if (skipUntil == null) { } else { skip = skipUntil.Value; } } } if (skip == null) { break; } if (skip.Value) { continue; } if (!_configCheckDone) { if (Configuration.SupportsMultiRecordTypes && Configuration.RecordSelector != null && !Configuration.RecordTypeMapped) { } else { Configuration.Validate(pair); } var dict = Configuration.ParquetRecordFieldConfigurations.ToDictionary(i => i.Name, i => i.FieldType == null ? null : i.FieldType); if (Configuration.MaxScanRows == 0) { RaiseMembersDiscovered(dict); } Configuration.UpdateFieldTypesIfAny(dict); _configCheckDone = true; } object rec = null; if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading node [{0}]...".FormatString(pair.Item1)); } if (!LoadNode(pair, ref rec)) { yield break; } if (rec == null) { continue; } if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject) { if (Configuration.AreAllFieldTypesNull && Configuration.AutoDiscoverFieldTypes && Configuration.MaxScanRows > 0 && counter <= Configuration.MaxScanRows) { buffer.Add(rec); if (recFieldTypes == null) { recFieldTypes = Configuration.ParquetRecordFieldConfigurations.ToDictionary(i => i.FieldName, i => i.FieldType == null ? null : i.FieldType); } RaiseRecordFieldTypeAssessment(recFieldTypes, (IDictionary <string, object>)rec, counter == Configuration.MaxScanRows); if (counter == Configuration.MaxScanRows) { Configuration.UpdateFieldTypesIfAny(recFieldTypes); var dict = recFieldTypes = Configuration.ParquetRecordFieldConfigurations.ToDictionary(i => i.FieldName, i => i.FieldType == null ? null : i.FieldType); RaiseMembersDiscovered(dict); foreach (object rec1 in buffer) { yield return(new ChoDynamicObject(MigrateToNewSchema(rec1 as IDictionary <string, object>, recFieldTypes))); } buffer.Clear(); } } else { yield return(rec); } } else { yield return(rec); } if (Configuration.NotifyAfter > 0 && pair.Item1 % Configuration.NotifyAfter == 0) { if (RaisedRowsLoaded(pair.Item1)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested."); abortRequested = true; yield break; } } if (doWhile != null) { doWhile = RaiseDoWhile(pair); if (doWhile != null && doWhile.Value) { break; } } } } if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject) { if (buffer.Count > 0) { Configuration.UpdateFieldTypesIfAny(recFieldTypes); var dict = recFieldTypes = Configuration.ParquetRecordFieldConfigurations.ToDictionary(i => i.FieldName, i => i.FieldType == null ? null : i.FieldType); RaiseMembersDiscovered(dict); foreach (object rec1 in buffer) { yield return(new ChoDynamicObject(MigrateToNewSchema(rec1 as IDictionary <string, object>, recFieldTypes))); } } } if (!abortRequested && pair != null) { RaisedRowsLoaded(pair.Item1); } }
public static void Error(string msg) { ChoETLFramework.WriteLog(ChoETLFramework.Switch.TraceError, msg); }
private bool LoadNode(Tuple <long, IDictionary <string, object> > pair, ref object rec) { bool ignoreFieldValue = pair.Item2.IgnoreFieldValue(Configuration.IgnoreFieldValueMode); if (ignoreFieldValue) { return(false); } else if (pair.Item2 == null && !Configuration.IsDynamicObject) { rec = RecordType.CreateInstanceAndDefaultToMembers(Configuration.RecordFieldConfigurationsDict.ToDictionary(kvp => kvp.Key, kvp => kvp.Value as ChoRecordFieldConfiguration)); return(true); } if (Configuration.SupportsMultiRecordTypes && Configuration.RecordSelector != null) { Type recType = Configuration.RecordSelector(pair); if (recType == null) { if (Configuration.IgnoreIfNoRecordTypeFound) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, $"No record type found for [{pair.Item1}] line to parse."); return(true); } else { throw new ChoParserException($"No record type found for [{pair.Item1}] line to parse."); } } if (!Configuration.RecordTypeMapped) { Configuration.MapRecordFields(recType); Configuration.Validate(null); } rec = recType.IsDynamicType() ? new ChoDynamicObject() { ThrowExceptionIfPropNotExists = true } : ChoActivator.CreateInstance(recType); } else if (Configuration.IsDynamicObject) { rec = Configuration.IsDynamicObject ? new ChoDynamicObject() { ThrowExceptionIfPropNotExists = true } } : ChoActivator.CreateInstance(RecordType); try { if (!RaiseBeforeRecordLoad(rec, ref pair)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping..."); rec = null; return(true); } //if (Configuration.CustomNodeSelecter != null) //{ // pair = new Tuple<long, IDictionary<string, object>>(pair.Item1, Configuration.CustomNodeSelecter(pair.Item2)); //} if (pair.Item2 == null) { rec = null; return(true); } if (!FillRecord(ref rec, pair)) { return(false); } if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel) { rec.DoObjectLevelValidation(Configuration, Configuration.ParquetRecordFieldConfigurations); } bool skip = false; if (!RaiseAfterRecordLoad(rec, pair, ref skip)) { return(false); } else if (skip) { rec = null; return(true); } } catch (Exception ex) { Reader.IsValid = false; if (ex is ChoMissingRecordFieldException && Configuration.ThrowAndStopOnMissingField) { if (!RaiseRecordLoadError(rec, pair, ex)) { throw; } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); rec = null; } } else { ChoETLFramework.HandleException(ref ex); if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); rec = null; } else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue) { if (!RaiseRecordLoadError(rec, pair, ex)) { throw; } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); rec = null; } } else { throw; } } return(true); } return(true); } object fieldValue = null; ChoParquetRecordFieldConfiguration fieldConfig = null; PropertyInfo pi = null;
public static void Write(bool condition, string msg) { ChoETLFramework.WriteLog(condition, msg); }
private bool LoadLine(Tuple <long, string> pair, ref object rec) { Type recType = rec.GetType(); try { if (!RaiseBeforeRecordLoad(rec, ref pair)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping..."); rec = null; return(true); } if (pair.Item2 == null) { rec = null; return(true); } else if (pair.Item2 == String.Empty) { return(true); } if (!pair.Item2.IsNullOrWhiteSpace()) { var config = GetConfiguration(rec.GetType()); if (config == null) { throw new ChoParserException("No parser found to parse record line."); } if (config.GetType() == typeof(ChoCSVRecordConfiguration)) { var r = ChoCSVReader.LoadText(rec.GetType(), pair.Item2, config as ChoCSVRecordConfiguration, Configuration.Encoding, Configuration.BufferSize, TraceSwitch, Reader); rec = r.FirstOrDefault <object>(); } else if (config.GetType() == typeof(ChoFixedLengthRecordConfiguration)) { var r = ChoFixedLengthReader.LoadText(rec.GetType(), pair.Item2, config as ChoFixedLengthRecordConfiguration, Configuration.Encoding, Configuration.BufferSize, TraceSwitch, Reader); rec = r.FirstOrDefault <object>(); } else { throw new ChoParserException("Unsupported record line found to parse."); } } bool skip = false; if (!RaiseAfterRecordLoad(rec, pair, ref skip)) { return(false); } else if (skip) { rec = null; return(true); } } catch (ChoParserException pEx) { throw new ChoParserException($"Failed to parse line to '{recType}' object.", pEx); } catch (ChoMissingRecordFieldException mEx) { throw new ChoParserException($"Failed to parse line to '{recType}' object.", mEx); } catch (Exception ex) { Reader.IsValid = false; ChoETLFramework.HandleException(ref ex); if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue) { ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); rec = null; } else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue) { if (!RaiseRecordLoadError(rec, pair, ex)) { throw; } else { //ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); //rec = null; } } else { throw; } //if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue) //{ // rec = null; //} //else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue) //{ // if (!RaiseRecordLoadError(rec, pair, ex)) // throw new ChoReaderException($"Failed to parse line to '{recType}' object.", ex); //} //else // throw new ChoReaderException($"Failed to parse line to '{recType}' object.", ex); return(true); } return(true); }
public override IEnumerable <object> WriteTo(object writer, IEnumerable <object> records, Func <object, bool> predicate = null) { TextWriter sw = writer as TextWriter; ChoGuard.ArgumentNotNull(sw, "TextWriter"); if (records == null) { yield break; } if (!RaiseBeginWrite(sw)) { yield break; } CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture; System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture; string recText = String.Empty; try { foreach (object record in records) { _index++; if (TraceSwitch.TraceVerbose) { if (record is IChoETLNameableObject) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name)); } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index)); } } recText = String.Empty; if (record != null) { if (predicate == null || predicate(record)) { //Discover and load FixedLength columns from first record if (!_configCheckDone) { string[] fieldNames = null; if (Configuration.IsDynamicObject) { var dict = record.ToDynamicObject() as IDictionary <string, Object>; fieldNames = dict.Keys.ToArray(); } else { fieldNames = ChoTypeDescriptor.GetProperties <ChoFixedLengthRecordFieldAttribute>(record.GetType()).Select(pd => pd.Name).ToArray(); if (fieldNames.Length == 0) { fieldNames = ChoType.GetProperties(record.GetType()).Select(p => p.Name).ToArray(); } } Configuration.Validate(fieldNames); WriteHeaderLine(sw); _configCheckDone = true; } if (!RaiseBeforeRecordWrite(record, _index, ref recText)) { yield break; } if (recText == null) { continue; } else if (recText.Length > 0) { sw.Write("{1}{0}", recText, Configuration.FileHeaderConfiguration.HasHeaderRecord ? Configuration.EOLDelimiter : ""); continue; } try { if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel) { record.DoObjectLevelValidation(Configuration, Configuration.FixedLengthRecordFieldConfigurations); } if (ToText(_index, record, out recText)) { if (_index == 1) { sw.Write("{1}{0}", recText, Configuration.FileHeaderConfiguration.HasHeaderRecord ? Configuration.EOLDelimiter : ""); } else { sw.Write("{1}{0}", recText, Configuration.EOLDelimiter); } if (!RaiseAfterRecordWrite(record, _index, recText)) { yield break; } } } catch (ChoParserException) { throw; } catch (Exception ex) { ChoETLFramework.HandleException(ex); if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue) { } else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue) { if (!RaiseRecordWriteError(record, _index, recText, ex)) { throw; } } else { throw; } } } } yield return(record); if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0) { if (RaisedRowsWritten(_index)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested."); yield break; } } } } finally { System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo; } RaiseEndWrite(sw); }
private bool LoadNode(Tuple <long, XElement> pair, ref object rec) { if (!Configuration.UseXmlSerialization || Configuration.IsDynamicObject) { rec = Configuration.IsDynamicObject ? new ExpandoObject() : Activator.CreateInstance(RecordType); } try { if (!RaiseBeforeRecordLoad(rec, ref pair)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping..."); rec = null; return(true); } if (pair.Item2 == null) { rec = null; return(true); } if (!Configuration.UseXmlSerialization) { if (!FillRecord(rec, pair)) { return(false); } if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel) { rec.DoObjectLevelValidation(Configuration, Configuration.XmlRecordFieldConfigurations); } } else { if (Configuration.IsDynamicObject) { Parse(rec, pair.Item2); } else { rec = _se.Value.Deserialize(pair.Item2.CreateReader()); } if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.Off) != ChoObjectValidationMode.Off) { rec.DoObjectLevelValidation(Configuration, Configuration.XmlRecordFieldConfigurations); } } if (!RaiseAfterRecordLoad(rec, pair)) { return(false); } } catch (ChoParserException) { throw; } catch (ChoMissingRecordFieldException) { throw; } catch (Exception ex) { ChoETLFramework.HandleException(ex); if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue) { rec = null; } else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue) { if (!RaiseRecordLoadError(rec, pair, ex)) { throw; } } else { throw; } return(true); } return(true); }
private bool LoadLines(Tuple <long, List <Tuple <long, string> > > pairs, ref object rec) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading [{0}] record...".FormatString(pairs.Item1)); Tuple <long, string> pair = null; long recNo = pairs.Item1; if (!Configuration.AutoDiscoveredColumns) { if (Configuration.ColumnCountStrict) { if (pairs.Item2.Count != Configuration.KVPRecordFieldConfigurations.Count) { throw new ChoParserException("Incorrect number of field values found at record [{2}]. Expected [{0}] field values. Found [{1}] field values.".FormatString(Configuration.KVPRecordFieldConfigurations.Count, pairs.Item2.Count, recNo)); } List <string> keys = (from p in pairs.Item2 select ToKVP(p.Item1, p.Item2).Key).ToList(); if (!Enumerable.SequenceEqual(keys.OrderBy(t => t), Configuration.FCArray.Select(a => a.Value.FieldName).OrderBy(t => t), Configuration.FileHeaderConfiguration.StringComparer)) { throw new ChoParserException("Column count mismatch detected at [{0}] record.".FormatString(recNo)); } } if (Configuration.ColumnOrderStrict) { List <string> keys = (from p in pairs.Item2 select ToKVP(p.Item1, p.Item2).Key).ToList(); int runnngIndex = -1; foreach (var k in keys) { runnngIndex++; if (runnngIndex < Configuration.FCArray.Length) { if (Configuration.FileHeaderConfiguration.IsEqual(Configuration.FCArray[runnngIndex].Value.FieldName, k)) { continue; } } throw new ChoParserException("Found incorrect order on '{1}' column at [{0}] record.".FormatString(recNo, k)); } } } object fieldValue = String.Empty; PropertyInfo pi = null; object rootRec = rec; //Set default values foreach (KeyValuePair <string, ChoKVPRecordFieldConfiguration> kvp in Configuration.FCArray) { rec = GetDeclaringRecord(kvp.Value.DeclaringMember, rootRec); if (Configuration.PIDict != null) { Configuration.PIDict.TryGetValue(kvp.Key, out pi); } try { if (kvp.Value.IsDefaultValueSpecified) { fieldValue = kvp.Value.DefaultValue; } if (Configuration.IsDynamicObject) { var dict = rec as IDictionary <string, Object>; dict.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture); } else { if (pi != null) { rec.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture); } } } catch { } } rec = rootRec; _propInit.Clear(); foreach (var pair1 in pairs.Item2) { pair = pair1; try { if (!RaiseBeforeRecordLoad(rec, ref pair)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping..."); rec = null; return(true); } if (pair.Item2 == null) { rec = null; return(true); } else if (pair.Item2 == String.Empty) { return(true); } if (!pair.Item2.IsNullOrWhiteSpace()) { if (!FillRecord(rec, pair)) { return(false); } if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel) { rec.DoObjectLevelValidation(Configuration, Configuration.KVPRecordFieldConfigurations); } } bool skip = false; if (!RaiseAfterRecordLoad(rec, pair, ref skip)) { return(false); } else if (skip) { rec = null; return(true); } } //catch (ChoParserException) //{ // throw; //} //catch (ChoMissingRecordFieldException) //{ // throw; //} catch (Exception ex) { Reader.IsValid = false; if (ex is ChoMissingRecordFieldException && Configuration.ThrowAndStopOnMissingField) { if (!RaiseRecordLoadError(rec, pair, ex)) { throw; } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); rec = null; } } else { ChoETLFramework.HandleException(ref ex); if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); rec = null; } else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue) { if (!RaiseRecordLoadError(rec, pair, ex)) { throw; } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); rec = null; } } else { throw; } } return(true); } } return(true); }
private bool LoadLine(Tuple <long, string> pair, ref object rec) { try { if (!RaiseBeforeRecordLoad(rec, ref pair)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping..."); rec = null; return(true); } if (pair.Item2 == null) { rec = null; return(true); } else if (pair.Item2 == String.Empty) { return(true); } if (!pair.Item2.IsNullOrWhiteSpace()) { if (!FillRecord(rec, pair)) { return(false); } if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel) { rec.DoObjectLevelValidation(Configuration, Configuration.FixedLengthRecordFieldConfigurations); } } bool skip = false; if (!RaiseAfterRecordLoad(rec, pair, ref skip)) { return(false); } else if (skip) { rec = null; return(true); } } //catch (ChoParserException) //{ // throw; //} //catch (ChoMissingRecordFieldException) //{ // throw; //} catch (Exception ex) { Reader.IsValid = false; if (ex is ChoMissingRecordFieldException && Configuration.ThrowAndStopOnMissingField) { throw; } ChoETLFramework.HandleException(ref ex); if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); rec = null; } else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue) { if (!RaiseRecordLoadError(rec, pair, ex)) { throw; } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); rec = null; } } else { throw; } return(true); } return(true); }
private bool LoadLine(Tuple <long, string> pair, ref object rec) { try { if (!RaiseBeforeRecordLoad(rec, ref pair)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping..."); rec = null; return(true); } if (pair.Item2 == null) { rec = null; return(true); } else if (pair.Item2 == String.Empty) { return(true); } if (!pair.Item2.IsNullOrWhiteSpace()) { if (!FillRecord(rec, pair)) { return(false); } if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel) { rec.DoObjectLevelValidation(Configuration, Configuration.FixedLengthRecordFieldConfigurations); } } if (!RaiseAfterRecordLoad(rec, pair)) { return(false); } } catch (ChoParserException) { throw; } catch (ChoMissingRecordFieldException) { throw; } catch (Exception ex) { ChoETLFramework.HandleException(ex); if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue) { rec = null; } else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue) { if (!RaiseRecordLoadError(rec, pair, ex)) { throw; } } else { throw; } return(true); } return(true); }
private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null) { TraceSwitch = traceSwitch; TextReader sr = source as TextReader; ChoGuard.ArgumentNotNull(sr, "TextReader"); if (sr is StreamReader) { ((StreamReader)sr).Seek(0, SeekOrigin.Begin); } if (!RaiseBeginLoad(sr)) { yield break; } string[] commentTokens = Configuration.Comments; bool? skip = false; bool abortRequested = false; long runningCount = 0; long recCount = 0; bool headerLineLoaded = false; List <object> buffer = new List <object>(); IDictionary <string, Type> recFieldTypes = null; bool?skipUntil = true; bool?doWhile = true; using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >( new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, false /*Configuration.MayContainEOLInData*/)).ToEnumerable(), (pair) => { //bool isStateAvail = IsStateAvail(); skip = false; if (skipUntil != null) { if (skipUntil.Value) { skipUntil = RaiseSkipUntil(pair); if (skipUntil == null) { } else { if (skipUntil.Value) { skip = skipUntil; } else { skip = true; } } } } //if (isStateAvail) //{ // if (!IsStateMatches(item)) // { // skip = filterFunc != null ? filterFunc(item) : false; // } // else // skip = true; //} //else // skip = filterFunc != null ? filterFunc(item) : false; if (skip == null) { return(null); } if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine); if (!skip.Value) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1)); } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1)); } } if (skip.Value) { return(skip); } //if (!(sr.BaseStream is MemoryStream)) // ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1)); //if (Task != null) // return !IsStateNOTExistsOrNOTMatch(item); if (pair.Item2.IsNullOrWhiteSpace()) { if (!Configuration.IgnoreEmptyLine) { throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1)); } else { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1)); } return(true); } } if (commentTokens != null && commentTokens.Length > 0) { foreach (string comment in commentTokens) { if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal)) //, true, Configuration.Culture)) { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1)); } return(true); } } } if (Configuration.FileHeaderConfiguration.HeaderLineAt > 0) { if (pair.Item1 < Configuration.FileHeaderConfiguration.HeaderLineAt) { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Header line at {1}. Skipping [{0}] line...".FormatString(pair.Item1, Configuration.FileHeaderConfiguration.HeaderLineAt)); } return(true); } } if (!_configCheckDone) { if (Configuration.SupportsMultiRecordTypes && Configuration.RecordSelector != null && !Configuration.RecordTypeMapped) { } else { Configuration.Validate(pair); // GetHeaders(pair.Item2)); } var dict = recFieldTypes = Configuration.FixedLengthRecordFieldConfigurations.ToDictionary(i => i.Name, i => i.FieldType == null ? null : i.FieldType); RaiseMembersDiscovered(dict); Configuration.UpdateFieldTypesIfAny(dict); _configCheckDone = true; } //LoadHeader if any if ((Configuration.FileHeaderConfiguration.HasHeaderRecord || Configuration.FileHeaderConfiguration.HeaderLineAt > 0) && !_headerFound) { if (Configuration.FileHeaderConfiguration.IgnoreHeader) { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring header line at [{0}]...".FormatString(pair.Item1)); } } else { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading header line at [{0}]...".FormatString(pair.Item1)); } headerLineLoaded = true; LoadHeaderLine(pair); } _headerFound = true; return(true); } return(false); })) { while (true) { recCount++; Tuple <long, string> pair = e.Peek; if (pair == null) { if (!abortRequested) { RaisedRowsLoaded(runningCount); } RaiseEndLoad(sr); yield break; } runningCount = pair.Item1; object rec = null; if (Configuration.SupportsMultiRecordTypes && Configuration.RecordSelector != null) { Type recType = Configuration.RecordSelector(pair); if (recType == null) { if (Configuration.IgnoreIfNoRecordTypeFound) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, $"No record type found for [{pair.Item1}] line to parse."); e.MoveNext(); continue; } else { throw new ChoParserException($"No record type found for [{pair.Item1}] line to parse."); } } if (!Configuration.RecordTypeMapped) { Configuration.MapRecordFields(recType); Configuration.Validate(null); } rec = recType.IsDynamicType() ? new ChoDynamicObject(new Dictionary <string, object>(Configuration.FileHeaderConfiguration.StringComparer)) { ThrowExceptionIfPropNotExists = true, AlternativeKeys = Configuration.AlternativeKeys } : ChoActivator.CreateInstance(recType); } else { rec = Configuration.IsDynamicObject ? new ChoDynamicObject(new Dictionary <string, object>(Configuration.FileHeaderConfiguration.StringComparer)) { ThrowExceptionIfPropNotExists = true, AlternativeKeys = Configuration.AlternativeKeys } : ChoActivator.CreateInstance(RecordType); } if (!LoadLine(pair, ref rec)) { yield break; } //StoreState(e.Current, rec != null); e.MoveNext(); if (rec == null) { continue; } if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject) { if (Configuration.AreAllFieldTypesNull && Configuration.AutoDiscoverFieldTypes && Configuration.MaxScanRows > 0 && recCount <= Configuration.MaxScanRows) { buffer.Add(rec); RaiseRecordFieldTypeAssessment(recFieldTypes, (IDictionary <string, object>)rec, recCount == Configuration.MaxScanRows); if (recCount == Configuration.MaxScanRows || e.Peek == null) { Configuration.UpdateFieldTypesIfAny(recFieldTypes); var dict = recFieldTypes = Configuration.FixedLengthRecordFieldConfigurations.ToDictionary(i => i.Name, i => i.FieldType == null ? null : i.FieldType); RaiseMembersDiscovered(dict); foreach (object rec1 in buffer) { yield return(ConvertToNestedObjectIfApplicable(new ChoDynamicObject(MigrateToNewSchema(rec1 as IDictionary <string, object>, recFieldTypes)) as object, headerLineLoaded)); } } } else { yield return(ConvertToNestedObjectIfApplicable(rec, headerLineLoaded)); } } else { yield return(rec); } if (Configuration.NotifyAfter > 0 && pair.Item1 % Configuration.NotifyAfter == 0) { if (RaisedRowsLoaded(pair.Item1)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested."); yield break; } } if (doWhile != null) { doWhile = RaiseDoWhile(pair); if (doWhile != null && doWhile.Value) { break; } } } } }
public override IEnumerable <object> WriteTo(object writer, IEnumerable <object> records, Func <object, bool> predicate = null) { TextWriter sw = writer as TextWriter; ChoGuard.ArgumentNotNull(sw, "TextWriter"); if (records == null) { yield break; } if (!RaiseBeginWrite(sw)) { yield break; } CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture; System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture; string recText = String.Empty; Type recType; try { foreach (object record in records) { _index++; recType = record.GetType(); if (record is IChoETLNameableObject) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name)); } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index)); } recText = String.Empty; if (record != null) { if (predicate == null || predicate(record)) { //Discover and load Manifold columns from first record if (!_configCheckDone) { //string[] fieldNames = null; //if (record is ExpandoObject) //{ // var dict = record as IDictionary<string, Object>; // fieldNames = dict.Keys.ToArray(); //} //else //{ // fieldNames = ChoTypeDescriptor.GetProperties<ChoManifoldRecordFieldAttribute>(record.GetType()).Select(pd => pd.Name).ToArray(); // if (fieldNames.Length == 0) // { // fieldNames = ChoType.GetProperties(record.GetType()).Select(p => p.Name).ToArray(); // } //} //Configuration.Validate(fieldNames); //WriteHeaderLine(sw); _configCheckDone = true; } if (!RaiseBeforeRecordWrite(record, _index, ref recText)) { yield break; } if (recText == null) { continue; } else if (recText.Length > 0) { sw.Write("{1}{0}", recText, Configuration.FileHeaderConfiguration.HasHeaderRecord ? Configuration.EOLDelimiter : ""); continue; } try { var config = GetConfiguration(record.GetType()); if (config == null) { throw new ChoParserException("No writer found to write record."); } if (config.GetType() == typeof(ChoCSVRecordConfiguration)) { recText = ChoCSVWriter.ToText(record, config as ChoCSVRecordConfiguration, Configuration.Encoding, Configuration.BufferSize); } else if (config.GetType() == typeof(ChoFixedLengthRecordConfiguration)) { recText = ChoFixedLengthWriter.ToText(record, config as ChoFixedLengthRecordConfiguration, Configuration.Encoding, Configuration.BufferSize); } else { throw new ChoParserException("Unsupported record found to write."); } if (recText != null) { if (_index == 1) { sw.Write("{1}{0}", recText, Configuration.FileHeaderConfiguration.HasHeaderRecord ? Configuration.EOLDelimiter : ""); } else { sw.Write("{1}{0}", recText, Configuration.EOLDelimiter); } if (!RaiseAfterRecordWrite(record, _index, recText)) { yield break; } } } catch (ChoParserException pEx) { throw new ChoParserException($"Failed to write line for '{recType}' object.", pEx); } catch (Exception ex) { ChoETLFramework.HandleException(ex); if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue) { } else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue) { if (!RaiseRecordWriteError(record, _index, recText, ex)) { throw new ChoWriterException($"Failed to write line for '{recType}' object.", ex); } } else { throw new ChoWriterException($"Failed to write line for '{recType}' object.", ex); } } } } yield return(record); if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0) { if (RaisedRowsWritten(_index)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested."); yield break; } } } } finally { System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo; } RaiseEndWrite(sw); }
public IEnumerable <T> WriteTo <T>(object writer, IEnumerable <object> records, Func <object, bool> predicate = null) { Configuration.Init(); if (records == null) { yield break; } var recEnum = records.GetEnumerator(); object notNullRecord = GetFirstNotNullRecord(recEnum); if (notNullRecord == null) { yield break; } DiscoverKnownTypes(notNullRecord); StreamWriter sw = null; if (writer is Lazy <StreamWriter> ) { var lsw = writer as Lazy <StreamWriter>; ChoGuard.ArgumentNotNull(lsw, "StreamWriter"); _sw = sw = lsw.Value; if (!Configuration.UseAvroSerializer) { if (_avroWriter == null) { _avroWriter = new SequentialWriter <T>(CreateAvroWriter <T>(sw), Configuration.SyncNumberOfObjects); } } else { _avroSerializer = CreateAvroSerializer <T>(); } } else { _avroWriter = writer as IAvroWriter <T>; if (_avroWriter == null) { throw new ChoParserException("Missing valid writer object passed."); } } if (!BeginWrite.Value) { yield break; } CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture; System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture; object recOutput = null; bool abortRequested = false; try { foreach (object record in GetRecords(recEnum)) { _index++; if (TraceSwitch.TraceVerbose) { if (record is IChoETLNameableObject) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name)); } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index)); } } recOutput = record; if (record != null) { if (predicate == null || predicate(record)) { if (!RaiseBeforeRecordWrite(record, _index, ref recOutput)) { yield break; } if (recOutput == null) { continue; } else if (!(recOutput is T)) { continue; } try { if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel) { record.DoObjectLevelValidation(Configuration, Configuration.AvroRecordFieldConfigurations); } if (recOutput is T) { if (recOutput is ChoDynamicObject) { recOutput = new Dictionary <string, object>((ChoDynamicObject)recOutput); } if (_sw != null) { if (Configuration.UseAvroSerializer) { IAvroSerializer <T> avroSerializer = _avroSerializer as IAvroSerializer <T>; avroSerializer.Serialize(sw.BaseStream, (T)(recOutput as object)); } else { SequentialWriter <T> avroWriter = _avroWriter as SequentialWriter <T>; avroWriter.Write((T)(recOutput as object)); } } else { SequentialWriter <T> avroWriter = _avroWriter as SequentialWriter <T>; avroWriter.Write((T)(recOutput as object)); } if (!RaiseAfterRecordWrite(record, _index, recOutput)) { yield break; } } } catch (Exception ex) { ChoETLFramework.HandleException(ref ex); if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue) { ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); } else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue) { if (!RaiseRecordWriteError(record, _index, recOutput, ex)) { throw; } else { //ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); } } else { throw; } } } } yield return((T)recOutput); recOutput = null; if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0) { if (RaisedRowsWritten(_index)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested."); abortRequested = true; yield break; } } } if (!abortRequested && recOutput != null) { RaisedRowsWritten(_index, true); } } finally { System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo; } }