public static bool Contains(string key) { ChoGuard.ArgumentNotNullOrEmpty(key, "Key"); return(Configuation.AppSettings.Settings.AllKeys.Contains(key)); }
public override IEnumerable <object> WriteTo(object writer, IEnumerable <object> records, Func <object, bool> predicate = null) { TextWriter sw = writer as TextWriter; ChoGuard.ArgumentNotNull(sw, "TextWriter"); if (records == null) { yield break; } CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture; System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture; _se = new Lazy <XmlSerializer>(() => Configuration.XmlSerializer == null ? null : Configuration.XmlSerializer); string recText = String.Empty; if (!Configuration.OmitXmlDeclaration) { sw.Write("{0}{1}", GetXmlDeclaration(), Configuration.EOLDelimiter); } try { foreach (object record in records) { _index++; if (TraceSwitch.TraceVerbose) { if (record is IChoETLNameableObject) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name)); } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index)); } } recText = String.Empty; if (predicate == null || predicate(record)) { //Discover and load Xml columns from first record if (!_configCheckDone) { if (record == null) { continue; } string[] fieldNames = null; Type recordType = ElementType == null?record.GetType() : ElementType; if (typeof(ICollection).IsAssignableFrom(recordType)) { recordType = recordType.GetEnumerableItemType().GetUnderlyingType(); } else { recordType = recordType.GetUnderlyingType(); } Configuration.IsDynamicObject = recordType.IsDynamicType(); if (!Configuration.IsDynamicObject) { if (recordType.IsSimple()) { Configuration.RecordType = typeof(ChoScalarObject <>).MakeGenericType(recordType); } else { Configuration.RecordType = recordType; } } if (Configuration.IsDynamicObject) { var dict = record.ToDynamicObject() as IDictionary <string, Object>; fieldNames = dict.Keys.ToArray(); } else { fieldNames = ChoTypeDescriptor.GetProperties <ChoXmlNodeRecordFieldAttribute>(Configuration.RecordType).Select(pd => pd.Name).ToArray(); if (fieldNames.Length == 0) { fieldNames = ChoType.GetProperties(Configuration.RecordType).Select(p => p.Name).ToArray(); } } Configuration.Validate(fieldNames); _configCheckDone = true; if (!RaiseBeginWrite(sw)) { yield break; } sw.Write("<{0}{1}>".FormatString(Configuration.RootName, GetNamespaceText())); } if (!RaiseBeforeRecordWrite(record, _index, ref recText)) { yield break; } if (recText == null) { continue; } try { if (!Configuration.UseXmlSerialization) { if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel) { record.DoObjectLevelValidation(Configuration, Configuration.XmlRecordFieldConfigurations); } if (ToText(_index, record, out recText)) { if (!recText.IsNullOrEmpty()) { sw.Write("{1}{0}", recText, Configuration.EOLDelimiter); } if (!RaiseAfterRecordWrite(record, _index, recText)) { yield break; } } } else { if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.Off) != ChoObjectValidationMode.Off) { record.DoObjectLevelValidation(Configuration, Configuration.XmlRecordFieldConfigurations); } if (record != null) { if (_se.Value != null) { _se.Value.Serialize(sw, record); } else { sw.Write("{1}{0}", ChoUtility.XmlSerialize(record).Indent(2, Configuration.IndentChar.ToString()), Configuration.EOLDelimiter); } if (!RaiseAfterRecordWrite(record, _index, null)) { yield break; } } } } //catch (ChoParserException) //{ // throw; //} catch (Exception ex) { ChoETLFramework.HandleException(ref ex); if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); } else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue) { if (!RaiseRecordWriteError(record, _index, recText, ex)) { throw; } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); } } else { throw; } } } yield return(record); if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0) { if (RaisedRowsWritten(_index)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested."); yield break; } } } } finally { System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo; } }
public static object ConvertFrom(object value, PropertyInfo propertyInfo, object sourceObject = null, CultureInfo culture = null) { ChoGuard.ArgumentNotNull(propertyInfo, "PropertyInfo"); return(ChoConvert.ConvertFrom(value, propertyInfo.PropertyType, sourceObject, ChoTypeDescriptor.GetTypeConverters(propertyInfo), ChoTypeDescriptor.GetTypeConverterParams(propertyInfo), culture)); }
internal ChoParquetRecordFieldConfigurationMap(ChoParquetRecordFieldConfiguration config) { ChoGuard.ArgumentNotNull(config, nameof(config)); _config = config; }
private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null) { TraceSwitch = traceSwitch; TextReader sr = source as TextReader; ChoGuard.ArgumentNotNull(sr, "TextReader"); if (sr is StreamReader) { ((StreamReader)sr).Seek(0, SeekOrigin.Begin); } if (!RaiseBeginLoad(sr)) { yield break; } string[] commentTokens = Configuration.Comments; bool? skip = false; bool abortRequested = false; long runningCount = 0; using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >( new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, Configuration.MayContainEOLInData)).ToEnumerable(), (pair) => { //bool isStateAvail = IsStateAvail(); skip = false; //if (isStateAvail) //{ // if (!IsStateMatches(item)) // { // skip = filterFunc != null ? filterFunc(item) : false; // } // else // skip = true; //} //else // skip = filterFunc != null ? filterFunc(item) : false; if (skip == null) { return(null); } if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine); if (!skip.Value) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1)); } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1)); } } if (skip.Value) { return(skip); } //if (!(sr.BaseStream is MemoryStream)) // ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1)); //if (Task != null) // return !IsStateNOTExistsOrNOTMatch(item); if (pair.Item2.IsNullOrWhiteSpace()) { if (!Configuration.IgnoreEmptyLine) { throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1)); } else { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1)); } return(true); } } //LoadExcelSeparator if any if (pair.Item1 == 1 && !_excelSeparatorFound) { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Inspecting for excel separator at [{0}]...".FormatString(pair.Item1)); } bool retVal = LoadExcelSeperatorIfAny(pair); _excelSeparatorFound = true; if (Configuration.HasExcelSeparator.HasValue && Configuration.HasExcelSeparator.Value && !retVal) { throw new ChoParserException("Missing excel separator header line in the file."); } if (retVal) { return(true); } } if (commentTokens != null && commentTokens.Length > 0) { foreach (string comment in commentTokens) { if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal)) //, true, Configuration.Culture)) { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1)); } return(true); } } } if (!_configCheckDone) { Configuration.Validate(GetHeaders(pair.Item2)); RaiseMembersDiscovered(Configuration.CSVRecordFieldConfigurations.Select(i => new KeyValuePair <string, Type>(i.Name, i.FieldType == null ? typeof(string) : i.FieldType)).ToArray()); _configCheckDone = true; } //LoadHeader if any if (Configuration.FileHeaderConfiguration.HasHeaderRecord && !_headerFound) { if (Configuration.FileHeaderConfiguration.IgnoreHeader) { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring header line at [{0}]...".FormatString(pair.Item1)); } } else { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading header line at [{0}]...".FormatString(pair.Item1)); } LoadHeaderLine(pair); } _headerFound = true; return(true); } return(false); })) { while (true) { Tuple <long, string> pair = e.Peek; if (pair == null) { if (!abortRequested) { RaisedRowsLoaded(runningCount); } RaiseEndLoad(sr); yield break; } runningCount = pair.Item1; object rec = Activator.CreateInstance(RecordType); if (!LoadLine(pair, ref rec)) { yield break; } //StoreState(e.Current, rec != null); e.MoveNext(); if (rec == null) { continue; } yield return(rec); if (Configuration.NotifyAfter > 0 && pair.Item1 % Configuration.NotifyAfter == 0) { if (RaisedRowsLoaded(pair.Item1)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested."); abortRequested = true; yield break; } } } } }
public bool Contains(string key) { ChoGuard.ArgumentNotNullOrEmpty(key, "Key"); return(_keyValues.ContainsKey(key)); }
public override IEnumerable <object> WriteTo(object writer, IEnumerable <object> records, Func <object, bool> predicate = null) { TextWriter sw = writer as TextWriter; ChoGuard.ArgumentNotNull(sw, "TextWriter"); if (records == null) { yield break; } if (!RaiseBeginWrite(sw)) { yield break; } CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture; System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture; string recText = String.Empty; long recCount = 0; var recEnum = records.GetEnumerator(); try { object notNullRecord = GetFirstNotNullRecord(recEnum); if (notNullRecord == null) { yield break; } if (Configuration.IsDynamicObject) { if (Configuration.MaxScanRows > 0) { List <string> fns = new List <string>(); foreach (object record1 in GetRecords(recEnum)) { recCount++; if (record1 != null) { if (recCount <= Configuration.MaxScanRows) { if (!record1.GetType().IsDynamicType()) { throw new ChoParserException("Invalid record found."); } _recBuffer.Value.Add(record1); fns = fns.Union(GetFields(record1)).ToList(); if (recCount == Configuration.MaxScanRows) { Configuration.Validate(fns.ToArray()); WriteHeaderLine(sw); _configCheckDone = true; break; } } } } } } foreach (object record in GetRecords(recEnum)) { _index++; if (TraceSwitch.TraceVerbose) { if (record is IChoETLNameableObject) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name)); } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index)); } } recText = String.Empty; if (record != null) { if (predicate == null || predicate(record)) { //Discover and load FixedLength columns from first record if (!_configCheckDone) { if (notNullRecord != null) { string[] fieldNames = GetFields(notNullRecord); Configuration.Validate(fieldNames); WriteHeaderLine(sw); _configCheckDone = true; } } //Check record if (record != null) { Type rt = record.GetType().ResolveType(); if (Configuration.IsDynamicObject) { if (ElementType != null) { } else if (!rt.IsDynamicType()) { throw new ChoWriterException("Invalid record found."); } } else { if (rt != Configuration.RecordType) { throw new ChoWriterException("Invalid record found."); } } } if (!RaiseBeforeRecordWrite(record, _index, ref recText)) { yield break; } if (recText == null) { continue; } else if (recText.Length > 0) { sw.Write("{1}{0}", recText, _hadHeaderWritten ? Configuration.EOLDelimiter : ""); continue; } try { if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel) { record.DoObjectLevelValidation(Configuration, Configuration.FixedLengthRecordFieldConfigurations); } if (ToText(_index, record, out recText)) { if (_index == 1) { sw.Write("{1}{0}", recText, _hadHeaderWritten ? Configuration.EOLDelimiter : ""); } else { sw.Write("{1}{0}", recText, Configuration.EOLDelimiter); } if (!RaiseAfterRecordWrite(record, _index, recText)) { yield break; } } } //catch (ChoParserException) //{ // throw; //} catch (Exception ex) { ChoETLFramework.HandleException(ref ex); if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); } else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue) { if (!RaiseRecordWriteError(record, _index, recText, ex)) { throw; } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); } } else { throw; } } } } yield return(record); if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0) { if (RaisedRowsWritten(_index)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested."); yield break; } } } } finally { System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo; } RaiseEndWrite(sw); }
public static bool ContainsAssembly(Assembly assembly) { ChoGuard.ArgumentNotNull(assembly, "Assembly"); return(_assemblyCache.ContainsKey(assembly.FullName)); }
public override IEnumerable <object> WriteTo(object writer, IEnumerable <object> records, Func <object, bool> predicate = null) { TextWriter sw = writer as TextWriter; ChoGuard.ArgumentNotNull(sw, "TextWriter"); if (records == null) { yield break; } if (!RaiseBeginWrite(sw)) { yield break; } CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture; System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture; string recText = String.Empty; Type recType; try { foreach (object record in records) { _index++; recType = record.GetType(); if (record is IChoETLNameableObject) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name)); } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index)); } recText = String.Empty; if (record != null) { if (predicate == null || predicate(record)) { //Discover and load Manifold columns from first record if (!_configCheckDone) { //string[] fieldNames = null; //if (record is ExpandoObject) //{ // var dict = record as IDictionary<string, Object>; // fieldNames = dict.Keys.ToArray(); //} //else //{ // fieldNames = ChoTypeDescriptor.GetProperties<ChoManifoldRecordFieldAttribute>(record.GetType()).Select(pd => pd.Name).ToArray(); // if (fieldNames.Length == 0) // { // fieldNames = ChoType.GetProperties(record.GetType()).Select(p => p.Name).ToArray(); // } //} //Configuration.Validate(fieldNames); //WriteHeaderLine(sw); _configCheckDone = true; } if (!RaiseBeforeRecordWrite(record, _index, ref recText)) { yield break; } if (recText == null) { continue; } else if (recText.Length > 0) { sw.Write("{1}{0}", recText, Configuration.FileHeaderConfiguration.HasHeaderRecord ? Configuration.EOLDelimiter : ""); continue; } try { var config = GetConfiguration(record.GetType()); if (config == null) { throw new ChoParserException("No writer found to write record."); } if (config.GetType() == typeof(ChoCSVRecordConfiguration)) { recText = ChoCSVWriter.ToText(record, config as ChoCSVRecordConfiguration, Configuration.Encoding, Configuration.BufferSize, _offSwitch); } else if (config.GetType() == typeof(ChoFixedLengthRecordConfiguration)) { recText = ChoFixedLengthWriter.ToText(record, config as ChoFixedLengthRecordConfiguration, Configuration.Encoding, Configuration.BufferSize, _offSwitch); } else { throw new ChoParserException("Unsupported record found to write."); } if (recText != null) { if (_index == 1) { sw.Write("{1}{0}", recText, Configuration.FileHeaderConfiguration.HasHeaderRecord ? Configuration.EOLDelimiter : ""); } else { sw.Write("{1}{0}", recText, Configuration.EOLDelimiter); } if (!RaiseAfterRecordWrite(record, _index, recText)) { yield break; } } } //catch (ChoParserException pEx) //{ // throw new ChoParserException($"Failed to write line for '{recType}' object.", pEx); //} catch (Exception ex) { ChoETLFramework.HandleException(ref ex); if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); } else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue) { if (!RaiseRecordWriteError(record, _index, recText, ex)) { throw new ChoWriterException($"Failed to write line for '{recType}' object.", ex); } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); } } else { throw new ChoWriterException($"Failed to write line for '{recType}' object.", ex); } } } } yield return(record); if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0) { if (RaisedRowsWritten(_index)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested."); yield break; } } } } finally { System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo; } RaiseEndWrite(sw); }
public static object ConvertTo(object value, MemberInfo memberInfo, Type targetType, object sourceObject = null, CultureInfo culture = null) { ChoGuard.ArgumentNotNull((object)memberInfo, "MemberInfo"); return(ChoConvert.ConvertTo(value, targetType, sourceObject, ChoTypeDescriptor.GetTypeConverters(memberInfo), ChoTypeDescriptor.GetTypeConverterParams(memberInfo), culture, memberInfo.Name)); }
public ChoManifoldRecordWriter(ChoManifoldRecordConfiguration configuration) : base(typeof(object)) { ChoGuard.ArgumentNotNull(configuration, "Configuration"); Configuration = configuration; }
public ChoPeekEnumerator(IEnumerable <T> enumerable) { ChoGuard.ArgumentNotNull(enumerable, "enumerable"); _enumerable = enumerable; Reset(); }
private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null) { TraceSwitch = traceSwitch; TextReader sr = source as TextReader; ChoGuard.ArgumentNotNull(sr, "TextReader"); if (sr is StreamReader) { ((StreamReader)sr).Seek(0, SeekOrigin.Begin); } if (!RaiseBeginLoad(sr)) { yield break; } string[] commentTokens = Configuration.Comments; bool? skip = false; bool isRecordStartFound = false; bool isRecordEndFound = false; long seekOriginPos = sr is StreamReader ? ((StreamReader)sr).BaseStream.Position : 0; List <string> headers = new List <string>(); Tuple <long, string> lastLine = null; List <Tuple <long, string> > recLines = new List <Tuple <long, string> >(); long recNo = 0; int loopCount = Configuration.AutoDiscoverColumns && Configuration.KVPRecordFieldConfigurations.Count == 0 ? 2 : 1; bool isHeaderFound = loopCount == 1; bool IsHeaderLoaded = false; Tuple <long, string> pairIn; bool abortRequested = false; for (int i = 0; i < loopCount; i++) { if (i == 1) { if (sr is StreamReader) { ((StreamReader)sr).Seek(seekOriginPos, SeekOrigin.Begin); } TraceSwitch = traceSwitch; } else { TraceSwitch = ChoETLFramework.TraceSwitchOff; } lastLine = null; recLines.Clear(); isRecordEndFound = false; isRecordStartFound = false; using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >( new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, Configuration.MayContainEOLInData)).ToEnumerable(), (pair) => { //bool isStateAvail = IsStateAvail(); skip = false; //if (isStateAvail) //{ // if (!IsStateMatches(item)) // { // skip = filterFunc != null ? filterFunc(item) : false; // } // else // skip = true; //} //else // skip = filterFunc != null ? filterFunc(item) : false; if (skip == null) { return(null); } if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine); if (!skip.Value) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1)); } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1)); } } if (skip.Value) { return(skip); } //if (!(sr.BaseStream is MemoryStream)) // ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1)); //if (Task != null) // return !IsStateNOTExistsOrNOTMatch(item); if (pair.Item2.IsNullOrWhiteSpace()) { if (!Configuration.IgnoreEmptyLine) { throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1)); } else { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1)); } return(true); } } if (commentTokens != null && commentTokens.Length > 0) { foreach (string comment in commentTokens) { if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal)) //, true, Configuration.Culture)) { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1)); } return(true); } } } if (!_configCheckDone) { Configuration.Validate(null); var dict = Configuration.KVPRecordFieldConfigurations.ToDictionary(i1 => i1.Name, i1 => i1.FieldType == null ? null : i1.FieldType); RaiseMembersDiscovered(dict); Configuration.UpdateFieldTypesIfAny(dict); _configCheckDone = true; } return(false); })) { while (true) { pairIn = e.Peek; if (!isRecordStartFound) { if (pairIn == null) { break; } lastLine = null; recLines.Clear(); isRecordEndFound = false; isRecordStartFound = true; if (!Configuration.RecordStart.IsNullOrWhiteSpace()) { //Move to record start while (!(Configuration.IsRecordStartMatch(e.Peek.Item2))) { e.MoveNext(); if (e.Peek == null) { break; } } } if (e.Peek != null) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Record start found at [{0}] line...".FormatString(e.Peek.Item1)); } e.MoveNext(); continue; } else { string recordEnd = !Configuration.RecordEnd.IsNullOrWhiteSpace() ? Configuration.RecordEnd : Configuration.RecordStart; if (!recordEnd.IsNullOrWhiteSpace()) { if (e.Peek == null) { if (Configuration.RecordEnd.IsNullOrWhiteSpace()) { isRecordEndFound = true; } else { break; } } else { //Move to record start if (Configuration.IsRecordEndMatch(e.Peek.Item2)) { isRecordEndFound = true; isRecordStartFound = false; if (e.Peek != null) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Record end found at [{0}] line...".FormatString(e.Peek.Item1)); } } } } else if (e.Peek == null) { isRecordEndFound = true; } if (!isHeaderFound) { //if (isRecordEndFound && headers.Count == 0) //{ // //throw new ChoParserException("Unexpected EOF found."); //} if (!isRecordEndFound) { e.MoveNext(); if (e.Peek != null) { //If line empty or line continuation, skip if (pairIn.Item2.IsNullOrWhiteSpace() || IsLineContinuationCharFound(pairIn.Item2)) //.Item2[0] == ' ' || pairIn.Item2[0] == '\t') { } else { string header = ToKVP(pairIn.Item1, pairIn.Item2).Key; if (!header.IsNullOrWhiteSpace()) { headers.Add(header); } } } } else { Configuration.Validate(headers.ToArray()); isHeaderFound = true; isRecordEndFound = false; IsHeaderLoaded = true; break; } } else { if (!IsHeaderLoaded) { Configuration.Validate(new string[] { }); IsHeaderLoaded = true; } if (isRecordEndFound && recLines.Count == 0) { //throw new ChoParserException("Unexpected EOF found."); } else if (!isRecordEndFound) { e.MoveNext(); if (e.Peek != null) { //If line empty or line continuation, skip if (pairIn.Item2.IsNullOrWhiteSpace()) { if (!Configuration.IgnoreEmptyLine) { throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pairIn.Item1)); } else { Tuple <long, string> t = new Tuple <long, string>(lastLine.Item1, lastLine.Item2 + Configuration.EOLDelimiter); recLines.RemoveAt(recLines.Count - 1); recLines.Add(t); } } else if (IsLineContinuationCharFound(pairIn.Item2)) //pairIn.Item2[0] == ' ' || pairIn.Item2[0] == '\t') { if (lastLine == null) { throw new ChoParserException("Unexpected line continuation found at {0} location.".FormatString(pairIn.Item1)); } else { Tuple <long, string> t = new Tuple <long, string>(lastLine.Item1, lastLine.Item2 + Configuration.EOLDelimiter + pairIn.Item2); recLines.RemoveAt(recLines.Count - 1); recLines.Add(t); } } else { lastLine = pairIn; recLines.Add(pairIn); } } } else { object rec = Configuration.IsDynamicObject ? new ChoDynamicObject(new Dictionary <string, object>(Configuration.FileHeaderConfiguration.StringComparer)) { ThrowExceptionIfPropNotExists = true, AlternativeKeys = Configuration.AlternativeKeys } : Activator.CreateInstance(RecordType); if (!LoadLines(new Tuple <long, List <Tuple <long, string> > >(++recNo, recLines), ref rec)) { yield break; } isRecordStartFound = false; //StoreState(e.Current, rec != null); if (!Configuration.RecordEnd.IsNullOrWhiteSpace()) { e.MoveNext(); } if (rec == null) { continue; } yield return(rec); if (Configuration.NotifyAfter > 0 && recNo % Configuration.NotifyAfter == 0) { if (RaisedRowsLoaded(recNo)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested."); abortRequested = true; yield break; } } if (e.Peek == null) { break; } } } } } } } if (!abortRequested) { RaisedRowsLoaded(recNo, true); } RaiseEndLoad(sr); }
public IEnumerable <T> WriteTo <T>(object writer, IEnumerable <object> records, Func <object, bool> predicate = null) { _sw = writer; IAvroWriter <T> sw1 = writer as IAvroWriter <T>; ChoGuard.ArgumentNotNull(sw1, "AvroWriter"); using (SequentialWriter <T> sw = new SequentialWriter <T>(sw1, 24)) { if (records == null) { yield break; } if (!BeginWrite.Value) { yield break; } CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture; System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture; object recOutput = null; var recEnum = records.GetEnumerator(); try { object notNullRecord = GetFirstNotNullRecord(recEnum); if (notNullRecord == null) { yield break; } foreach (object record in GetRecords(recEnum)) { _index++; if (TraceSwitch.TraceVerbose) { if (record is IChoETLNameableObject) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name)); } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index)); } } recOutput = record; if (record != null) { if (predicate == null || predicate(record)) { if (!RaiseBeforeRecordWrite(record, _index, ref recOutput)) { yield break; } if (recOutput == null) { continue; } else if (!(recOutput is T)) { continue; } try { if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel) { record.DoObjectLevelValidation(Configuration, Configuration.AvroRecordFieldConfigurations); } if (recOutput is T) { sw.Write((T)recOutput); if (!RaiseAfterRecordWrite(record, _index, recOutput)) { yield break; } } } catch (Exception ex) { ChoETLFramework.HandleException(ref ex); if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue) { ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); } else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue) { if (!RaiseRecordWriteError(record, _index, recOutput, ex)) { throw; } else { //ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); } } else { throw; } } } } yield return((T)recOutput); if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0) { if (RaisedRowsWritten(_index)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested."); yield break; } } } } finally { System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo; } } }
private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null) { TraceSwitch = traceSwitch; TextReader sr = source as TextReader; ChoGuard.ArgumentNotNull(sr, "TextReader"); if (sr is StreamReader) { ((StreamReader)sr).Seek(0, SeekOrigin.Begin); } if (!RaiseBeginLoad(sr)) { yield break; } string[] commentTokens = Configuration.Comments; bool? skip = false; bool abortRequested = false; long runningCount = 0; long recCount = 0; bool headerLineLoaded = false; List <object> buffer = new List <object>(); IDictionary <string, Type> recFieldTypes = null; using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >( new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, false /*Configuration.MayContainEOLInData*/)).ToEnumerable(), (pair) => { //bool isStateAvail = IsStateAvail(); skip = false; //if (isStateAvail) //{ // if (!IsStateMatches(item)) // { // skip = filterFunc != null ? filterFunc(item) : false; // } // else // skip = true; //} //else // skip = filterFunc != null ? filterFunc(item) : false; if (skip == null) { return(null); } if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine); if (!skip.Value) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1)); } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1)); } } if (skip.Value) { return(skip); } //if (!(sr.BaseStream is MemoryStream)) // ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1)); //if (Task != null) // return !IsStateNOTExistsOrNOTMatch(item); if (pair.Item2.IsNullOrWhiteSpace()) { if (!Configuration.IgnoreEmptyLine) { throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1)); } else { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1)); } return(true); } } if (commentTokens != null && commentTokens.Length > 0) { foreach (string comment in commentTokens) { if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal)) //, true, Configuration.Culture)) { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1)); } return(true); } } } if (Configuration.FileHeaderConfiguration.HeaderLineAt > 0) { if (pair.Item1 < Configuration.FileHeaderConfiguration.HeaderLineAt) { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Header line at {1}. Skipping [{0}] line...".FormatString(pair.Item1, Configuration.FileHeaderConfiguration.HeaderLineAt)); } return(true); } } if (!_configCheckDone) { Configuration.Validate(pair); // GetHeaders(pair.Item2)); var dict = recFieldTypes = Configuration.FixedLengthRecordFieldConfigurations.ToDictionary(i => i.Name, i => i.FieldType == null ? null : i.FieldType); RaiseMembersDiscovered(dict); Configuration.UpdateFieldTypesIfAny(dict); _configCheckDone = true; } //LoadHeader if any if ((Configuration.FileHeaderConfiguration.HasHeaderRecord || Configuration.FileHeaderConfiguration.HeaderLineAt > 0) && !_headerFound) { if (Configuration.FileHeaderConfiguration.IgnoreHeader) { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring header line at [{0}]...".FormatString(pair.Item1)); } } else { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading header line at [{0}]...".FormatString(pair.Item1)); } headerLineLoaded = true; LoadHeaderLine(pair); } _headerFound = true; return(true); } return(false); })) { while (true) { recCount++; Tuple <long, string> pair = e.Peek; if (pair == null) { if (!abortRequested) { RaisedRowsLoaded(runningCount); } RaiseEndLoad(sr); yield break; } runningCount = pair.Item1; object rec = Configuration.IsDynamicObject ? new ChoDynamicObject(new Dictionary <string, object>(Configuration.FileHeaderConfiguration.StringComparer)) { ThrowExceptionIfPropNotExists = true, AlternativeKeys = Configuration.AlternativeKeys } : Activator.CreateInstance(RecordType); if (!LoadLine(pair, ref rec)) { yield break; } //StoreState(e.Current, rec != null); e.MoveNext(); if (rec == null) { continue; } if (Configuration.IsDynamicObject) { if (Configuration.AreAllFieldTypesNull && Configuration.MaxScanRows > 0 && recCount <= Configuration.MaxScanRows) { buffer.Add(rec); RaiseRecordFieldTypeAssessment(recFieldTypes, (IDictionary <string, object>)rec, recCount == Configuration.MaxScanRows); if (recCount == Configuration.MaxScanRows) { Configuration.UpdateFieldTypesIfAny(recFieldTypes); var dict = recFieldTypes = Configuration.FixedLengthRecordFieldConfigurations.ToDictionary(i => i.Name, i => i.FieldType == null ? null : i.FieldType); RaiseMembersDiscovered(dict); foreach (object rec1 in buffer) { yield return(ConvertToNestedObjectIfApplicable(new ChoDynamicObject(MigrateToNewSchema(rec1 as IDictionary <string, object>, recFieldTypes)) as object, headerLineLoaded)); } } } else { yield return(ConvertToNestedObjectIfApplicable(rec, headerLineLoaded)); } } else { yield return(rec); } if (Configuration.NotifyAfter > 0 && pair.Item1 % Configuration.NotifyAfter == 0) { if (RaisedRowsLoaded(pair.Item1)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested."); yield break; } } } } }
public static bool ContainsAssembly(string assemblyName) { ChoGuard.ArgumentNotNullOrEmpty(assemblyName, "AssemblyName"); return(_assemblyCache.ContainsKey(assemblyName)); }
internal void WithField(string name, string yamlPath = null, Type fieldType = null, ChoFieldValueTrimOption fieldValueTrimOption = ChoFieldValueTrimOption.Trim, string fieldName = null, Func <object, object> valueConverter = null, Func <object, object> itemConverter = null, Func <object, object> customSerializer = null, object defaultValue = null, object fallbackValue = null, string fullyQualifiedMemberName = null, string formatText = null, bool isArray = true, string nullValue = null, Type recordType = null, Type subRecordType = null, Func <IDictionary <string, object>, Type> fieldTypeSelector = null) { ChoGuard.ArgumentNotNull(recordType, nameof(recordType)); if (!name.IsNullOrEmpty()) { if (subRecordType != null) { MapRecordFieldsForType(subRecordType); } string fnTrim = name.NTrim(); ChoYamlRecordFieldConfiguration fc = null; PropertyDescriptor pd = null; if (YamlRecordFieldConfigurations.Any(o => o.Name == fnTrim)) { fc = YamlRecordFieldConfigurations.Where(o => o.Name == fnTrim).First(); YamlRecordFieldConfigurations.Remove(fc); } else if (subRecordType != null) { pd = ChoTypeDescriptor.GetNestedProperty(subRecordType, fullyQualifiedMemberName.IsNullOrWhiteSpace() ? name : fullyQualifiedMemberName); } else { pd = ChoTypeDescriptor.GetNestedProperty(recordType, fullyQualifiedMemberName.IsNullOrWhiteSpace() ? name : fullyQualifiedMemberName); } var nfc = new ChoYamlRecordFieldConfiguration(fnTrim, yamlPath) { FieldType = fieldType, FieldValueTrimOption = fieldValueTrimOption, FieldName = fieldName.IsNullOrWhiteSpace() ? name : fieldName, ValueConverter = valueConverter, CustomSerializer = customSerializer, DefaultValue = defaultValue, FallbackValue = fallbackValue, FormatText = formatText, ItemConverter = itemConverter, IsArray = isArray, NullValue = nullValue, FieldTypeSelector = fieldTypeSelector, }; if (fullyQualifiedMemberName.IsNullOrWhiteSpace()) { nfc.PropertyDescriptor = fc != null ? fc.PropertyDescriptor : pd; nfc.DeclaringMember = fc != null ? fc.DeclaringMember : fullyQualifiedMemberName; } else { if (subRecordType == null) { pd = ChoTypeDescriptor.GetNestedProperty(recordType, fullyQualifiedMemberName); } else { pd = ChoTypeDescriptor.GetNestedProperty(subRecordType, fullyQualifiedMemberName); } nfc.PropertyDescriptor = pd; nfc.DeclaringMember = fullyQualifiedMemberName; } if (pd != null) { if (nfc.FieldType == null) { nfc.FieldType = pd.PropertyType; } } if (subRecordType == null) { YamlRecordFieldConfigurations.Add(nfc); } else { AddFieldForType(subRecordType, nfc); } } }
public static Assembly GetAssemblyFromCache(string assemblyFileName) { ChoGuard.ArgumentNotNullOrEmpty(assemblyFileName, "AssemblyFileName"); return(_assemblyCache[assemblyFileName]); }
private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null) { TraceSwitch = traceSwitch; StreamReader sr = source as StreamReader; ChoGuard.ArgumentNotNull(sr, "StreamReader"); sr.Seek(0, SeekOrigin.Begin); if (!RaiseBeginLoad(sr)) { yield break; } string[] commentTokens = Configuration.Comments; using (ChoPeekEnumerator <Tuple <int, string> > e = new ChoPeekEnumerator <Tuple <int, string> >( new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar)).ToEnumerable(), (pair) => { //bool isStateAvail = IsStateAvail(); bool?skip = false; //if (isStateAvail) //{ // if (!IsStateMatches(item)) // { // skip = filterFunc != null ? filterFunc(item) : false; // } // else // skip = true; //} //else // skip = filterFunc != null ? filterFunc(item) : false; if (skip == null) { return(null); } //if (!(sr.BaseStream is MemoryStream)) //{ ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine); if (!skip.Value) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1)); } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1)); } //} if (skip.Value) { return(skip); } //if (!(sr.BaseStream is MemoryStream)) // ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1)); //if (Task != null) // return !IsStateNOTExistsOrNOTMatch(item); if (pair.Item2.IsNullOrWhiteSpace()) { if (!Configuration.IgnoreEmptyLine) { throw new ChoParserException("Empty line found at {0} location.".FormatString(pair.Item1)); } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1)); return(true); } } //LoadExcelSeparator if any if (pair.Item1 == 1 && !_excelSeparatorFound) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Inspecting for excel separator at [{0}]...".FormatString(pair.Item1)); bool retVal = LoadExcelSeperatorIfAny(pair); _excelSeparatorFound = true; if (Configuration.HasExcelSeparator.HasValue && Configuration.HasExcelSeparator.Value && !retVal) { throw new ChoParserException("Missing excel separator header line in the file."); } if (retVal) { return(true); } } if (commentTokens == null) { return(false); } else { var x = (from comment in commentTokens where !pair.Item2.IsNull() && pair.Item2.StartsWith(comment, true, Configuration.Culture) select comment).FirstOrDefault(); if (x != null) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1)); return(true); } } if (!_configCheckDone) { Configuration.Validate(GetHeaders(pair.Item2)); _configCheckDone = true; } //LoadHeader if any if (Configuration.FileHeaderConfiguration.HasHeaderRecord && !_headerFound) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading header line at [{0}]...".FormatString(pair.Item1)); LoadHeaderLine(pair); _headerFound = true; return(true); } return(false); })) { while (true) { Tuple <int, string> pair = e.Peek; if (pair == null) { RaiseEndLoad(sr); yield break; } object rec = ChoActivator.CreateInstance(RecordType); if (!LoadLine(pair, ref rec)) { yield break; } //StoreState(e.Current, rec != null); e.MoveNext(); if (rec == null) { continue; } yield return(rec); } } }
public override IEnumerable <object> WriteTo(object writer, IEnumerable <object> records, Func <object, bool> predicate = null) { TextWriter sw = writer as TextWriter; ChoGuard.ArgumentNotNull(sw, "TextWriter"); if (records == null) { yield break; } if (!RaiseBeginWrite(sw)) { yield break; } CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture; System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture; string recText = String.Empty; try { foreach (object record in records) { _index++; if (TraceSwitch.TraceVerbose) { if (record is IChoETLNameableObject) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name)); } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index)); } } recText = String.Empty; if (record != null) { if (predicate == null || predicate(record)) { //Discover and load Xml columns from first record if (!_configCheckDone) { string[] fieldNames = null; if (record is ExpandoObject) { var dict = record as IDictionary <string, Object>; fieldNames = dict.Keys.ToArray(); } else { fieldNames = ChoTypeDescriptor.GetProperties <ChoXmlNodeRecordFieldAttribute>(record.GetType()).Select(pd => pd.Name).ToArray(); if (fieldNames.Length == 0) { fieldNames = ChoType.GetProperties(record.GetType()).Select(p => p.Name).ToArray(); } } Configuration.Validate(fieldNames); _configCheckDone = true; sw.Write("<{0}{1}>".FormatString(Configuration.RootName, GetNamespaceText())); } if (!RaiseBeforeRecordWrite(record, _index, ref recText)) { yield break; } if (recText == null) { continue; } try { if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel) { record.DoObjectLevelValidation(Configuration, Configuration.XmlRecordFieldConfigurations); } if (ToText(_index, record, out recText)) { sw.Write("{1}{0}", recText, Configuration.EOLDelimiter); if (!RaiseAfterRecordWrite(record, _index, recText)) { yield break; } } } catch (ChoParserException) { throw; } catch (Exception ex) { ChoETLFramework.HandleException(ex); if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue) { } else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue) { if (!RaiseRecordWriteError(record, _index, recText, ex)) { throw; } } else { throw; } } } } yield return(record); if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0) { if (RaisedRowsWritten(_index)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested."); yield break; } } } if (_configCheckDone) { sw.Write("{1}</{0}>".FormatString(Configuration.RootName, Configuration.EOLDelimiter)); } } finally { System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo; } RaiseEndWrite(sw); }
public ChoManifoldRecordReader(ChoManifoldRecordConfiguration configuration) : base(typeof(object)) { ChoGuard.ArgumentNotNull(configuration, "Configuration"); Configuration = configuration; //Configuration.Validate(); }
public static object ConvertFrom(object value, MemberInfo memberInfo, object sourceObject = null, CultureInfo culture = null) { ChoGuard.ArgumentNotNull(memberInfo, "MemberInfo"); return(ChoConvert.ConvertFrom(value, ChoType.GetMemberType(memberInfo), sourceObject, ChoTypeDescriptor.GetTypeConverters(memberInfo), ChoTypeDescriptor.GetTypeConverterParams(memberInfo), culture)); }
public override IEnumerable <object> WriteTo(object writer, IEnumerable <object> records, Func <object, bool> predicate = null) { _sw = writer; TextWriter sw = writer as TextWriter; ChoGuard.ArgumentNotNull(sw, "TextWriter"); if (Configuration.JsonSerializerSettings.ContractResolver is ChoPropertyRenameAndIgnoreSerializerContractResolver) { ChoPropertyRenameAndIgnoreSerializerContractResolver cr = Configuration.JsonSerializerSettings.ContractResolver as ChoPropertyRenameAndIgnoreSerializerContractResolver; cr.CallbackRecordFieldWrite = _callbackRecordFieldWrite; cr.Writer = Writer; } if (records == null) { yield break; } if (Configuration.SingleDocument == null) { Configuration.SingleDocument = false; } CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture; System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture; string recText = String.Empty; bool recordIgnored = false; try { foreach (object record in records) { _index++; //if (!isFirstRec) //{ // if (!recordIgnored) // sw.Write(","); // else // recordIgnored = false; //} if (TraceSwitch.TraceVerbose) { if (record is IChoETLNameableObject) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name)); } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index)); } } recText = String.Empty; if (predicate == null || predicate(record)) { //Discover and load Xml columns from first record if (!Configuration.IsInitialized) { if (record == null) { continue; } string[] fieldNames = null; if (Configuration.RecordType == typeof(object)) { Type recordType = ElementType == null?record.GetType() : ElementType; RecordType = Configuration.RecordType = recordType.GetUnderlyingType(); //.ResolveType(); Configuration.IsDynamicObject = recordType.IsDynamicType(); } if (typeof(IDictionary).IsAssignableFrom(Configuration.RecordType) || typeof(IList).IsAssignableFrom(Configuration.RecordType)) { Configuration.UseYamlSerialization = true; } if (!Configuration.IsDynamicObject) { if (Configuration.YamlRecordFieldConfigurations.Count == 0) { Configuration.MapRecordFields(Configuration.RecordType); } } if (Configuration.IsDynamicObject) { var dict = record.ToDynamicObject() as IDictionary <string, Object>; fieldNames = dict.Keys.ToArray(); } else { fieldNames = ChoTypeDescriptor.GetProperties <ChoYamlRecordFieldAttribute>(Configuration.RecordType).Select(pd => pd.Name).ToArray(); if (fieldNames.Length == 0) { fieldNames = ChoType.GetProperties(Configuration.RecordType).Select(p => p.Name).ToArray(); } } Configuration.Validate(fieldNames); Configuration.IsInitialized = true; if (!BeginWrite.Value) { yield break; } } if (!RaiseBeforeRecordWrite(record, _index, ref recText)) { yield break; } if (recText == null) { continue; } try { if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel) { record.DoObjectLevelValidation(Configuration, Configuration.YamlRecordFieldConfigurations); } if (ToText(_index, record, out recText)) { if (recText.EndsWith("...")) { recText = recText.Left(recText.Length - 3); } if (recText.EndsWith($"...{Environment.NewLine}")) { recText = recText.Left(recText.Length - $"...{Environment.NewLine}".Length); } if (!Configuration.SingleDocument.Value) { sw.Write($"---{EOLDelimiter}"); } sw.Write("{0}", recText); if (!Configuration.SingleDocument.Value) { sw.Write($"...{EOLDelimiter}"); } if (!RaiseAfterRecordWrite(record, _index, recText)) { yield break; } } } //catch (ChoParserException) //{ // throw; //} catch (Exception ex) { ChoETLFramework.HandleException(ref ex); if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue) { recordIgnored = true; ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); } else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue) { if (!RaiseRecordWriteError(record, _index, recText, ex)) { throw; } else { recordIgnored = true; ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); } } else { throw; } } } yield return(record); if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0) { if (RaisedRowsWritten(_index)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested."); yield break; } } } } finally { System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo; } }
public static IEnumerable <string> ReadLines(this TextReader reader, string EOLDelimiter = null, char quoteChar = ChoCharEx.NUL, bool mayContainEOLInData = false, int maxLineSize = 32768) { ChoGuard.ArgumentNotNull(reader, "TextReader"); EOLDelimiter = EOLDelimiter ?? Environment.NewLine; if (!mayContainEOLInData && (EOLDelimiter == Environment.NewLine || (EOLDelimiter.Length == 1 && EOLDelimiter[0] == '\r') || (EOLDelimiter.Length == 1 && EOLDelimiter[0] == '\n') )) { string line; while ((line = reader.ReadLine()) != null) { yield return(line); } } bool isPrevEscape = false; bool inQuote = false; List <char> buffer = new List <char>(); char c = ChoCharEx.NUL; CircularBuffer <char> delim_buffer = new CircularBuffer <char>(EOLDelimiter.Length); while (reader.Peek() >= 0) { isPrevEscape = c == ChoCharEx.Backslash; c = (char)reader.Read(); delim_buffer.Enqueue(c); if (quoteChar != ChoCharEx.NUL && quoteChar == c) { if (!isPrevEscape) { inQuote = !inQuote; } else if (reader.Peek() == quoteChar) { inQuote = false; } } if (!inQuote) { if (delim_buffer.ToString() == EOLDelimiter) { if (buffer.Count > 0) { string x = new String(buffer.ToArray()); yield return(x.Substring(0, x.Length - (EOLDelimiter.Length - 1))); buffer.Clear(); } continue; } } buffer.Add(c); if (buffer.Count > maxLineSize) { throw new ApplicationException("Large line found. Check and correct the end of line delimiter."); } } if (buffer.Count > 0) { yield return(new String(buffer.ToArray())); } else { yield break; } }
public static T JsonDeserialize <T>(string JsonString, Encoding encoding = null) { ChoGuard.ArgumentNotNullOrEmpty(JsonString, "JsonString"); return((T)JsonDeserialize(JsonString, typeof(T), encoding)); }