public static TValue GetValue <TValue>(this IDictionary <string, TValue> dict, string key, bool ignoreCase = false, CultureInfo culture = null, TValue defaultValue = default(TValue)) { ChoGuard.ArgumentNotNull(dict, "Dictionary"); if (culture == null) { culture = System.Threading.Thread.CurrentThread.CurrentCulture; } string cultureSpecificKeyName = dict.Keys.Where(i => String.Compare(i, key, ignoreCase, culture) == 0).FirstOrDefault(); if (!cultureSpecificKeyName.IsNullOrWhiteSpace()) { return(dict[cultureSpecificKeyName]); } else { return(defaultValue); } }
public static IEnumerable <T> GetPropetyAttributes <T>(Type type, string propName) where T : Attribute { ChoGuard.ArgumentNotNull(type, "Type"); ChoGuard.ArgumentNotNullOrEmpty(propName, "PropName"); Init(type); PropertyDescriptor pd = _pdDict[type].Where(pd1 => pd1.Name == propName && pd1.Attributes.OfType <T>().Any()).FirstOrDefault(); if (pd == null) { return(Enumerable.Empty <T>()); } else { return(GetPropetyAttributes <T>(pd)); } }
public void Write(DataTable dt) { ChoGuard.ArgumentNotNull(dt, "DataTable"); if (Configuration.UseJSONSerialization) { Write(dt); return; } DataTable schemaTable = dt; if (Configuration.JSONRecordFieldConfigurations.IsNullOrEmpty()) { string colName = null; Type colType = null; int startIndex = 0; int fieldLength = 0; foreach (DataColumn col in schemaTable.Columns) { colName = col.ColumnName; colType = col.DataType; //if (!colType.IsSimple()) continue; var obj = new ChoJSONRecordFieldConfiguration(colName, jsonPath: null); Configuration.JSONRecordFieldConfigurations.Add(obj); startIndex += fieldLength; } } Configuration.RootName = dt.TableName.IsNullOrWhiteSpace() ? null : dt.TableName; foreach (DataRow row in dt.Rows) { dynamic expando = new ExpandoObject(); var expandoDic = (IDictionary <string, object>)expando; foreach (var fc in Configuration.JSONRecordFieldConfigurations) { expandoDic.Add(fc.Name, row[fc.Name] == DBNull.Value ? null : row[fc.Name]); } Write(expando); } }
public static void AddOrUpdateValue <TValue>(this IDictionary <string, TValue> dict, string key, TValue value, bool ignoreCase = false, CultureInfo culture = null) { ChoGuard.ArgumentNotNull(dict, "Dictionary"); if (culture == null) { culture = System.Threading.Thread.CurrentThread.CurrentCulture; } string cultureSpecificKeyName = dict.Keys.Where(i => String.Compare(i, key, ignoreCase, culture) == 0).FirstOrDefault(); if (cultureSpecificKeyName.IsNullOrWhiteSpace()) { dict.Add(cultureSpecificKeyName, value); } else { dict[cultureSpecificKeyName] = value; } }
public void Write(IDataReader dr) { ChoGuard.ArgumentNotNull(dr, "DataReader"); DataTable schemaTable = dr.GetSchemaTable(); dynamic expando = new ExpandoObject(); var expandoDic = (IDictionary <string, object>)expando; Configuration.UseNestedKeyFormat = false; int ordinal = 0; if (Configuration.CSVRecordFieldConfigurations.IsNullOrEmpty()) { string colName = null; Type colType = null; foreach (DataRow row in schemaTable.Rows) { colName = row["ColumnName"].CastTo <string>(); colType = row["DataType"] as Type; //if (!colType.IsSimple()) continue; Configuration.CSVRecordFieldConfigurations.Add(new ChoCSVRecordFieldConfiguration(colName, ++ordinal) { FieldType = colType }); } } var ordinals = Configuration.CSVRecordFieldConfigurations.ToDictionary(c => c.Name, c => dr.HasColumn(c.Name) ? dr.GetOrdinal(c.Name) : -1); while (dr.Read()) { expandoDic.Clear(); foreach (var fc in ordinals) { expandoDic.Add(fc.Key, fc.Value == -1 ? null : dr[fc.Value]); } Write(expando); } }
public void Write(IDataReader dr) { ChoGuard.ArgumentNotNull(dr, "DataReader"); DataTable schemaTable = dr.GetSchemaTable(); dynamic expando = new ExpandoObject(); var expandoDic = (IDictionary <string, object>)expando; Configuration.UseNestedKeyFormat = false; //int ordinal = 0; if (Configuration.FixedLengthRecordFieldConfigurations.IsNullOrEmpty()) { string colName = null; Type colType = null; int startIndex = 0; int fieldLength = 0; foreach (DataRow row in schemaTable.Rows) { colName = row["ColumnName"].CastTo <string>(); colType = row["DataType"] as Type; //if (!colType.IsSimple()) continue; fieldLength = ChoFixedLengthFieldDefaultSizeConfiguation.Instance.GetSize(colType); var obj = new ChoFixedLengthRecordFieldConfiguration(colName, startIndex, fieldLength); Configuration.FixedLengthRecordFieldConfigurations.Add(obj); startIndex += fieldLength; } } while (dr.Read()) { expandoDic.Clear(); foreach (var fc in Configuration.FixedLengthRecordFieldConfigurations) { expandoDic.Add(fc.Name, dr[fc.Name]); } Write(expando); } }
public static string JsonSerialize(object target, Encoding encoding = null) { ChoGuard.ArgumentNotNull(target, "Target"); encoding = encoding == null ? Encoding.UTF8 : encoding; StringBuilder JsonString = new StringBuilder(); using (MemoryStream ms = new MemoryStream()) { using (var writer = JsonReaderWriterFactory.CreateJsonWriter( ms, encoding, true, true, " ")) { DataContractJsonSerializer serializer = new DataContractJsonSerializer(target.GetType(), jsonSettings); serializer.WriteObject(writer, target); writer.Flush(); byte[] json = ms.ToArray(); return(encoding.GetString(json, 0, json.Length)); } } }
private static Assembly GetWebApplicationAssembly(HttpContext context) { ChoGuard.ArgumentNotNull(context, "context"); IHttpHandler handler = context.CurrentHandler; if (handler == null) { return(null); } Type type = handler.GetType(); while (type != null && type != typeof(object) && type.Namespace == AspNetNamespace) { type = type.BaseType; } return(type.Assembly); }
public override IEnumerable <object> AsEnumerable(object source, Func <object, bool?> filterFunc = null) { ParquetReader sr = source as ParquetReader; ChoGuard.ArgumentNotNull(sr, "ParquetReader"); InitializeRecordConfiguration(Configuration); if (!RaiseBeginLoad(sr)) { yield break; } foreach (var item in AsEnumerable(ReadObjects(sr), TraceSwitch, filterFunc)) { yield return(item); } RaiseEndLoad(sr); }
public ChoJObjectWriter(string propName, ChoJObjectWriter writer) { ChoGuard.ArgumentNotNull(writer, nameof(writer)); ChoGuard.ArgumentNotNull(propName, nameof(propName)); _writer = writer._writer; _options = writer._options; _formatting = writer.Formatting; _writeStartObject = new Lazy <bool>(() => { _writer.WriteStartObject(); return(true); }); var _ = writer._writeStartObject.Value; _writer.WritePropertyName(propName); var x = _writeStartObject.Value; }
public static void Write(ChoFixedLengthWriter <dynamic> w, DataTable dt) { ChoGuard.ArgumentNotNull(w, "Writer"); ChoGuard.ArgumentNotNull(dt, "DataTable"); DataTable schemaTable = dt; w.Configuration.UseNestedKeyFormat = false; if (w.Configuration.FixedLengthRecordFieldConfigurations.IsNullOrEmpty()) { string colName = null; Type colType = null; int startIndex = 0; int fieldLength = 0; foreach (DataColumn col in schemaTable.Columns) { colName = col.ColumnName; colType = col.DataType; //if (!colType.IsSimple()) continue; fieldLength = ChoFixedLengthFieldDefaultSizeConfiguation.Instance.GetSize(colType); var obj = new ChoFixedLengthRecordFieldConfiguration(colName, startIndex, fieldLength); w.Configuration.FixedLengthRecordFieldConfigurations.Add(obj); startIndex += fieldLength; } } foreach (DataRow row in dt.Rows) { dynamic expando = new ExpandoObject(); var expandoDic = (IDictionary <string, object>)expando; foreach (var fc in w.Configuration.FixedLengthRecordFieldConfigurations) { expandoDic.Add(fc.Name, row[fc.Name]); } w.Write(expando); } }
public void Write(DataTable dt) { ChoGuard.ArgumentNotNull(dt, "DataTable"); DataTable schemaTable = dt; dynamic expando = new ExpandoObject(); var expandoDic = (IDictionary <string, object>)expando; Configuration.UseNestedKeyFormat = false; int ordinal = 0; if (Configuration.CSVRecordFieldConfigurations.IsNullOrEmpty()) { string colName = null; Type colType = null; foreach (DataColumn col in schemaTable.Columns) { colName = col.ColumnName; colType = col.DataType; //if (!colType.IsSimple()) continue; Configuration.CSVRecordFieldConfigurations.Add(new ChoCSVRecordFieldConfiguration(colName, ++ordinal) { FieldType = colType }); } } foreach (DataRow row in dt.Rows) { expandoDic.Clear(); foreach (var fc in Configuration.CSVRecordFieldConfigurations) { expandoDic.Add(fc.Name, row[fc.Name]); } Write(expando); } }
public ChoFixedLengthRecordWriter(Type recordType, ChoFixedLengthRecordConfiguration configuration) : base(recordType) { ChoGuard.ArgumentNotNull(configuration, "Configuration"); Configuration = configuration; _callbackFileHeaderArrange = ChoMetadataObjectCache.CreateMetadataObject <IChoNotifyFileHeaderArrange>(recordType); _callbackFileHeaderWrite = ChoMetadataObjectCache.CreateMetadataObject <IChoNotifyFileHeaderWrite>(recordType); _callbackRecordWrite = ChoMetadataObjectCache.CreateMetadataObject <IChoNotifyRecordWrite>(recordType); _callbackFileWrite = ChoMetadataObjectCache.CreateMetadataObject <IChoNotifyFileWrite>(recordType); _callbackRecordFieldWrite = ChoMetadataObjectCache.CreateMetadataObject <IChoNotifyRecordFieldWrite>(recordType); _recBuffer = new Lazy <List <object> >(() => { if (Writer != null) { var b = Writer.Context.ContainsKey("RecBuffer") ? Writer.Context.RecBuffer : null; if (b == null) { Writer.Context.RecBuffer = new List <object>(); } return(Writer.Context.RecBuffer); } else { return(new List <object>()); } }); BeginWrite = new Lazy <bool>(() => { TextWriter sw = _sw as TextWriter; if (sw != null) { return(RaiseBeginWrite(sw)); } return(false); }); //Configuration.Validate(); }
public void Write(IDataReader dr) { throw new NotSupportedException(); ChoGuard.ArgumentNotNull(dr, "DataReader"); DataTable schemaTable = dr.GetSchemaTable(); dynamic expando = new ExpandoObject(); var expandoDic = (IDictionary <string, object>)expando; Configuration.UseNestedKeyFormat = false; if (Configuration.AvroRecordFieldConfigurations.IsNullOrEmpty()) { string colName = null; Type colType = null; foreach (DataRow row in schemaTable.Rows) { colName = row["ColumnName"].CastTo <string>(); colType = row["DataType"] as Type; //if (!colType.IsSimple()) continue; Configuration.AvroRecordFieldConfigurations.Add(new ChoBSONRecordFieldConfiguration(colName) { FieldType = colType }); } } while (dr.Read()) { expandoDic.Clear(); foreach (var fc in Configuration.AvroRecordFieldConfigurations) { expandoDic.Add(fc.Name, dr[fc.Name]); } Write(expando); } }
public object GetDefaultValue(MemberInfo mi) { ChoGuard.ArgumentNotNull(mi, "MemberInfo"); Type declaringType = mi.ReflectedType; if (!typeof(ChoRecord).IsAssignableFrom(declaringType) || declaringType.IsDynamicRecord()) { return(null); } InitDefaults(declaringType); if (!_defaultsCache.ContainsKey(declaringType) || !_defaultsCache[declaringType].ContainsKey(mi.Name)) { return(null); } return(_defaultsCache[declaringType][mi.Name]); }
public object[] GetConverterParams(MemberInfo mi) { ChoGuard.ArgumentNotNull(mi, "MemberInfo"); Type declaringType = mi.ReflectedType; if (!typeof(ChoRecord).IsAssignableFrom(declaringType) || declaringType.IsDynamicRecord()) { return(null); } InitConverterParams(declaringType); if (!_converterParams.ContainsKey(declaringType) || !_converterParams[declaringType].ContainsKey(mi.Name)) { return(null); } return(_converterParams[declaringType][mi.Name]); }
public bool IsRequired(MemberInfo mi) { ChoGuard.ArgumentNotNull(mi, "MemberInfo"); Type declaringType = mi.ReflectedType; if (!typeof(ChoRecord).IsAssignableFrom(declaringType) || declaringType.IsDynamicRecord()) { return(false); } InitIsRequireds(declaringType); if (!_isReqCache.ContainsKey(declaringType) || !_isReqCache[declaringType].ContainsKey(mi.Name)) { return(false); } return(_isReqCache[declaringType][mi.Name]); }
public ChoEnumerableDataReader(IEnumerable collection, IChoDeferedObjectMemberDiscoverer dom) { ChoGuard.ArgumentNotNull(collection, "Collection"); ChoGuard.ArgumentNotNull(dom, "DeferedObjectMemberDiscoverer"); _isDeferred = true; _type = GetElementType(collection); dom.MembersDiscovered += (o, e) => { _dynamicMembersInfo = e.Value.ToArray(); SetFields(_type, e.Value.ToArray()); }; _enumerator = collection.GetEnumerator(); _firstElementExists = _enumerator.MoveNext(); if (_enumerator.Current is ChoDynamicObject && ((ChoDynamicObject)_enumerator.Current).IsHeaderOnlyObject) { _firstElementExists = false; } }
public static IEnumerable <string> ReadLines(this StreamReader reader, char delimeter) { ChoGuard.ArgumentNotNull(reader, "TextReader"); List <char> chars = new List <char>(); while (reader.Peek() >= 0) { char c = (char)reader.Read(); if (c == delimeter) { yield return(new String(chars.ToArray())); chars.Clear(); continue; } chars.Add(c); } }
public static void RegisterConverters(Type type, TypeConverter[] typeConverters) { ChoGuard.ArgumentNotNull(type, "Type"); lock (_typeMemberTypeConverterCacheLockObject) { if (typeConverters == null) { typeConverters = EmptyTypeConverters; } if (_typeTypeConverterCache.ContainsKey(type)) { _typeTypeConverterCache[type] = typeConverters; } else { _typeTypeConverterCache.Add(type, typeConverters); } } }
public static void RegisterConverters(MemberInfo memberInfo, TypeConverter[] typeConverters) { ChoGuard.ArgumentNotNull(memberInfo, "MemberInfo"); lock (_typeMemberTypeConverterCacheLockObject) { if (typeConverters == null) { typeConverters = EmptyTypeConverters; } if (_typeMemberTypeConverterCache.ContainsKey(memberInfo)) { _typeMemberTypeConverterCache[memberInfo] = typeConverters; } else { _typeMemberTypeConverterCache.Add(memberInfo, typeConverters); } } }
public ChoAvroRecordWriter(Type recordType, ChoBSONRecordConfiguration configuration) : base(recordType) { ChoGuard.ArgumentNotNull(configuration, "Configuration"); Configuration = configuration; _callbackFileHeaderArrange = ChoMetadataObjectCache.CreateMetadataObject <IChoNotifyFileHeaderArrange>(recordType); _callbackRecordWrite = ChoMetadataObjectCache.CreateMetadataObject <IChoNotifyRecordWrite>(recordType); _callbackFileWrite = ChoMetadataObjectCache.CreateMetadataObject <IChoNotifyFileWrite>(recordType); _callbackRecordFieldWrite = ChoMetadataObjectCache.CreateMetadataObject <IChoNotifyRecordFieldWrite>(recordType); System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture; _recBuffer = new Lazy <List <object> >(() => { if (Writer != null) { var b = Writer.Context.ContainsKey("RecBuffer") ? Writer.Context.RecBuffer : null; if (b == null) { Writer.Context.RecBuffer = new List <object>(); } return(Writer.Context.RecBuffer); } else { return(new List <object>()); } }, true); BeginWrite = new Lazy <bool>(() => { if (_sw != null) { return(RaiseBeginWrite(_sw)); } return(false); }); //Configuration.Validate(); }
public static Dictionary <string, object> ToDictionary(this object target) { ChoGuard.ArgumentNotNull(target, "Target"); if (target is IDictionary <string, object> ) { return((Dictionary <string, object>)target); } if (target is IDictionary) { Dictionary <string, object> dict1 = new Dictionary <string, object>(); foreach (var kvp in ((IDictionary)target).Keys) { dict1.Add(kvp.ToNString(), ((IDictionary)target)[kvp]); } return(dict1); } if (target is IEnumerable <KeyValuePair <string, object> > ) { return(new List <KeyValuePair <string, object> >(target as IEnumerable <KeyValuePair <string, object> >).ToDictionary(x => x.Key, x => x.Value)); } if (target is IEnumerable <Tuple <string, object> > ) { return(new List <Tuple <string, object> >(target as IEnumerable <Tuple <string, object> >).ToDictionary(x => x.Item1, x => x.Item2)); } if (target is IList) { return(((IList)(target)).OfType <object>().Select((item, index) => new KeyValuePair <string, object>("Column_{0}".FormatString(index), item)).ToDictionary(kvp => kvp.Key, kvp => kvp.Value)); } Dictionary <string, object> dict = new Dictionary <string, object>(); foreach (PropertyDescriptor pd in ChoTypeDescriptor.GetProperties(target.GetType())) { dict.Add(pd.Name, ChoType.GetPropertyValue(target, pd.Name)); } return(dict); }
public void Write(IDataReader dr) { ChoGuard.ArgumentNotNull(dr, "DataReader"); DataTable schemaTable = dr.GetSchemaTable(); dynamic expando = new ExpandoObject(); var expandoDic = (IDictionary <string, object>)expando; int ordinal = 0; if (Configuration.CSVRecordFieldConfigurations.IsNullOrEmpty()) { string colName = null; Type colType = null; foreach (DataRow row in schemaTable.Rows) { colName = row["ColumnName"].CastTo <string>(); colType = row["DataType"] as Type; //if (!colType.IsSimple()) continue; Configuration.CSVRecordFieldConfigurations.Add(new ChoCSVRecordFieldConfiguration(colName, ++ordinal) { FieldType = colType }); } } while (dr.Read()) { expandoDic.Clear(); foreach (var fc in Configuration.CSVRecordFieldConfigurations) { expandoDic.Add(fc.Name, dr[fc.Name]); } Write(expando); } }
public ChoYamlRecordWriter(Type recordType, ChoYamlRecordConfiguration configuration) : base(recordType, true) { ChoGuard.ArgumentNotNull(configuration, "Configuration"); Configuration = configuration; _callbackRecordWrite = ChoMetadataObjectCache.CreateMetadataObject <IChoNotifyRecordWrite>(recordType); _callbackFileWrite = ChoMetadataObjectCache.CreateMetadataObject <IChoNotifyFileWrite>(recordType); _callbackRecordFieldWrite = ChoMetadataObjectCache.CreateMetadataObject <IChoNotifyRecordFieldWrite>(recordType); _callbackRecordSeriablizable = ChoMetadataObjectCache.CreateMetadataObject <IChoRecordFieldSerializable>(recordType); System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture; //_recBuffer = new Lazy<List<object>>(() => //{ // if (Writer != null) // { // var b = Writer.Context.ContainsKey("RecBuffer") ? Writer.Context.RecBuffer : null; // if (b == null) // Writer.Context.RecBuffer = new List<object>(); // return Writer.Context.RecBuffer; // } // else // return new List<object>(); //}, true); //Configuration.Validate(); BeginWrite = new Lazy <bool>(() => { TextWriter sw = _sw as TextWriter; if (sw != null) { return(RaiseBeginWrite(sw)); } return(false); }); }
private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null) { TraceSwitch = traceSwitch; TextReader sr = source as TextReader; ChoGuard.ArgumentNotNull(sr, "TextReader"); if (sr is StreamReader) { ((StreamReader)sr).Seek(0, SeekOrigin.Begin); } if (!RaiseBeginLoad(sr)) { yield break; } string[] commentTokens = Configuration.Comments; bool? skip = false; bool isRecordStartFound = false; bool isRecordEndFound = false; long seekOriginPos = sr is StreamReader ? ((StreamReader)sr).BaseStream.Position : 0; List <string> headers = new List <string>(); Tuple <long, string> lastLine = null; List <Tuple <long, string> > recLines = new List <Tuple <long, string> >(); long recNo = 0; int loopCount = Configuration.AutoDiscoverColumns && Configuration.KVPRecordFieldConfigurations.Count == 0 ? 2 : 1; bool isHeaderFound = loopCount == 1; bool IsHeaderLoaded = false; Tuple <long, string> pairIn; bool abortRequested = false; for (int i = 0; i < loopCount; i++) { if (i == 1) { if (sr is StreamReader) { ((StreamReader)sr).Seek(seekOriginPos, SeekOrigin.Begin); } TraceSwitch = traceSwitch; } else { TraceSwitch = ChoETLFramework.TraceSwitchOff; } lastLine = null; recLines.Clear(); isRecordEndFound = false; isRecordStartFound = false; using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >( new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, Configuration.MayContainEOLInData)).ToEnumerable(), (pair) => { //bool isStateAvail = IsStateAvail(); skip = false; //if (isStateAvail) //{ // if (!IsStateMatches(item)) // { // skip = filterFunc != null ? filterFunc(item) : false; // } // else // skip = true; //} //else // skip = filterFunc != null ? filterFunc(item) : false; if (skip == null) { return(null); } if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine); if (!skip.Value) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1)); } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1)); } } if (skip.Value) { return(skip); } //if (!(sr.BaseStream is MemoryStream)) // ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1)); //if (Task != null) // return !IsStateNOTExistsOrNOTMatch(item); if (pair.Item2.IsNullOrWhiteSpace()) { if (!Configuration.IgnoreEmptyLine) { throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1)); } else { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1)); } return(true); } } if (commentTokens != null && commentTokens.Length > 0) { foreach (string comment in commentTokens) { if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal)) //, true, Configuration.Culture)) { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1)); } return(true); } } } if (!_configCheckDone) { Configuration.Validate(null); var dict = Configuration.KVPRecordFieldConfigurations.ToDictionary(i1 => i1.Name, i1 => i1.FieldType == null ? null : i1.FieldType); RaiseMembersDiscovered(dict); Configuration.UpdateFieldTypesIfAny(dict); _configCheckDone = true; } return(false); })) { while (true) { pairIn = e.Peek; if (!isRecordStartFound) { if (pairIn == null) { break; } lastLine = null; recLines.Clear(); isRecordEndFound = false; isRecordStartFound = true; if (!Configuration.RecordStart.IsNullOrWhiteSpace()) { //Move to record start while (!(Configuration.IsRecordStartMatch(e.Peek.Item2))) { e.MoveNext(); if (e.Peek == null) { break; } } } if (e.Peek != null) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Record start found at [{0}] line...".FormatString(e.Peek.Item1)); } e.MoveNext(); continue; } else { string recordEnd = !Configuration.RecordEnd.IsNullOrWhiteSpace() ? Configuration.RecordEnd : Configuration.RecordStart; if (!recordEnd.IsNullOrWhiteSpace()) { if (e.Peek == null) { if (Configuration.RecordEnd.IsNullOrWhiteSpace()) { isRecordEndFound = true; } else { break; } } else { //Move to record start if (Configuration.IsRecordEndMatch(e.Peek.Item2)) { isRecordEndFound = true; isRecordStartFound = false; if (e.Peek != null) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Record end found at [{0}] line...".FormatString(e.Peek.Item1)); } } } } else if (e.Peek == null) { isRecordEndFound = true; } if (!isHeaderFound) { //if (isRecordEndFound && headers.Count == 0) //{ // //throw new ChoParserException("Unexpected EOF found."); //} if (!isRecordEndFound) { e.MoveNext(); if (e.Peek != null) { //If line empty or line continuation, skip if (pairIn.Item2.IsNullOrWhiteSpace() || IsLineContinuationCharFound(pairIn.Item2)) //.Item2[0] == ' ' || pairIn.Item2[0] == '\t') { } else { string header = ToKVP(pairIn.Item1, pairIn.Item2).Key; if (!header.IsNullOrWhiteSpace()) { headers.Add(header); } } } } else { Configuration.Validate(headers.ToArray()); isHeaderFound = true; isRecordEndFound = false; IsHeaderLoaded = true; break; } } else { if (!IsHeaderLoaded) { Configuration.Validate(new string[] { }); IsHeaderLoaded = true; } if (isRecordEndFound && recLines.Count == 0) { //throw new ChoParserException("Unexpected EOF found."); } else if (!isRecordEndFound) { e.MoveNext(); if (e.Peek != null) { //If line empty or line continuation, skip if (pairIn.Item2.IsNullOrWhiteSpace()) { if (!Configuration.IgnoreEmptyLine) { throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pairIn.Item1)); } else { Tuple <long, string> t = new Tuple <long, string>(lastLine.Item1, lastLine.Item2 + Configuration.EOLDelimiter); recLines.RemoveAt(recLines.Count - 1); recLines.Add(t); } } else if (IsLineContinuationCharFound(pairIn.Item2)) //pairIn.Item2[0] == ' ' || pairIn.Item2[0] == '\t') { if (lastLine == null) { throw new ChoParserException("Unexpected line continuation found at {0} location.".FormatString(pairIn.Item1)); } else { Tuple <long, string> t = new Tuple <long, string>(lastLine.Item1, lastLine.Item2 + Configuration.EOLDelimiter + pairIn.Item2); recLines.RemoveAt(recLines.Count - 1); recLines.Add(t); } } else { lastLine = pairIn; recLines.Add(pairIn); } } } else { object rec = Configuration.IsDynamicObject ? new ChoDynamicObject(new Dictionary <string, object>(Configuration.FileHeaderConfiguration.StringComparer)) { ThrowExceptionIfPropNotExists = true, AlternativeKeys = Configuration.AlternativeKeys } : Activator.CreateInstance(RecordType); if (!LoadLines(new Tuple <long, List <Tuple <long, string> > >(++recNo, recLines), ref rec)) { yield break; } isRecordStartFound = false; //StoreState(e.Current, rec != null); if (!Configuration.RecordEnd.IsNullOrWhiteSpace()) { e.MoveNext(); } if (rec == null) { continue; } yield return(rec); if (Configuration.NotifyAfter > 0 && recNo % Configuration.NotifyAfter == 0) { if (RaisedRowsLoaded(recNo)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested."); abortRequested = true; yield break; } } if (e.Peek == null) { break; } } } } } } } if (!abortRequested) { RaisedRowsLoaded(recNo, true); } RaiseEndLoad(sr); }
public override IEnumerable <object> WriteTo(object writer, IEnumerable <object> records, Func <object, bool> predicate = null) { TextWriter sw = writer as TextWriter; ChoGuard.ArgumentNotNull(sw, "TextWriter"); if (records == null) { yield break; } if (!RaiseBeginWrite(sw)) { yield break; } CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture; System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture; string recText = String.Empty; long recCount = 0; var recEnum = records.GetEnumerator(); try { object notNullRecord = GetFirstNotNullRecord(recEnum); if (notNullRecord == null) { yield break; } if (Configuration.IsDynamicObject) { if (Configuration.MaxScanRows > 0) { List <string> fns = new List <string>(); foreach (object record1 in GetRecords(recEnum)) { recCount++; if (record1 != null) { if (recCount <= Configuration.MaxScanRows) { if (!record1.GetType().IsDynamicType()) { throw new ChoParserException("Invalid record found."); } _recBuffer.Value.Add(record1); fns = fns.Union(GetFields(record1)).ToList(); if (recCount == Configuration.MaxScanRows) { Configuration.Validate(fns.ToArray()); WriteHeaderLine(sw); _configCheckDone = true; break; } } } } } } foreach (object record in GetRecords(recEnum)) { _index++; if (TraceSwitch.TraceVerbose) { if (record is IChoETLNameableObject) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name)); } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index)); } } recText = String.Empty; if (record != null) { if (predicate == null || predicate(record)) { //Discover and load FixedLength columns from first record if (!_configCheckDone) { if (notNullRecord != null) { string[] fieldNames = GetFields(notNullRecord); Configuration.Validate(fieldNames); WriteHeaderLine(sw); _configCheckDone = true; } } //Check record if (record != null) { Type rt = record.GetType().ResolveType(); if (Configuration.IsDynamicObject) { if (ElementType != null) { } else if (!rt.IsDynamicType()) { throw new ChoWriterException("Invalid record found."); } } else { if (rt != Configuration.RecordType) { throw new ChoWriterException("Invalid record found."); } } } if (!RaiseBeforeRecordWrite(record, _index, ref recText)) { yield break; } if (recText == null) { continue; } else if (recText.Length > 0) { sw.Write("{1}{0}", recText, _hadHeaderWritten ? Configuration.EOLDelimiter : ""); continue; } try { if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel) { record.DoObjectLevelValidation(Configuration, Configuration.FixedLengthRecordFieldConfigurations); } if (ToText(_index, record, out recText)) { if (_index == 1) { sw.Write("{1}{0}", recText, _hadHeaderWritten ? Configuration.EOLDelimiter : ""); } else { sw.Write("{1}{0}", recText, Configuration.EOLDelimiter); } if (!RaiseAfterRecordWrite(record, _index, recText)) { yield break; } } } //catch (ChoParserException) //{ // throw; //} catch (Exception ex) { ChoETLFramework.HandleException(ref ex); if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); } else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue) { if (!RaiseRecordWriteError(record, _index, recText, ex)) { throw; } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); } } else { throw; } } } } yield return(record); if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0) { if (RaisedRowsWritten(_index)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested."); yield break; } } } } finally { System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo; } RaiseEndWrite(sw); }
internal ChoBSONRecordFieldConfigurationMap(ChoBSONRecordFieldConfiguration config) { ChoGuard.ArgumentNotNull(config, nameof(config)); _config = config; }
public static IEnumerable <string> ReadLines(this TextReader reader, string EOLDelimiter = null, char quoteChar = ChoCharEx.NUL, bool mayContainEOLInData = false, int maxLineSize = 32768) { ChoGuard.ArgumentNotNull(reader, "TextReader"); EOLDelimiter = EOLDelimiter ?? Environment.NewLine; if (!mayContainEOLInData) { if (EOLDelimiter == Environment.NewLine || (EOLDelimiter.Length == 1 && EOLDelimiter[0] == '\r') || (EOLDelimiter.Length == 1 && EOLDelimiter[0] == '\n') ) { string line; while ((line = reader.ReadLine()) != null) { yield return(line); } } else { string line; while ((line = ReadLineWithFixedNewlineDelimeter(reader as StreamReader, EOLDelimiter, maxLineSize)) != null) { yield return(line); } } } bool isPrevEscape = false; bool inQuote = false; List <char> buffer = new List <char>(); char c = ChoCharEx.NUL; CircularBuffer <char> delim_buffer = new CircularBuffer <char>(EOLDelimiter.Length); while (reader.Peek() >= 0) { isPrevEscape = c == ChoCharEx.Backslash; c = (char)reader.Read(); delim_buffer.Enqueue(c); if (quoteChar != ChoCharEx.NUL && quoteChar == c) { if (!isPrevEscape) { inQuote = !inQuote; } else if (reader.Peek() == quoteChar) { inQuote = false; } } if (!inQuote) { if (delim_buffer.ToString() == EOLDelimiter) { if (buffer.Count > 0) { string x = new String(buffer.ToArray()); yield return(x.Substring(0, x.Length - (EOLDelimiter.Length - 1))); buffer.Clear(); } continue; } } buffer.Add(c); if (buffer.Count > maxLineSize) { throw new ApplicationException("Large line found. Check and correct the end of line delimiter."); } } if (buffer.Count > 0) { yield return(new String(buffer.ToArray())); } else { yield break; } }
private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null) { TraceSwitch = traceSwitch; TextReader sr = source as TextReader; ChoGuard.ArgumentNotNull(sr, "TextReader"); if (sr is StreamReader) { ((StreamReader)sr).Seek(0, SeekOrigin.Begin); } if (Configuration.RecordSelector == null) { throw new ChoRecordConfigurationException("Missing record selector."); } if (!RaiseBeginLoad(sr)) { yield break; } string[] commentTokens = Configuration.Comments; bool? skip = false; bool _headerFound = false; bool? skipUntil = true; bool? doWhile = true; using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >( new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, Configuration.MayContainEOLInData, Configuration.MaxLineSize)).ToEnumerable(), (pair) => { //bool isStateAvail = IsStateAvail(); skip = false; if (skipUntil != null) { if (skipUntil.Value) { skipUntil = RaiseSkipUntil(pair); if (skipUntil == null) { } else { skip = skipUntil.Value; } } } //if (isStateAvail) //{ // if (!IsStateMatches(item)) // { // skip = filterFunc != null ? filterFunc(item) : false; // } // else // skip = true; //} //else // skip = filterFunc != null ? filterFunc(item) : false; if (skip == null) { return(null); } if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine); if (!skip.Value) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1)); } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1)); } } if (skip.Value) { return(skip); } //if (!(sr.BaseStream is MemoryStream)) // ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1)); //if (Task != null) // return !IsStateNOTExistsOrNOTMatch(item); if (pair.Item2.IsNullOrWhiteSpace()) { if (!Configuration.IgnoreEmptyLine) { throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1)); } else { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1)); } return(true); } } if (commentTokens != null && commentTokens.Length > 0) { foreach (string comment in commentTokens) { if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal)) //, true, Configuration.Culture)) { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1)); } return(true); } } } if (!_configCheckDone) { Configuration.Validate(pair); // GetHeaders(pair.Item2)); _configCheckDone = true; } //Ignore Header if any if (Configuration.FileHeaderConfiguration.HasHeaderRecord && !_headerFound) { if (TraceSwitch.TraceVerbose) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring header line at [{0}]...".FormatString(pair.Item1)); } _headerFound = true; return(true); } return(false); })) { while (true) { Tuple <long, string> pair = e.Peek; if (pair == null) { RaiseEndLoad(sr); yield break; } Type recType = Configuration.RecordSelector(pair.Item2); if (recType == null) { if (Configuration.IgnoreIfNoRecordParserExists) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, $"No record type found for [{pair.Item1}] line to parse."); } else { throw new ChoParserException($"No record type found for [{pair.Item1}] line to parse."); } } object rec = ChoActivator.CreateInstance(recType); if (!LoadLine(pair, ref rec)) { yield break; } //StoreState(e.Current, rec != null); e.MoveNext(); if (rec == null) { continue; } yield return(rec); if (Configuration.NotifyAfter > 0 && pair.Item1 % Configuration.NotifyAfter == 0) { if (RaisedRowsLoaded(pair.Item1)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested."); yield break; } } if (doWhile != null) { doWhile = RaiseDoWhile(pair); if (doWhile != null && doWhile.Value) { break; } } } } }