public void TestLexer3() { string[] expected = { "1", "25", "A", "26", "10", "ZD", "D" }; var lexer = new Lexer("1,25,A,26,10,ZD,D"); var results = new List<string>(); while (lexer.MoveNext()) { results.Add(lexer.Current); } Assert.IsTrue(results.SequenceEqual(expected)); }
public void TestLexer4() { string[] expected = { "(", "C", "'srchfor ''", "1", "7", "C", "'''", "80:X", ")" }; var lexer = new Lexer("(C'srchfor ''',1,7,C'''',80:X)"); var results = new List<string>(); while (lexer.MoveNext()) { results.Add(lexer.Current); } Assert.IsTrue(results.SequenceEqual(expected)); }
// Parses a single outfil configuration. private LegacyOutputFile ParseOutfil(Lexer lexer) { LegacyOutputFile outputFile = null; if (lexer.MoveNext()) { outputFile = new LegacyOutputFile(); string outrec = null; string include = null; string omit = null; while (lexer.Current != null) { // Look for the supported commands if (string.Equals(Outrec, lexer.Current, StringComparison.InvariantCultureIgnoreCase)) { lexer.MoveNext(); outrec = ParseCommandParameters(lexer); } else if (string.Equals(Include, lexer.Current, StringComparison.InvariantCultureIgnoreCase)) { lexer.MoveNext(); include = ParseCommandParameters(lexer); } else if (string.Equals(Omit, lexer.Current, StringComparison.InvariantCultureIgnoreCase)) { lexer.MoveNext(); omit = ParseCommandParameters(lexer); } else if (lexer.Current != SemiColon) { throw new ParsingException(string.Format("Unexpected token at index {0}: {1}", lexer.Index, lexer.Current)); } else { break; } } // Set the required properties of the output file if (!string.IsNullOrEmpty(outrec)) { var formatterParser = new FormatterParser { Encoding = Encoding }; outputFile.Formatter = formatterParser.GetFormatter(outrec); } if (!string.IsNullOrWhiteSpace(include) || !string.IsNullOrWhiteSpace(omit)) { var filterParser = new FilterParser { Encoding = Encoding, SortEncoding = SortEncoding }; outputFile.Filter = filterParser.GetFilter(include, omit); } } return outputFile; }
/// <summary> /// Parses an accessor. /// </summary> /// <param name="lexer">the lexer to read the tokens from</param> /// <param name="defaultFormat">the default format</param> /// <returns>the parsed <see cref="IAccessor{T}"/></returns> private IAccessor<decimal> ParseAccessor(Lexer lexer, string defaultFormat) { var start = lexer.ParseInt() - 1; var length = lexer.ParseInt(); var format = defaultFormat; if (Formats.Contains(lexer.Current)) { format = lexer.Parse(); } return (IAccessor<decimal>) GetAccessor(start, length, format, Encoding); }
/// <summary> /// Parses the outfil configurations and computes a list of <see cref="IOutputFile{T}"/>. /// </summary> /// <param name="outfils">The outfil configurations, separated by commas.</param> /// <returns></returns> public IList<IOutputFile<byte[]>> GetOutputFiles(string outfils) { var outputFiles = new List<IOutputFile<byte[]>>(); var lexer = new Lexer(outfils); var outputFile = ParseOutfil(lexer); while (outputFile != null) { outputFiles.Add(outputFile); outputFile = ParseOutfil(lexer); } return outputFiles; }
/// <summary> /// Creates a lexer from a regex and a configuration card. The regex is used to get the default format /// which is then written in <paramref name="defaultFormat"/>. /// </summary> /// <param name="formatRegex">a regex that can extract the card and the default format</param> /// <param name="configuration">the configuration card</param> /// <param name="defaultFormat">a variable where the default format will be stored</param> /// <returns>a new <see cref="Lexer"/></returns> protected Lexer GetLexer(Regex formatRegex, string configuration, out string defaultFormat) { Lexer lexer; defaultFormat = null; var matches = formatRegex.Matches(configuration); if (matches.Count == 1) { var groups = matches[0].Groups; lexer = new Lexer(groups[Card].Value); if (groups[Format1].Value != string.Empty) { defaultFormat = groups[Format1].Value; } if (groups[Format2].Value != string.Empty) { defaultFormat = groups[Format2].Value; } } else { lexer = new Lexer(configuration); } return lexer; }
// Retrieves the parameter of a command so that it can be parsed by a specific parser. private static string ParseCommandParameters(Lexer lexer) { var start = lexer.Index; lexer.Parse(OpeningPar); while (lexer.MoveNext() && lexer.Current != ClosingPar) { } var length = lexer.Index - start - 1; lexer.Parse(ClosingPar); return lexer.SubString(start, length); }
/// <summary> /// Parses an accessor. /// </summary> /// <param name="lexer">the lexer to read the tokens from</param> /// <param name="defaultFormat">the default format</param> /// <param name="format">the format of the parsed accessor</param> /// <returns>an <see cref="IAccessor{T}"/></returns> private object ParseAccessor(Lexer lexer, string defaultFormat, out string format) { var elements = new List<string>(); format = null; do { elements.Add(lexer.Current); } while (lexer.MoveNext() && lexer.Current != ClosingPar && !LogicalOperators.Contains(lexer.Current) && !ComparisonOperators.Contains(lexer.Current)); if (elements.Count == 1) { // decimal constant return new ConstantAccessor<decimal> { Constant = decimal.Parse(elements[0]) }; } if (elements.Count == 2 && elements[0] == "C") { // string constant var constant = elements[1].Substring(1, elements[1].Length - 2); return new ConstantAccessor<string> { Constant = constant }; } if (elements.Count == 2 && elements[0] == "X") { // HexaDecimal constant var constant = elements[1].Substring(1, elements[1].Length - 2); BinaryAccessor accessor = new BinaryAccessor (); int value = Convert.ToInt32(constant, 16); byte[] bytes = Enumerable.Range(0, constant.Length).Where(x => x % 2 == 0).Select(x => Convert.ToByte(constant.Substring(x, 2), 16)).ToArray(); accessor.SetBytes(bytes, bytes, 0); decimal constant1 = Convert.ToInt32(constant, 16); return new ConstantAccessor<decimal> { Constant = constant1 }; } // field accesor var start = int.Parse(elements[0]) - 1; var length = int.Parse(elements[1]); format = elements.Count == 3 ? elements[2] : defaultFormat; return GetAccessor(start, length, format, Encoding); }
/// <summary> /// Parses an <see cref="IFilter{T}"/>. /// </summary> /// <param name="lexer">the lexer to read the tokens from</param> /// <param name="defaultFormat">the default format</param> /// <returns>an <see cref="IFilter{T}"/></returns> private IFilter<byte[]> ParseFilter(Lexer lexer, string defaultFormat) { IFilter<byte[]> filter = null; if (lexer.MoveNext()) { if (lexer.Current == OpeningPar) { filter = ParseFilter(lexer, defaultFormat); lexer.Parse(ClosingPar); } else { string format1; string format2; var leftAccessor = ParseAccessor(lexer, defaultFormat, out format1); var op = lexer.Parse(); var rightAccessor = ParseAccessor(lexer, defaultFormat, out format2); var format = format1 ?? format2; filter = GetFilter(leftAccessor, rightAccessor, format, op); } if (lexer.Current == Or) { var disjunction = new DisjunctionFilter<byte[]> { Filters = new List<IFilter<byte[]>> { filter, ParseFilter(lexer, defaultFormat) } }; filter = disjunction; } else if (lexer.Current == And) { var conjunction = new ConjunctionFilter<byte[]> { Filters = new List<IFilter<byte[]>> { filter, ParseFilter(lexer, defaultFormat) } }; filter = conjunction; } } return filter; }
public string ExtractElement(Lexer lexer, string Info, int recordLength) { string Content = ""; int currentLength = 0; while (Info != ClosingPar) { var parentheses = Info == (OpeningPar); if (parentheses) { Info = lexer.Parse(); } var matches = headerRegex.Matches(Info); if (matches.Count == 1) { string[] tokens = Info.Split(':'); if (tokens.Length == 2) { int lengthOfField = Convert.ToInt16(tokens[0]); String temp = tokens[1].PadLeft(lengthOfField - currentLength, space); Content += temp; currentLength += temp.Length; } } else if (Info.Contains("/") && string.IsNullOrWhiteSpace(Info.Replace("/", ""))) { if (recordLength == 0) { Content += Info.Replace("/", System.Environment.NewLine); } else { if (currentLength < recordLength) { String x = "".PadRight(recordLength - currentLength, space); Content += x; } } currentLength = 0; } else { String temp = Info.Replace("'", ""); Content += temp; currentLength += temp.Length; } Info = lexer.Parse(); if (string.IsNullOrWhiteSpace(Info)) { break; } } return Content; }
/// <summary> /// Parses an accessor. /// </summary> /// <param name="lexer">the lexer to read the tokens from</param> /// <param name="defaultFormat">the default format</param> /// <param name="format">the format of the parsed accessor</param> /// <returns>an <see cref="IAccessor{T}"/></returns> private object ParseAccessor(Lexer lexer, string defaultFormat, out string format) { var elements = new List<string>(); format = null; do { elements.Add(lexer.Current); } while (lexer.MoveNext() && lexer.Current != ClosingPar && !LogicalOperators.Contains(lexer.Current) && !ComparisonOperators.Contains(lexer.Current)); if (elements.Count == 1) { // decimal constant return new ConstantAccessor<decimal> { Constant = decimal.Parse(elements[0]) }; } if (elements.Count == 2 && elements[0] == "C") { // string constant var constant = elements[1].Substring(1, elements[1].Length - 2); return new ConstantAccessor<string> { Constant = constant }; } // field accesor var start = int.Parse(elements[0]) - 1; var length = int.Parse(elements[1]); format = elements.Count == 3 ? elements[2] : defaultFormat; return GetAccessor(start, length, format, Encoding); }
/// <summary> /// Parse all the individual comparers and combine them in a <see cref="ComparerChain{T}"/>. /// </summary> /// <param name="lexer">the lexer to read the tokens from</param> /// <param name="defaultFormat">the default format</param> /// <returns> /// a single <see cref="IComparer{T}"/> if there is only one comparer, or a <see cref="ComparerChain{T}"/> /// if there are more than one comparer. /// </returns> private IComparer<byte[]> ParseComparers(Lexer lexer, string defaultFormat) { IList<IComparer<byte[]>> comparers = new List<IComparer<byte[]>>(); do { comparers.Add(ParseComparer(lexer, defaultFormat)); } while (lexer.MoveNext() && lexer.Current != ClosingPar); if (comparers.Count == 0) { return null; } return comparers.Count > 1 ? new ComparerChain<byte[]> { Comparers = comparers } : comparers[0]; }
/// <summary> /// Parses an individual comparer. /// </summary> /// <param name="lexer">the lexer to read the tokens from</param> /// <param name="defaultFormat">the default format</param> /// <returns>a single <see cref="IComparer{T}"/></returns> private IComparer<byte[]> ParseComparer(Lexer lexer, string defaultFormat) { var start = lexer.ParseInt() - 1; var length = lexer.ParseInt(); string format; if (lexer.Current == AscendingOrder || lexer.Current == DescendingOrder) { // if we have the order token no format is specified, thus use the default one format = defaultFormat; } else { // otherwise the current token is the format format = lexer.Parse(); } var ascending = lexer.Current == AscendingOrder; IComparer<byte[]> comparer; if (format == StringFormat) { comparer = new StringComparer { Encoding = Encoding, SortEncoding = SortEncoding, Ascending = ascending, Start = start, Length = length }; } else { var accessor = GetAccessor(start, length, format, Encoding) as IAccessor<decimal>; comparer = new DefaultComparer<decimal> { Ascending = ascending, Accessor = accessor }; } return comparer; }