// ------------------ ScanWord_IsolatedWord_CommentToEnd -------------------
private static ScanPatternResults ScanWord_IsolateWord_CommentToEnd(
BoundedString InBoundedString,
int InWordBx,
ref WordCursor InOutResults, TextTraits InTraits)
{
string wordText;
ScanPatternResults spr = null;
// look for end of comment. ( either end of line or end of string )
int fx = Scanner.ScanEqual(InBoundedString, InWordBx, Environment.NewLine).ResultPos;
if (fx >= 0)
{
int Lx = fx - InWordBx;
wordText = InBoundedString.Substring(InWordBx, Lx);
ScanPattern pat =
InTraits.NonWordPatterns.FindPatternAtSubstring(InBoundedString, fx);
spr = new ScanPatternResults(fx, pat);
}
else
{
wordText = InBoundedString.Substring(InWordBx);
spr = new ScanPatternResults(-1);
}
// store info on the word found in the return WordCursor argument.
InOutResults.SetWord(wordText, WordClassification.CommentToEnd, InWordBx);
// return value of method contains info on the word delim.
return(spr);
}
public ScanPattern GetPathPartDelim(string Text, int Ix)
{
ScanPattern pat = null;
int matchLx = 0;
if (Ix > (Text.Length - 1))
{
}
else if (IsPathSepChar(Text[Ix]) == true)
{
var rv = NonWordPatterns.MatchPatternsAtStringLocation(Text, Ix, Text.Length - 1);
pat = rv.Item1;
matchLx = rv.Item2;
// the nonword is a path sep char. but is it a pathPart delim?
// It is if there is a word char before or after the path sep char.
// ex: /abc/efg vs. a = b / c ;
if ((IsWordChar(Text, Ix - 1) == false) &&
(IsWordChar(Text, Ix + 1) == false))
{
pat = null;
}
}
return(pat);
}
/// <summary>
/// calc if the char at location in string is a word char or not ( is not a
/// NonWordChar )
/// </summary>
/// <param name="InBoundedString"></param>
/// <param name="InIx"></param>
/// <returns></returns>
public bool IsWordChar(
string Text, int Ix)
{
bool isWordChar = false;
if (Ix > (Text.Length - 1))
{
isWordChar = false;
}
else
{
ScanPattern pat = NonWordPatterns.FindPatternAtSubstring(
Text, Ix, Text.Length - 1);
if (pat == null)
{
isWordChar = true;
}
else
{
isWordChar = false;
}
}
return(isWordChar);
}
/// <summary>
/// calc if the char at location in string is a word char or not ( is not a
/// NonWordChar )
/// </summary>
/// <param name="InBoundedString"></param>
/// <param name="InIx"></param>
/// <returns></returns>
public bool IsWordChar(
BoundedString InBoundedString, int InIx)
{
bool isWordChar = false;
if (InBoundedString.IsOutsideBounds(InIx))
{
isWordChar = false;
}
else
{
ScanPattern pat = NonWordPatterns.FindPatternAtSubstring(
InBoundedString, InIx);
if (pat == null)
{
isWordChar = true;
}
else
{
isWordChar = false;
}
}
return(isWordChar);
}
/// <summary>
/// return the NonWord info of path sep char which is classified as being a
/// path part delimiter ( it could be a division expression symbol )
/// </summary>
/// <param name="InBoundedString"></param>
/// <param name="InIx"></param>
/// <returns></returns>
public ScanPattern GetPathPartDelim(BoundedString InBoundedString, int InIx)
{
ScanPattern pat = null;
int matchLx = 0;
if (InBoundedString.IsOutsideBounds(InIx))
{
}
else if (IsPathSepChar(InBoundedString[InIx]) == true)
{
var rv = NonWordPatterns.MatchPatternsAtStringLocation(
InBoundedString.String, InIx, InBoundedString.Ex);
pat = rv.Item1;
matchLx = rv.Item2;
// the nonword is a path sep char. but is it a pathPart delim?
// It is if there is a word char before or after the path sep char.
// ex: /abc/efg vs. a = b / c ;
if ((IsWordChar(InBoundedString, InIx - 1) == false) &&
(IsWordChar(InBoundedString, InIx + 1) == false))
{
pat = null;
}
}
return(pat);
}
// ------------------ ScanWord_IsolatedWord_CommentToEnd -------------------
public static ScanPatternResults ScanWord_IsolateWord_CommentToEnd(
string Text,
int WordBx,
ref WordCursor Results, TextTraits Traits)
{
string wordText;
ScanPatternResults spr = null;
// look for end of comment. ( either end of line or end of string )
int fx = Text.IndexOf(Environment.NewLine, WordBx);
if (fx >= 0)
{
int Lx = fx - WordBx;
wordText = Text.Substring(WordBx, Lx);
ScanPattern pat =
Traits.NonWordPatterns.FindPatternAtSubstring(Text, fx, 2);
spr = new ScanPatternResults(fx, pat);
}
else
{
wordText = Text.Substring(WordBx);
spr = new ScanPatternResults(-1);
}
// store info on the word found in the return WordCursor argument.
Results.SetWord(wordText, WordClassification.CommentToEnd, WordBx);
// return value of method contains info on the word delim.
return(spr);
}
public bool IsNonWordPattern(ScanPattern Pattern)
{
if (this.DelimPatternsThatAreNonWords.Contains(Pattern))
{
return(true);
}
else
{
return(false);
}
}
public bool IsWhitespace(ScanPattern Pattern)
{
if (this.WhitespacePatterns.Contains(Pattern))
{
return(true);
}
else
{
return(false);
}
}
public void Test()
{
var lParen = new Symbol("(");
var rParen = new Symbol(")");
var num = new Symbol("NUM");
var ident = new Symbol("ID");
var qStr = new Symbol("QSTR");
var grammar = new Grammar
{
Symbols =
{
lParen,
rParen,
num,
ident,
qStr
},
Conditions =
{
new Condition("main")
{
Matchers =
{
new Matcher(@"blank+"),
new Matcher(@"digit+ ('.' digit+)? | '.' digit+", num),
new Matcher(
@"(alpha | [:.!@#$%^&|?*/+*=\\_-]) (alnum | [:.!@#$%^&|?*/+*=\\_-])*",
ident),
new Matcher("'\"' ('\\\\\"' | ~'\"')* '\"'", qStr),
new Matcher(ScanPattern.CreateLiteral("("), lParen),
new Matcher(ScanPattern.CreateLiteral(")"), rParen),
}
}
}
};
var target = new DfaSimulationLexer(
" (1 (\"bar\" +))",
ScannerDescriptor.FromScanRules(grammar.Conditions[0].Matchers, ExceptionLogging.Instance));
var collector = new Collector <Msg>();
target.Accept(collector);
Assert.AreEqual(
new int[] { lParen.Index, num.Index, lParen.Index, qStr.Index, ident.Index, rParen.Index, rParen.Index },
collector.Select(msg => msg.Id).ToArray());
Assert.AreEqual(
new object[] { "(", "1", "(", "\"bar\"", "+", ")", ")" },
collector.Select(msg => msg.Value).ToArray());
}
// --------------------------- ScanReverseNotEqual-------------------------------
// scan reverse for the first location in string not equal to all of the
// pattern characters.
public static int ScanReverseNotEqual(
string InString, int InBx, int InEx,
int InIx, ScanPatterns InNotEqualPatterns)
{
int foundIx = InIx + 1;
while (true)
{
foundIx -= 1;
if (foundIx < InBx)
{
foundIx = -1;
break;
}
int remLx = InEx - foundIx + 1;
char ch1 = InString[foundIx];
// the current char is not equal any of the pattern lead chars.
int patIx = Array.IndexOf <char>(InNotEqualPatterns.LeadChars, ch1);
if (patIx == -1)
{
break;
}
// lead char matches. check the entire pattern string for equality.
ScanPattern equalPat = null;
ScanPattern pat = InNotEqualPatterns.ScanPatternsArray[patIx];
while (pat != null)
{
bool rv = Stringer.CompareSubstringEqual(InString, foundIx, InEx, pat.PatternValue);
if (rv == true)
{
equalPat = pat;
break;
}
pat = pat.NextSameLeadChar;
}
// none of the patterns fully match the substring at the current location.
if (equalPat == null)
{
break;
}
}
return(foundIx);
}
// -------------------------------- ScanNotEqual ------------------------
public static int ScanNotEqual(
string Text, ScanPatterns Patterns, int Start)
{
// step thru the string 1 char at a time.
int ix = Start;
int ex = Text.Length - 1;
while (true)
{
if (ix > ex)
{
ix = -1;
break;
}
char ch1 = Text[ix];
// the current char is not equal any of the pattern lead chars.
int fx = Array.IndexOf <char>(Patterns.LeadChars, ch1);
if (fx == -1)
{
break;
}
ScanPattern equalPat = null;
int remLx = ex - ix + 1;
foreach (var pat in Patterns)
{
if (pat.PatternValue.Length <= remLx)
{
if (pat.PatternValue == Text.Substring(ix, pat.PatternValue.Length))
{
equalPat = pat;
break;
}
}
}
// text at the current location is not equal any of the patterns.
if (equalPat == null)
{
break;
}
ix += equalPat.Length;
}
return(ix);
}
private static CilMatcher CreateImplicitLiteralMatcher(string literal)
{
var outcome = CilSymbolRef.Create(literal);
// Generate implicit scan rule for the keyword
var result = new CilMatcher
{
Context = CilContextRef.None,
MainOutcome = outcome,
AllOutcomes = { outcome },
Disambiguation = Disambiguation.Exclusive,
Pattern = ScanPattern.CreateLiteral(literal),
ActionBuilder = code => code
.Emit(il => il.Ldnull())
.ReturnFromAction()
};
return(result);
}
/// <summary>
/// Scan string for any of the pattern strings in ScanPatterns.
/// </summary>
/// <param name="InString"></param>
/// <param name="InBx"></param>
/// <param name="InLx"></param>
/// <param name="InPatterns"></param>
/// <returns></returns>
public static ScanPatternResults ScanEqualAny(
string InString, int InIx, int InLx, ScanPatterns InPatterns)
{
ScanPattern pat = null;
ScanPatternResults spr = null;
int ix = InIx;
int ex = InIx + InLx - 1;
while (true)
{
spr = null;
int remLx = ex - ix + 1;
if (remLx <= 0)
{
break;
}
ScanCharResults scr = ScanEqualAny(
InString, ix, remLx, InPatterns.LeadChars);
if (scr.IsNotFound == true)
{
spr = new ScanPatternResults(-1);
break;
}
pat = InPatterns.FindPatternAtSubstring(InString, scr.ResultPos, ex);
if (pat != null)
{
spr = new ScanPatternResults(scr.ResultPos, pat);
break;
}
// advance ix to resume scan after the found lead char.
ix = scr.ResultPos + 1;
}
return(spr);
}
public ScanPattern GetPathPartDelim(string Text, int Ix)
{
ScanPattern pat = null;
if (Ix > (Text.Length - 1))
{
}
else if (IsPathSepChar(Text[Ix]) == true)
{
pat = NonWordPatterns.FindPatternAtSubstring(
Text, Ix, Text.Length - 1);
// the nonword is a path sep char. but is it a pathPart delim?
// It is if there is a word char before or after the path sep char.
// ex: /abc/efg vs. a = b / c ;
if ((IsWordChar(Text, Ix - 1) == false) &&
(IsWordChar(Text, Ix + 1) == false))
{
pat = null;
}
}
return(pat);
}
/// <summary>
/// return the NonWord info of path sep char which is classified as being a
/// path part delimiter ( it could be a division expression symbol )
/// </summary>
/// <param name="InBoundedString"></param>
/// <param name="InIx"></param>
/// <returns></returns>
public ScanPattern GetPathPartDelim(BoundedString InBoundedString, int InIx)
{
ScanPattern pat = null;
if (InBoundedString.IsOutsideBounds(InIx))
{
}
else if (IsPathSepChar(InBoundedString[InIx]) == true)
{
pat = NonWordPatterns.FindPatternAtSubstring(
InBoundedString.String, InIx, InBoundedString.Ex);
// the nonword is a path sep char. but is it a pathPart delim?
// It is if there is a word char before or after the path sep char.
// ex: /abc/efg vs. a = b / c ;
if ((IsWordChar(InBoundedString, InIx - 1) == false) &&
(IsWordChar(InBoundedString, InIx + 1) == false))
{
pat = null;
}
}
return(pat);
}
public static Tuple <ScanPattern, int> ScanEqualAny(
string Text, int Start, ScanPatterns Patterns)
{
ScanPattern pat = null;
int ix = Start;
while (true)
{
int remLx = Text.Length - ix;
if (remLx <= 0)
{
ix = -1;
break;
}
ScanCharResults scr = ScanEqualAny(
Text, ix, remLx, Patterns.LeadChars);
if (scr.IsNotFound == true)
{
ix = -1;
break;
}
ix = scr.ResultPos;
pat = Patterns.FindPatternAtSubstring(Text, ix, Text.Length - 1);
if (pat != null)
{
break;
}
// advance ix to resume scan after the found lead char.
ix = ix + 1;
}
return(new Tuple <ScanPattern, int>(pat, ix));
}
public void Test()
{
var num = new Symbol("NUM");
var ident = new Symbol("ID");
var qStr = new Symbol("QSTR");
var begin = new Symbol("begin");
var end = new Symbol("end");
var assign = new Symbol(":=");
var grammar = new Grammar
{
Symbols =
{
num,
ident,
qStr,
begin,
end,
assign
},
Conditions =
{
new Condition("main")
{
Matchers =
{
new Matcher(
@"digit+ ('.' digit+)? | '.' digit+",
num),
new Matcher(
@"[01234567]+ 'Q'",
num),
new Matcher(
@"alpha alnum*",
ident),
new Matcher(
@"
quot
~(quot | esc)*
(esc . ~(quot | esc)* )*
quot
",
qStr),
new Matcher(
ScanPattern.CreateLiteral("begin"),
begin),
new Matcher(
ScanPattern.CreateLiteral("end"),
end),
new Matcher(
ScanPattern.CreateLiteral(":="),
assign),
new Matcher(
@"blank+"),
}
}
}
};
var target = new TdfaSimulationLexer(
"b:=10Q \"foo\"",
ScannerDescriptor.FromScanRules(grammar.Conditions[0].Matchers, ExceptionLogging.Instance));
var collector = new Collector<Msg>();
target.Accept(collector);
Assert.AreEqual(
new int[] { ident.Index, assign.Index, num.Index, qStr.Index },
collector.Select(msg => msg.Id).ToArray());
Assert.AreEqual(
new object[] { "b", ":=", "10Q", "\"foo\"" },
collector.Select(msg => msg.Value).ToArray());
}
/// <summary>
/// The delim after the word is whitspace. If what follows the whitespace
/// is a delim char, then this whitspace is disregarded as the delim, and
/// the delim is what follows the whitespace.
/// </summary>
/// <param name="InBoundedString"></param>
/// <param name="InNonWordResults"></param>
/// <param name="InOutResults"></param>
/// <param name="InTraits"></param>
private static void ScanWord_IsolateDelim_WhitespaceFollows(
BoundedString InBoundedString,
ScanPatternResults InPatternResults,
ref WordCursor InOutResults,
TextTraits InTraits)
{
InOutResults.WhitespaceFollowsWord = true;
ScanPattern nwPat = null;
// Look for hard delim after the ws.
ScanPatternResults scanResults =
ScanNotEqual(
InBoundedString.String, InPatternResults.FoundPos,
InBoundedString.Ex, InTraits.WhitespacePatterns);
// look that the char after the ws is a nonword.
if (scanResults.FoundPos != -1)
{
nwPat = InTraits.NonWordPatterns.FindPatternAtSubstring(
InBoundedString, scanResults.FoundPos);
}
// the char after the whitespace is a non word (delim) char.
if (nwPat != null)
{
DelimClassification nwdc = nwPat.DelimClassification;
// is the delim actually a sep char in a path name.
// so the delim is the whitespace.
if (InTraits.IsPathPartDelim(InBoundedString, scanResults.FoundPos))
{
ScanWord_IsolateDelim_SetDelimIsWhitespace(
InBoundedString, InTraits, InOutResults, InPatternResults.FoundPos);
}
// is a content open brace char. delim stays as whitespace because
// content braces are considered standalone words.
else if (nwPat.DelimClassification.IsOpenBraced( ))
{
ScanWord_IsolateDelim_SetDelimIsWhitespace(
InBoundedString, InTraits, InOutResults, InPatternResults.FoundPos);
}
// is a quote char. the quoted string is considered a word.
else if (nwdc == DelimClassification.Quote)
{
ScanWord_IsolateDelim_SetDelimIsWhitespace(
InBoundedString, InTraits, InOutResults, InPatternResults.FoundPos);
}
// is an actual delim.
else
{
InOutResults.SetDelim(
InBoundedString,
nwPat.PatternValue, scanResults.FoundPos, nwdc);
}
}
// the whitespace char is the delim of record.
else
{
ScanWord_IsolateDelim_SetDelimIsWhitespace(
InBoundedString, InTraits, InOutResults, InPatternResults.FoundPos);
}
}
public override IEnumerable <CilMatcher> GetMatchers()
{
var tokenType = Method.ReturnType;
var nextConditionType = GetNextConditionType();
var matcher = new CilMatcher();
var contextType = GetContextType();
//var context = CilContextRef.ByType(contextType);
if (tokenType != typeof(void))
{
var outcome = CilSymbolRef.Create(tokenType, LiteralText);
matcher.MainOutcome = outcome;
matcher.AllOutcomes.Add(outcome);
}
matcher.DefiningMethod = Method;
matcher.Disambiguation = Disambiguation;
if (LiteralText == null)
{
matcher.Pattern = ScanPattern.CreateRegular(Pattern, RegexPattern);
}
else
{
matcher.Pattern = ScanPattern.CreateLiteral(LiteralText);
}
var parameters = Method.GetParameters().ToList();
matcher.Context = GetContext();
matcher.NextConditionType = nextConditionType;
matcher.ActionBuilder =
code =>
{
ParameterInfo nextModeParameter;
if (parameters.Count != 0 && parameters.Last().IsOut)
{
nextModeParameter = parameters.Last();
parameters.RemoveAt(parameters.Count - 1);
}
else
{
nextModeParameter = null;
}
if (parameters.Count == 0)
{
}
else if (parameters.Count == 1 &&
parameters[0].ParameterType == typeof(string))
{
code.LdMatcherTokenString();
}
else if (parameters.Count == 3 &&
parameters[0].ParameterType == typeof(char[]) &&
parameters[1].ParameterType == typeof(int) &&
parameters[2].ParameterType == typeof(int))
{
code
.LdMatcherBuffer()
.LdMatcherStartIndex()
.LdMatcherLength();
}
else
{
throw new InvalidOperationException(
"Unsupported match-method signature: "
+ string.Join(", ", parameters.Select(p => p.ParameterType.Name)));
}
Ref <Locals> nextModeVar = null;
if (nextModeParameter != null)
{
code
.Emit(il =>
{
nextModeVar = il.Locals.Generate().GetRef();
return(il
.Local(nextModeVar.Def, il.Types.Object)
.Ldloca(nextModeVar));
});
}
code.Emit(il => il.Call(Method));
if (nextModeParameter != null)
{
code
.Emit(il => il.Ldloc(nextModeVar))
.ChangeCondition(nextConditionType);
}
if (Method.ReturnType == typeof(void))
{
code.SkipAction();
}
else
{
if (Method.ReturnType.IsValueType)
{
code.Emit(il => il.Box(il.Types.Import(Method.ReturnType)));
}
code
.ReturnFromAction()
;
}
return(code);
};
return(new[] { matcher });
}
// -------------------- ScanWord_IsolateDelim ---------------------------
private static void ScanWord_IsolateDelim(
BoundedString InBoundedString,
ScanPatternResults InPatternResults,
ref WordCursor InOutResults,
TextTraits InTraits)
{
// did not find a nonword char. must have hit end of string.
if (InPatternResults.IsNotFound)
{
InOutResults.DelimClass = DelimClassification.EndOfString;
}
// we have a delimiter of some kind.
else
{
DelimClassification sprdc = InPatternResults.FoundPat.DelimClassification;
InOutResults.WhitespaceFollowsWord = false;
InOutResults.WhitespaceFollowsDelim = false;
InOutResults.DelimIsWhitespace = false;
// the delim is a hard delim ( not whitespace )
if (sprdc != DelimClassification.Whitespace)
{
// Want the openContent brace to be processed as a standalone word. Use
// virtual whitespace so the word that this open brace is the delim of will
// have what appears to be a whitespace delim. Then the following word will
// be the standalone open content brace char.
if (sprdc == DelimClassification.OpenContentBraced)
{
InOutResults.SetDelim(
InBoundedString,
null, InPatternResults.FoundPos, DelimClassification.VirtualWhitespace);
}
else
{
// delim is either as classified in the collection of NonWords or is
// a PathPart delim.
ScanPattern pat = InTraits.GetPathPartDelim(
InBoundedString, InPatternResults.FoundPos);
if (pat != null)
{
InOutResults.SetDelim(
InBoundedString,
pat.PatternValue,
InPatternResults.FoundPos,
DelimClassification.PathSep);
}
else
{
InOutResults.SetDelim(
InBoundedString,
InPatternResults.FoundPat.PatternValue,
InPatternResults.FoundPos,
sprdc);
}
}
}
// whitespace immed follows the word text
else
{
ScanWord_IsolateDelim_WhitespaceFollows(
InBoundedString, InPatternResults, ref InOutResults, InTraits);
}
}
}
// -------------------------------- ScanNotEqual ------------------------
public static ScanPatternResults ScanNotEqual(
string InString, int InIx, int InEx, ScanPatterns InScanPatterns)
{
// step thru the string 1 char at a time.
int stringIx = InIx;
while (true)
{
if (stringIx > InEx)
{
stringIx = -1;
break;
}
char ch1 = InString[stringIx];
// the current char is not equal any of the pattern lead chars.
int patIx = Array.IndexOf <char>(InScanPatterns.LeadChars, ch1);
if (patIx == -1)
{
break;
}
ScanPattern equalPat = null;
ScanPattern pat = InScanPatterns.ScanPatternsArray[patIx];
while (pat != null)
{
bool rv = Stringer.CompareSubstringEqual(InString, stringIx, InEx, pat.PatternValue);
if (rv == true)
{
if (equalPat == null)
{
equalPat = pat;
}
// Matching pattern already found, but this pattern also matches and it is
// longer. Always return the longer pattern.
else if (pat.PatternValue.Length > equalPat.PatternValue.Length)
{
equalPat = pat;
}
}
pat = pat.NextSameLeadChar;
}
// check for the substring at the current location in string as not equal any
// of the ScanNotEqual pattern strings.
if (equalPat == null)
{
break;
}
// advance past the whitespace string.
stringIx += equalPat.PatternValue.Length;
}
// return the scan results
ScanPatternResults spr = null;
if (stringIx == -1)
{
spr = new ScanPatternResults(-1);
}
else
{
spr = new ScanPatternResults(stringIx, InString[stringIx]);
}
spr.ScannedString = InString;
spr.ScanStartIx = InIx;
spr.ScanBoundsEx = InEx;
return(spr);
}
public void Test()
{
var lParen = new Symbol("(");
var rParen = new Symbol(")");
var num = new Symbol("NUM");
var ident = new Symbol("ID");
var qStr = new Symbol("QSTR");
var grammar = new Grammar
{
Symbols =
{
lParen,
rParen,
num,
ident,
qStr
},
Conditions =
{
new Condition("main")
{
Matchers =
{
new Matcher(
ScanPattern.CreateRegular(
null,
@"[ \t]+"))
{
Joint ={ new CilMatcher(typeof(void)) }
},
new Matcher(
ScanPattern.CreateRegular(
null,
@"[0-9]+(?:[.][0-9]+)? | [.][0-9]+"),
num)
{
Joint ={ new CilMatcher(typeof(Num)) }
},
new Matcher(
ScanPattern.CreateRegular(
null,
@"[a-zA-Z:.!@#$%^&|?*/+*=\\_-][a-zA-Z:\d.!@#$%^&|?*/+*=\\_-]*"),
ident)
{
Joint ={ new CilMatcher(typeof(string)) }
},
new Matcher(
ScanPattern.CreateRegular(
null,
@"[""](?: \\[""] | [^""])* [""]"),
qStr)
{
Joint ={ new CilMatcher(typeof(QStr)) }
},
new Matcher(
ScanPattern.CreateLiteral("("),
lParen),
new Matcher(
ScanPattern.CreateLiteral(")"),
rParen),
}
}
}
};
var target = new BootstrapScanner(
" (1 (\"bar\" +))",
ScannerDescriptor.FromScanRules(grammar.Conditions[0].Matchers, ExceptionLogging.Instance),
null,
ExceptionLogging.Instance);
var collector = new Collector <Msg>();
target.Accept(collector);
Assert.AreEqual(
new object[] { null, new Num("1"), null, new QStr("bar"), "+", null, null },
collector.Select(msg => msg.Value).ToArray());
}
public bool IsDividerPattern(ScanPattern Pattern)
{
return(false);
}
private ScanPatterns AssembleNonWordPatterns()
{
ScanPatterns pats = new ScanPatterns();
pats.Add(WhitespacePatterns);
pats.AddDistinct(NewLinePatterns);
pats.Add(CommentToEndPatterns);
pats.AddDistinct(QuotePatterns);
if (this.VerbatimLiteralPattern != null)
{
pats.AddDistinct(this.VerbatimLiteralPattern);
}
pats.AddDistinct(ExpressionPatterns);
pats.AddDistinct(EndStmtPatterns);
pats.AddDistinct(OpenContentBracedPatterns);
pats.AddDistinct(OpenNamedBracedPatterns);
pats.AddDistinct(CloseBracedPatterns);
pats.AddDistinct(PathSepPatterns);
// standard delimeters like "," and ":"
pats.AddDistinct(DividerPatterns);
// special value starter pattern. *LIBL is the special value. * is the starter
// value.
if (this.SpecialValueStarter != null)
{
ScanPattern pat =
new ScanPattern(this.SpecialValueStarter, DelimClassification.SpecialValueStarter);
pats.AddDistinct(pat);
}
// note. dont add keyword patterns to this collection of non word patterns.
// Keyword patterns are a subset of the text ( identifier ) tokens found
// between non word tokens.
// BgnTemp
// if (_KeywordPatterns != null)
// {
// pats.AddDistinct(KeywordPatterns);
// }
// EndTemp
// sort the list of patterns by patternValue and DelimClass.
var sortedPats = from a in pats
orderby a.PatternValue, a.DelimClassification, a.UserCode
select a;
// build a list of patterns that are distinct on patternValue, delimClass, and
// keyword code.
ScanPatterns distinctPats = new ScanPatterns();
ScanPattern pvPat = null;
foreach (var pat in sortedPats)
{
if ((pvPat == null) || (pat.Equals(pvPat) == false))
{
distinctPats.Add(pat.Duplicate( ));
}
pvPat = pat;
}
return(distinctPats);
}
