private TokenRegex()
{
Regexes.Add(TokenType.Comment, new Regex(@"//.*"));
Regexes.Add(TokenType.Whitespace, new Regex(@"\s+"));
Regexes.Add(TokenType.String, new Regex(@"""[^""]+"""));
Regexes.Add(TokenType.Number, new Regex(@"[0-9]+"));
Regexes.Add(TokenType.Operator, new Regex(@"\+|-|\*|\/|>=|<=|==|!=|<|>"));
// Characters
Regexes.Add(TokenType.Equals, new Regex(@"="));
Regexes.Add(TokenType.OpenBracket, new Regex(@"\("));
Regexes.Add(TokenType.CloseBracket, new Regex(@"\)"));
Regexes.Add(TokenType.Comma, new Regex(@","));
// Keywords
Regexes.Add(TokenType.If, new Regex(@"if"));
Regexes.Add(TokenType.EndIf, new Regex(@"end if"));
Regexes.Add(TokenType.Else, new Regex(@"else"));
Regexes.Add(TokenType.Then, new Regex(@"then"));
Regexes.Add(TokenType.While, new Regex(@"while"));
Regexes.Add(TokenType.EndWhile, new Regex(@"end while"));
Regexes.Add(TokenType.Nil, new Regex(@"nil|NIL"));
Regexes.Add(TokenType.False, new Regex(@"false|FALSE"));
Regexes.Add(TokenType.True, new Regex(@"true|TRUE"));
Regexes.Add(TokenType.Local, new Regex(@"local"));
Regexes.Add(TokenType.Return, new Regex(@"return"));
Regexes.Add(TokenType.End, new Regex(@"end"));
Regexes.Add(TokenType.Function, new Regex(@"function"));
Regexes.Add(TokenType.Identifier, new Regex(@"[a-zA-Z_]\w*")); // Must come after keywords (don't want to overwrite keywords with identifiers!)
}
/// <summary>
/// Finds a match in a list of potential regex match candidates and shoves resulting captured key/value pairs into a dictionary.
/// </summary>
/// <param name="line">Log line to apply regexes to.</param>
/// <returns>Dictionary of key/value pairs of captured groups using a matching regex.</returns>
protected virtual IDictionary <string, object> FindAndApplyRegexMatch(string line)
{
IDictionary <string, object> fields = null;
bool foundMatch = false;
int indexToTry = 0;
// Loop all over the known good patterns, looking for a match.
while (!foundMatch && indexToTry < Regexes.Count)
{
// Apply regex and dump all of the named capture groups into a dictionary.
fields = Regexes[indexToTry].MatchNamedCaptures(line);
if (fields.Count > 0)
{
foundMatch = true;
// Make sure the matching regex is at the front of the list to optimize future matching
if (indexToTry > 0)
{
Regexes.MoveToFront(indexToTry);
}
}
indexToTry++;
}
return(fields);
}
// -----------< Helper Function - Assemble message for posting >------------
private CsMessage AssembleMsg()
{
CsMessage msg = new CsMessage();
msg.add("SubDir", SubDir);
msg.add("Directory", WDirectory);
msg.add("path", WDirectory);
//msg.add("OutputPath", OutputPath);
for (int i = 0; i < Patterns.Count(); i++)
{
msg.add("Pattern" + (i + 1), Patterns[i]);
}
for (int i = 0; i < Regexes.Count(); i++)
{
msg.add("Regex" + (i + 1), Regexes[i]);
}
if (Analysers != null)
{
for (int i = 0; i < Analysers.Count(); i++)
{
msg.add("Analyser" + (i + 1), Analysers[i]);
}
}
return(msg);
}
internal void RemoveFilter(string v)
{
switch (v.ToLower())
{
case "extensions":
{
if (string.IsNullOrEmpty(TextBoxExtension))
{
return;
}
Extensions.Remove(TextBoxExtension);
TextBoxExtension = string.Empty;
break;
}
case "directories":
{
if (string.IsNullOrEmpty(TextBoxDirectories))
{
return;
}
Directories.Remove(TextBoxDirectories);
TextBoxDirectories = string.Empty;
break;
}
case "regex":
{
if (string.IsNullOrEmpty(TextBoxRegex))
{
return;
}
Regexes.Remove(TextBoxRegex);
TextBoxRegex = string.Empty;
break;
}
case "strings":
{
if (string.IsNullOrEmpty(TextBoxStrings))
{
return;
}
Strings.Remove(TextBoxStrings);
TextBoxStrings = string.Empty;
break;
}
default:
break;
}
}
public PathLinkedImplementationFactory() : base(new Regex[] {
new Regex(@"(?<![\w])(?:file:\/\/(?:localhost)?)?\/([A-Z]([:$])?\/[^\?<>:\\|\*""\s]*)(?![\w])", RegexOptions.Compiled | RegexOptions.IgnoreCase),
new Regex(@"""([A-Z]:\\[^\?<>:\/|\*""]*)""|\b[A-Z]:\\[^\?<>:\/|\*\s""]*\b", RegexOptions.Compiled | RegexOptions.IgnoreCase)
})
{
using (var enumerator = Regexes.GetEnumerator())
{
enumerator.MoveNext();
urlLinuxRegex = enumerator.Current;
enumerator.MoveNext();
windowsEscaped = enumerator.Current;
}
}
public string GetAnnotation(string text)
{
var matches = Regexes.Select(x => x.Match(text)).ToList();
if (!matches.Any(x => x.Success))
{
return(null);
}
var values = matches.Select(x => x.Success ? x.Groups[1].Value : null).ToList();
return(Projection(values));
}
private void ExtractRegexes(ExporterConsoleParameters pars)
{
if (AssemblyRegex != null && !string.IsNullOrEmpty(AssemblyRegex))
{
string str = $"<Regexes><Items>{AssemblyRegex}</Items></Regexes>";
Regexes rxs = null;
var xmlSer = new XmlSerializer(typeof(Regexes));
using (var sr = new StringReader(str))
{
using (var tr = new NamespaceIgnorantXmlTextReader(sr))
{
rxs = (Regexes)xmlSer.Deserialize(tr);
}
}
pars.AssemblyRegex.AddRange(rxs.Items);
}
}
public override List <ExtractResult> Extract(string text)
{
var result = new List <ExtractResult>();
if (string.IsNullOrEmpty(text))
{
return(result);
}
var matchSource = new Dictionary <Match, string>();
var matched = new bool[text.Length];
// Traverse every match results to see each position in the text is matched or not.
var collections = Regexes.ToDictionary(o => o.Key.Matches(text), p => p.Value);
foreach (var collection in collections)
{
for (int k = 0; k < text.Length; k++)
{
matched[k] = false;
}
foreach (Match m in collection.Key)
{
if (IsValidMatch(m))
{
for (var j = 0; j < m.Length; j++)
{
matched[m.Index + j] = true;
}
// Keep Source Data for extra information
matchSource.Add(m, collection.Value);
}
}
GetResult(matched, text, matchSource, result);
}
// Form the extracted results mark all the matched intervals in the text.
return(PostFilter(result));
}
private Regex CreateRegex(string pattern)
{
Regex regex;
if (parameters == null)
{
if (!Regexes.TryGetValue(pattern, out regex))
{
Regexes.Add(pattern, regex = new Regex(pattern));
}
}
else
{
var key = new KeyValuePair <string, RegexOptions>(pattern, ignorePatternWhitespace);
if (!RegexesParameters.TryGetValue(key, out regex))
{
RegexesParameters.Add(key, regex = new Regex(pattern, ignorePatternWhitespace));
}
}
return(regex);
}
private INIHighlighter()
{
Regexes.Add(new Pair <Regex, Color>(new Regex(@"(#|;).*", ropt), Color.ForestGreen));
Regexes.Add(new Pair <Regex, Color>(new Regex(@"^\s*\[.*\]", ropt), Color.Indigo));
Regexes.Add(new Pair <Regex, Color>(new Regex(@"^([^\=])+", ropt), Color.Blue));
}
public virtual List <ExtractResult> Extract(string source)
{
if (string.IsNullOrEmpty(source))
{
return(new List <ExtractResult>());
}
var result = new List <ExtractResult>();
var matchSource = new Dictionary <Match, TypeTag>();
var matched = new bool[source.Length];
var collections = Regexes.ToDictionary(o => o.Key.Matches(source), p => p.Value);
foreach (var collection in collections)
{
foreach (Match m in collection.Key)
{
// In ExperimentalMode, AmbigiuousFraction like "30000 in 2009" needs to be skipped
if (Options == NumberOptions.ExperimentalMode && AmbiguousFractionConnectorsRegex.Match(m.Value).Success)
{
continue;
}
for (var j = 0; j < m.Length; j++)
{
matched[m.Index + j] = true;
}
// Keep Source Data for extra information
matchSource.Add(m, collection.Value);
}
}
var last = -1;
for (var i = 0; i < source.Length; i++)
{
if (matched[i])
{
if (i + 1 == source.Length || !matched[i + 1])
{
var start = last + 1;
var length = i - last;
var substr = source.Substring(start, length);
if (matchSource.Keys.Any(o => o.Index == start && o.Length == length))
{
var type = matchSource.Where(p => p.Key.Index == start && p.Key.Length == length)
.Select(p => (p.Value.Priority, p.Value.Name)).Min().Item2;
// Extract negative numbers
if (NegativeNumberTermsRegex != null)
{
var match = NegativeNumberTermsRegex.Match(source.Substring(0, start));
if (match.Success)
{
start = match.Index;
length = length + match.Length;
substr = match.Value + substr;
}
}
var er = new ExtractResult
{
Start = start,
Length = length,
Text = substr,
Type = ExtractType,
Data = type
};
result.Add(er);
}
}
}
else
{
last = i;
}
}
result = FilterAmbiguity(result, source);
return(result);
}
public virtual List <ExtractResult> Extract(string source)
{
if (string.IsNullOrEmpty(source))
{
return(new List <ExtractResult>());
}
var results = new List <ExtractResult>();
var matchSource = new Dictionary <Tuple <int, int>, string>();
var matched = new bool[source.Length];
var collections = Regexes.ToDictionary(o => o.Key.Matches(source), p => p.Value);
foreach (var collection in collections)
{
foreach (Match m in collection.Key)
{
GetMatchedStartAndLength(m, collection.Value, source, out int start, out int length);
if (start >= 0 && length > 0)
{
for (var j = 0; j < length; j++)
{
matched[start + j] = true;
}
// Keep Source Data for extra information
matchSource.Add(new Tuple <int, int>(start, length), collection.Value);
}
}
}
var last = -1;
for (var i = 0; i < source.Length; i++)
{
if (matched[i])
{
if (i + 1 == source.Length || !matched[i + 1])
{
var start = last + 1;
var length = i - last;
var substr = source.Substring(start, length);
if (matchSource.Keys.Any(o => o.Item1 == start && o.Item2 == length))
{
var srcMatch = matchSource.Keys.First(o => o.Item1 == start && o.Item2 == length);
var er = new ExtractResult
{
Start = start,
Length = length,
Text = substr,
Type = ExtractType,
Data = matchSource.ContainsKey(srcMatch) ? matchSource[srcMatch] : null,
};
results.Add(er);
}
}
}
else
{
last = i;
}
}
// In ExperimentalMode, cases like "from 3 to 5" and "between 10 and 15" are set to closed at both start and end
if ((Options & NumberOptions.ExperimentalMode) != 0)
{
foreach (var result in results)
{
if (result.Data.ToString() == NumberRangeConstants.TWONUMBETWEEN ||
result.Data.ToString() == NumberRangeConstants.TWONUMTILL)
{
result.Data = NumberRangeConstants.TWONUMCLOSED;
}
}
}
return(results);
}
public virtual List <ExtractResult> Extract(string source)
{
if (string.IsNullOrEmpty(source))
{
return(new List <ExtractResult>());
}
var result = new List <ExtractResult>();
var matchSource = new Dictionary <Match, TypeTag>();
var matched = new bool[source.Length];
var collections = Regexes.ToDictionary(o => o.Key.Matches(source), p => p.Value);
foreach (var collection in collections)
{
foreach (Match m in collection.Key)
{
// In ExperimentalMode, AmbiguousFraction like "30000 in 2009" needs to be skipped
if ((Options & NumberOptions.ExperimentalMode) != 0 && AmbiguousFractionConnectorsRegex.Match(m.Value).Success)
{
continue;
}
// In EnablePreview, cases like "last", "next" should not be skipped
if ((Options & NumberOptions.EnablePreview) == 0 && IsRelativeOrdinal(m.Value))
{
continue;
}
for (var j = 0; j < m.Length; j++)
{
matched[m.Index + j] = true;
}
// Fliter out cases like "first two", "last one"
// only support in English now
if (ExtractType.Contains(Constants.MODEL_ORDINAL) && RelativeOrdinalFilterRegex != null && RelativeOrdinalFilterRegex.IsMatch(source))
{
continue;
}
// Keep Source Data for extra information
matchSource.Add(m, collection.Value);
}
}
var last = -1;
for (var i = 0; i < source.Length; i++)
{
if (matched[i])
{
if (i + 1 == source.Length || !matched[i + 1])
{
var start = last + 1;
var length = i - last;
var substr = source.Substring(start, length);
if (matchSource.Keys.Any(o => o.Index == start && o.Length == length))
{
var type = matchSource.Where(p => p.Key.Index == start && p.Key.Length == length)
.Select(p => (p.Value.Priority, p.Value.Name)).Min().Item2;
// Extract negative numbers
if (NegativeNumberTermsRegex != null)
{
var match = NegativeNumberTermsRegex.Match(source.Substring(0, start));
if (match.Success)
{
start = match.Index;
length = length + match.Length;
substr = match.Value + substr;
}
}
var er = new ExtractResult
{
Start = start,
Length = length,
Text = substr,
Type = ExtractType,
Data = type,
};
// Add Metadata information for Ordinal
if (ExtractType.Contains(Constants.MODEL_ORDINAL))
{
er.Metadata = new Metadata();
if (IsRelativeOrdinal(substr))
{
er.Metadata.IsOrdinalRelative = true;
}
}
result.Add(er);
}
}
}
else
{
last = i;
}
}
result = FilterAmbiguity(result, source);
return(result);
}
public virtual List <ExtractResult> Extract(string source)
{
if (string.IsNullOrEmpty(source))
{
return(new List <ExtractResult>());
}
var results = new List <ExtractResult>();
var matchSource = new Dictionary <Tuple <int, int>, string>();
var matched = new bool[source.Length];
var collections = Regexes.ToDictionary(o => o.Key.Matches(source), p => p.Value);
foreach (var collection in collections)
{
foreach (Match m in collection.Key)
{
GetMatchedStartAndLength(m, collection.Value, source, out int start, out int length);
if (start >= 0 && length > 0)
{
// Keep Source Data for extra information
matchSource.Add(new Tuple <int, int>(start, length), collection.Value);
}
}
}
foreach (var match in matchSource)
{
var start = match.Key.Item1;
var length = match.Key.Item2;
// Filter wrong two number ranges such as "more than 20 and less than 10" and "大于20小于10".
if (match.Value.Equals(NumberRangeConstants.TWONUM, StringComparison.Ordinal))
{
int moreIndex = 0, lessIndex = 0;
var text = source.Substring(match.Key.Item1, match.Key.Item2);
var er = numberExtractor.Extract(text);
if (er.Count != 2)
{
er = ordinalExtractor.Extract(text);
if (er.Count != 2)
{
continue;
}
}
var nums = er.Select(r => (double)(numberParser.Parse(r).Value ?? 0)).ToList();
// Order matchSource by decreasing match length so that "no less than x" is before "less than x"
var matchList = matchSource.ToList();
matchList.Sort((pair1, pair2) => pair2.Key.Item2.CompareTo(pair1.Key.Item2));
moreIndex = matchList.First(r =>
r.Value.Equals(NumberRangeConstants.MORE, StringComparison.Ordinal) &&
r.Key.Item1 >= start && r.Key.Item1 + r.Key.Item2 <= start + length).Key.Item1;
lessIndex = matchList.First(r =>
r.Value.Equals(NumberRangeConstants.LESS, StringComparison.Ordinal) &&
r.Key.Item1 >= start && r.Key.Item1 + r.Key.Item2 <= start + length).Key.Item1;
if (!((nums[0] < nums[1] && moreIndex <= lessIndex) || (nums[0] > nums[1] && moreIndex >= lessIndex)))
{
continue;
}
}
// The entity is longer than 1, so don't mark the last char to represent the end.
// To avoid no connector cases like "大于20小于10" being marked as a whole entity.
for (var j = 0; j < length - 1; j++)
{
matched[start + j] = true;
}
}
var last = -1;
for (var i = 0; i < source.Length; i++)
{
if (matched[i])
{
if (i + 1 == source.Length || !matched[i + 1])
{
var start = last + 1;
var length = i - last + 1;
var substr = source.Substring(start, length);
if (matchSource.Keys.Any(o => o.Item1 == start && o.Item2 == length))
{
var srcMatch = matchSource.Keys.First(o => o.Item1 == start && o.Item2 == length);
var er = new ExtractResult
{
Start = start,
Length = length,
Text = substr,
Type = ExtractType,
Data = matchSource.ContainsKey(srcMatch) ? matchSource[srcMatch] : null,
};
results.Add(er);
}
}
}
else
{
last = i;
}
}
// In ExperimentalMode, cases like "from 3 to 5" and "between 10 and 15" are set to closed at both start and end
if ((Config.Options & NumberOptions.ExperimentalMode) != 0)
{
foreach (var result in results)
{
var data = result.Data.ToString();
if (data == NumberRangeConstants.TWONUMBETWEEN ||
data == NumberRangeConstants.TWONUMTILL)
{
result.Data = NumberRangeConstants.TWONUMCLOSED;
}
}
}
return(results);
}
public virtual List <ExtractResult> Extract(string source)
{
if (string.IsNullOrEmpty(source))
{
return(new List <ExtractResult>());
}
var result = new List <ExtractResult>();
var matchSource = new Dictionary <Match, string>();
var matched = new bool[source.Length];
var collections = Regexes.ToDictionary(o => o.Key.Matches(source), p => p.Value);
foreach (var collection in collections)
{
foreach (Match m in collection.Key)
{
for (var j = 0; j < m.Length; j++)
{
matched[m.Index + j] = true;
}
// Keep Source Data for extra information
matchSource.Add(m, collection.Value);
}
}
var last = -1;
for (var i = 0; i < source.Length; i++)
{
if (matched[i])
{
if (i + 1 == source.Length || !matched[i + 1])
{
var start = last + 1;
var length = i - last;
var substr = source.Substring(start, length);
if (matchSource.Keys.Any(o => o.Index == start && o.Length == length))
{
var srcMatch = matchSource.Keys.First(o => o.Index == start && o.Length == length);
var er = new ExtractResult
{
Start = start,
Length = length,
Text = substr,
Type = ExtractType,
Data = matchSource.ContainsKey(srcMatch) ? matchSource[srcMatch] : null
};
result.Add(er);
}
}
}
else
{
last = i;
}
}
return(result);
}
public virtual List <ExtractResult> Extract(string text)
{
var result = new List <ExtractResult>();
if (string.IsNullOrEmpty(text))
{
return(result);
}
var matchSource = new Dictionary <Match, string>();
var matched = new bool[text.Length];
//Traverse every match results to see each position in the text is matched or not.
var collections = Regexes.ToDictionary(o => o.Key.Matches(text), p => p.Value);
foreach (var collection in collections)
{
foreach (Match m in collection.Key)
{
if (IsValidMatch(m))
{
for (var j = 0; j < m.Length; j++)
{
matched[m.Index + j] = true;
}
// Keep Source Data for extra information
matchSource.Add(m, collection.Value);
}
}
}
// Form the extracted results mark all the matched intervals in the text.
var lastNotMatched = -1;
for (var i = 0; i < text.Length; i++)
{
if (matched[i])
{
if (i + 1 == text.Length || !matched[i + 1])
{
var start = lastNotMatched + 1;
var length = i - lastNotMatched;
var substr = text.Substring(start, length);
bool matchFunc(Match o) => o.Index == start && o.Length == length;
if (matchSource.Keys.Any(matchFunc))
{
var srcMatch = matchSource.Keys.First(matchFunc);
result.Add(new ExtractResult
{
Start = start,
Length = length,
Text = substr,
Type = ExtractType,
Data = matchSource.ContainsKey(srcMatch) ? matchSource[srcMatch] : null
});
}
}
}
else
{
lastNotMatched = i;
}
}
return(result);
}
public virtual List <ExtractResult> Extract(string source)
{
if (string.IsNullOrEmpty(source))
{
return(new List <ExtractResult>());
}
var result = new List <ExtractResult>();
var matchSource = new Dictionary <Match, string>();
var matched = new bool[source.Length];
var collections = Regexes.ToDictionary(o => o.Key.Matches(source), p => p.Value);
foreach (var collection in collections)
{
foreach (Match m in collection.Key)
{
matchSource.Add(m, collection.Value);
}
}
matchSource = RankMatches(matchSource);
matchSource = RemoveRedundantMatches(matchSource);
foreach (var match in matchSource)
{
for (var i = 0; i < match.Key.Length; i++)
{
matched[match.Key.Index + i] = true;
}
}
var last = -1;
for (var i = 0; i < source.Length; i++)
{
if (matched[i])
{
if (i + 1 == source.Length || !matched[i + 1])
{
var start = last + 1;
var length = i - last;
var substr = source.Substring(start, length);
if (matchSource.Keys.Any(o => o.Index == start && o.Length == length))
{
var srcMatch = matchSource.Keys.First(o => o.Index == start && o.Length == length);
// Extract negative numbers
if (NegativeNumberTermsRegex != null)
{
var match = NegativeNumberTermsRegex.Match(source.Substring(0, start));
if (match.Success)
{
start = match.Index;
length = length + match.Length;
substr = match.Value + substr;
}
}
var er = new ExtractResult
{
Start = start,
Length = length,
Text = substr,
Type = ExtractType,
Data = matchSource.ContainsKey(srcMatch) ? matchSource[srcMatch] : null
};
result.Add(er);
}
}
}
else
{
last = i;
}
}
return(result);
}
internal void AddFilter(string v)
{
switch (v.ToLower())
{
case "extensions":
{
if (string.IsNullOrEmpty(TextBoxExtension))
{
return;
}
// Add leading dot if not present.
if (!TextBoxExtension.StartsWith("."))
{
TextBoxExtension = "." + TextBoxExtension;
}
Extensions.Add(TextBoxExtension.ToLower());
TextBoxExtension = string.Empty;
break;
}
case "directories":
{
if (string.IsNullOrEmpty(TextBoxDirectories))
{
return;
}
Directories.Add(TextBoxDirectories.ToLower());
TextBoxDirectories = string.Empty;
break;
}
case "regex":
{
if (string.IsNullOrEmpty(TextBoxRegex))
{
return;
}
Regexes.Add(TextBoxRegex.ToLower());
TextBoxRegex = string.Empty;
break;
}
case "strings":
{
if (string.IsNullOrEmpty(TextBoxStrings))
{
return;
}
Strings.Add(TextBoxStrings.ToLower());
TextBoxStrings = string.Empty;
break;
}
default:
break;
}
}
public override List <ExtractResult> Extract(string text)
{
var result = new List <ExtractResult>();
if (string.IsNullOrEmpty(text))
{
return(result);
}
var matchSource = new Dictionary <Match, string>();
var matched = new bool[text.Length];
var collections = Regexes.ToDictionary(o => o.Key.Matches(text), p => p.Value);
foreach (var collection in collections)
{
foreach (Match m in collection.Key)
{
for (var j = 0; j < m.Length; j++)
{
matched[m.Index + j] = true;
}
// Keep Source Data for extra information
matchSource.Add(m, collection.Value);
}
}
var lastNotMatched = -1;
for (var i = 0; i < text.Length; i++)
{
if (matched[i])
{
if (i + 1 == text.Length || !matched[i + 1])
{
var start = lastNotMatched + 1;
var length = i - lastNotMatched;
var substr = text.Substring(start, length);
if (substr.StartsWith(Constants.IPV6_ELLIPSIS) &&
(start > 0 && char.IsLetterOrDigit(text[start - 1]) && !SimpleTokenizer.IsCjk(text[start - 1])))
{
continue;
}
if (substr.EndsWith(Constants.IPV6_ELLIPSIS) &&
(i + 1 < text.Length && char.IsLetterOrDigit(text[i + 1]) && !SimpleTokenizer.IsCjk(text[start + 1])))
{
continue;
}
bool MatchFunc(Match o) => o.Index == start && o.Length == length;
if (matchSource.Keys.Any(MatchFunc))
{
var srcMatch = matchSource.Keys.First(MatchFunc);
result.Add(new ExtractResult
{
Start = start,
Length = length,
Text = substr,
Type = ExtractType,
Data = matchSource.ContainsKey(srcMatch) ? matchSource[srcMatch] : null,
});
}
}
}
else
{
lastNotMatched = i;
}
}
return(result);
}
public virtual List <ExtractResult> Extract(string source)
{
if (string.IsNullOrEmpty(source))
{
return(new List <ExtractResult>());
}
var result = new List <ExtractResult>();
var matchSource = new Dictionary <Match, TypeTag>();
var matched = new bool[source.Length];
var collections = Regexes.ToDictionary(o => o.Key.Matches(source), p => p.Value);
foreach (var collection in collections)
{
foreach (Match m in collection.Key)
{
// In ExperimentalMode, AmbiguousFraction like "30000 in 2009" needs to be skipped
if ((Options & NumberOptions.ExperimentalMode) != 0 && AmbiguousFractionConnectorsRegex.Match(m.Value).Success)
{
continue;
}
// If SuppressExtendedTypes is on, cases like "last", "next" should be skipped
if ((Options & NumberOptions.SuppressExtendedTypes) != 0 && m.Groups[Constants.RelativeOrdinalGroupName].Success)
{
continue;
}
// Matches containing separators 'in', 'out of' should be considered fractions only when numerator < denominator
if (m.Groups["ambiguousSeparator"].Success)
{
var numerator = m.Groups["numerator"];
var denominator = m.Groups["denominator"];
int num = ParseNumber(numerator);
int den = ParseNumber(denominator);
if (num > den)
{
continue;
}
}
for (var j = 0; j < m.Length; j++)
{
matched[m.Index + j] = true;
}
// Keep Source Data for extra information
matchSource.Add(m, collection.Value);
}
}
var last = -1;
for (var i = 0; i < source.Length; i++)
{
if (matched[i])
{
if (i + 1 == source.Length || !matched[i + 1])
{
var start = last + 1;
var length = i - last;
var substr = source.Substring(start, length);
if (matchSource.Keys.Any(o => o.Index == start && o.Length == length))
{
var(_, type, originalMatch) = matchSource.Where(p => p.Key.Index == start && p.Key.Length == length)
.Select(p => (p.Value.Priority, p.Value.Name, p.Key)).Min();
// Extract negative numbers
if (NegativeNumberTermsRegex != null)
{
var match = NegativeNumberTermsRegex.Match(source.Substring(0, start));
if (match.Success)
{
start = match.Index;
length += match.Length;
substr = match.Value + substr;
}
}
var er = new ExtractResult
{
Start = start,
Length = length,
Text = substr,
Type = ExtractType,
Data = type,
};
// Add Metadata information for Ordinal
if (ExtractType.Contains(Constants.MODEL_ORDINAL))
{
er.Metadata = new Metadata();
if ((Options & NumberOptions.SuppressExtendedTypes) == 0 &&
originalMatch.Groups[Constants.RelativeOrdinalGroupName].Success)
{
er.Metadata.IsOrdinalRelative = true;
}
}
result.Add(er);
}
}
}
else
{
last = i;
}
}
result = FilterAmbiguity(result, source);
return(result);
}
static void Main(string[] args)
{
Trace.Write("START");
Debuggers.LogError("error occured");
Debuggers.LogError(new Exception("ex occured"));
Dictionary <string, int> dict = new Dictionary <string, int>()
{
{ "Marketing", 1 }, { "Sales", 2 }, { "IT", 3 }
};
var dictExists = dict.Contains(new KeyValuePair <string, int>("IT", 2));
Formaters.DateAndTime(DateTime.Now, 10.23456D);
Debuggers.Start();
var customers = Linqers.CustomersWithOrdersByYear(2017);
Linqers.GetProductsLongestNameByCategory();
string asyncResult;
Task.Run(async() =>
{
asyncResult = await Threads.StartAsync();
}).GetAwaiter().GetResult();
Threads.RunTimer();
var publicTypes = new Reflections().GetPublicTypes();
var assemblyName = new Reflections().GetAssemblyName();
var isPositiveDecimal = Regexes.PositiveWithTwoDecimalPlaces(5.666M);
Console.WriteLine("Available memory: " + new PerformanceCounter("Memory", "Available MBytes").NextValue());
var assemblies = new Reflections().GetTypesFromCurrentDomain();
Product productForSerialization = new Product()
{
CategoryId = 1, Id = 2, IsValid = true
};
Serializators.SerializeWithBinaryFormatter(productForSerialization, "bin.dat");
Serializators.SerializeWithDataContractToFile(productForSerialization, "datacontract.dat");
string userSerialized = Serializators.SerializeWithBinaryWriter(new Product {
Id = 10
});
DateTime?nullableDateTime = null;
bool isDateNotNull = nullableDateTime.HasValue;
RateCollection rateCollection = new RateCollection(new Rate[] { new Rate {
Value = 1
} });
foreach (var item in rateCollection)
{
Console.WriteLine(item);
}
var currentAssembly = Assembly.GetExecutingAssembly();
var sb = new StringBuilder();
sb.Append("First");
sb.AppendLine();
sb.Append("Second");
Console.WriteLine(sb);
SortedList <string, string> sortedList = new SortedList <string, string>()
{
{ "asd", "dsa" }
};
Debug.Assert(false, "stop");
float amount = 1.6F;
object amountObj = amount;
int amountInt = (int)(float)amountObj;
new Product().Add("book1");
User newUser = new User()
{
UserGroup = Group.Supervisors | Group.Users
};
bool isTrue = newUser.UserGroup < Group.Administrator;
var userGroup = newUser.UserGroup;
Console.WriteLine(userGroup);
string stringNull = null;
string stringNotNull = "asd";
Comparers.AreEqual(stringNull, stringNotNull);
Rate rate1 = new Rate()
{
Value = 1, Category = "cat"
};
string xml = Serializators.SerializeWithDataContract(rate1);
string json = Serializators.SerializeWithDataContractJson(rate1);
Console.WriteLine("xml:\r\n" + xml);
Console.WriteLine("json:\r\n" + json);
Subscriber sub = new Subscriber();
sub.Subscribe();
sub.Execute();
Console.Read();
return;
DataContractJsonSerializer js = new DataContractJsonSerializer(typeof(Rate));
Console.WriteLine(string.Format("{0} asdasd {1:000#} asd", 4, 159));
Console.WriteLine(123.ToString("000#"));
Rate ratenull = null;
int wynik;
int.TryParse(ratenull.Category, out wynik);
Console.Read();
BaseLogger logger = new Logger();
logger.Log("Log started");
logger.Log("Base: Log contiuniug");
((Logger)logger).LogCompleted();
Console.Read();
return;
Reflections.SetPropertiesOnObject(new Rate()
{
MyInt = 10
}, "MyInt", "MyIntSpecified");
float mojfloat = 1.6F;
double dable = (double)mojfloat;
var hashed = Hashers.HashByAlgName(@"C:\windows-version.txt", "SHA");
Threads.ConcurrentDict();
var x = from i in new List <int> {
1, 2
}
group i by i into grouped
where grouped.Key > 1
select grouped.Key;
string xmlInput = "<xml><RateSheet><rate category=\"boutou\" date=\"2012-12-12\"><value>0.03</value></rate><rate category=\"druga\" date=\"2011-11-11\"><value>0.04</value></rate></RateSheet></xml>";
var result = Serializators.ReadFromXml(xmlInput);
SHA1Managed SHhash = new SHA1Managed();
//new Class2().Method1();
Class1 class1 = new Class1();
INewInterface interf = class1;
interf.Method1();
IEnumerable <Person> people = new List <Person>()
{
new Person {
PhoneNumbers = new List <PhoneNumber> {
new PhoneNumber {
Number = "1"
}, new PhoneNumber {
Number = "2"
}
}
},
new Person {
PhoneNumbers = new List <PhoneNumber> {
new PhoneNumber {
Number = "2"
}, new PhoneNumber {
Number = "3"
}
}
},
};
IEnumerable <IEnumerable <PhoneNumber> > phoneLists = people.Select(p => p.PhoneNumbers);
IEnumerable <PhoneNumber> phoneNumbers = people.SelectMany(p => p.PhoneNumbers);
}
public static IEnumerator<object> FromFile(string filename, IProgressListener progress)
{
progress.Status = "Loading diff...";
Future<string> fText;
// We could stream the lines in from the IO thread while we parse them, but this
// part of the load is usually pretty quick even on a regular hard disk, and
// loading the whole diff at once eliminates some context switches
using (var fda = new FileDataAdapter(
filename, FileMode.Open,
FileAccess.Read, FileShare.Read, 1024 * 128
)) {
var fBytes = fda.ReadToEnd();
yield return fBytes;
fText = Future.RunInThread(
() => Encoding.ASCII.GetString(fBytes.Result)
);
yield return fText;
}
yield return fText;
var lr = new LineReader(fText.Result);
LineReader.Line line;
progress.Status = "Parsing diff...";
var frames = new List<TracebackFrame>();
var moduleNames = new NameTable(StringComparer.Ordinal);
var symbolTypes = new NameTable(StringComparer.Ordinal);
var functionNames = new NameTable(StringComparer.Ordinal);
var deltas = new List<DeltaInfo>();
var tracebacks = new Dictionary<UInt32, TracebackInfo>();
var regexes = new Regexes();
// Regex.Groups[string] does an inefficient lookup, so we do that lookup once here
int groupModule = regexes.DiffModule.GroupNumberFromName("module");
int groupSymbolType = regexes.DiffModule.GroupNumberFromName("symbol_type");
int groupTraceId = regexes.BytesDelta.GroupNumberFromName("trace_id");
int groupType = regexes.BytesDelta.GroupNumberFromName("type");
int groupDeltaBytes = regexes.BytesDelta.GroupNumberFromName("delta_bytes");
int groupNewBytes = regexes.BytesDelta.GroupNumberFromName("new_bytes");
int groupOldBytes = regexes.BytesDelta.GroupNumberFromName("old_bytes");
int groupNewCount = regexes.BytesDelta.GroupNumberFromName("new_count");
int groupOldCount = regexes.CountDelta.GroupNumberFromName("old_count");
int groupCountDelta = regexes.CountDelta.GroupNumberFromName("delta_count");
int groupTracebackModule = regexes.TracebackFrame.GroupNumberFromName("module");
int groupTracebackFunction = regexes.TracebackFrame.GroupNumberFromName("function");
int groupTracebackOffset = regexes.TracebackFrame.GroupNumberFromName("offset");
int groupTracebackOffset2 = regexes.TracebackFrame.GroupNumberFromName("offset2");
int groupTracebackPath = regexes.TracebackFrame.GroupNumberFromName("path");
int groupTracebackLine = regexes.TracebackFrame.GroupNumberFromName("line");
int i = 0;
while (lr.ReadLine(out line)) {
if (i % ProgressInterval == 0) {
progress.Maximum = lr.Length;
progress.Progress = lr.Position;
// Suspend processing until any messages in the windows message queue have been processed
yield return new Yield();
}
retryFromHere:
Match m;
if (regexes.DiffModule.TryMatch(ref line, out m)) {
moduleNames.Add(m.Groups[groupModule].Value);
} else if (regexes.BytesDelta.TryMatch(ref line, out m)) {
var traceId = UInt32.Parse(m.Groups[groupTraceId].Value, NumberStyles.HexNumber);
var info = new DeltaInfo {
Added = (m.Groups[groupType].Value == "+"),
BytesDelta = int.Parse(m.Groups[groupDeltaBytes].Value, NumberStyles.HexNumber),
NewBytes = int.Parse(m.Groups[groupNewBytes].Value, NumberStyles.HexNumber),
OldBytes = int.Parse(m.Groups[groupOldBytes].Value, NumberStyles.HexNumber),
NewCount = int.Parse(m.Groups[groupNewCount].Value, NumberStyles.HexNumber),
};
if (lr.ReadLine(out line)) {
if (regexes.CountDelta.TryMatch(ref line, out m)) {
info.OldCount = int.Parse(m.Groups[groupOldCount].Value, NumberStyles.HexNumber);
info.CountDelta = int.Parse(m.Groups[groupCountDelta].Value, NumberStyles.HexNumber);
}
}
bool readingLeadingWhitespace = true, doRetry = false;
frames.Clear();
var itemModules = new NameTable(StringComparer.Ordinal);
var itemFunctions = new NameTable(StringComparer.Ordinal);
while (lr.ReadLine(out line)) {
if (line.ToString().Trim().Length == 0) {
if (readingLeadingWhitespace)
continue;
else
break;
} else if (regexes.TracebackFrame.TryMatch(ref line, out m)) {
readingLeadingWhitespace = false;
var moduleName = moduleNames[m.Groups[groupTracebackModule].Value];
itemModules.Add(moduleName);
var functionName = functionNames[m.Groups[groupTracebackFunction].Value];
itemFunctions.Add(functionName);
var frame = new TracebackFrame {
Module = moduleName,
Function = functionName,
Offset = UInt32.Parse(m.Groups[groupTracebackOffset].Value, NumberStyles.HexNumber)
};
if (m.Groups[groupTracebackOffset2].Success)
frame.Offset2 = UInt32.Parse(m.Groups[groupTracebackOffset2].Value, NumberStyles.HexNumber);
if (m.Groups[groupTracebackPath].Success)
frame.SourceFile = m.Groups[groupTracebackPath].Value;
if (m.Groups[groupTracebackLine].Success)
frame.SourceLine = int.Parse(m.Groups[groupTracebackLine].Value);
frames.Add(frame);
} else {
// We hit the beginning of a new allocation, so make sure it gets parsed
doRetry = true;
break;
}
}
if (tracebacks.ContainsKey(traceId)) {
info.Traceback = tracebacks[traceId];
Console.WriteLine("Duplicate traceback for id {0}!", traceId);
} else {
var frameArray = ImmutableArrayPool<TracebackFrame>.Allocate(frames.Count);
frames.CopyTo(frameArray.Array, frameArray.Offset);
info.Traceback = tracebacks[traceId] = new TracebackInfo {
TraceId = traceId,
Frames = frameArray,
Modules = itemModules,
Functions = itemFunctions
};
}
deltas.Add(info);
if (doRetry)
goto retryFromHere;
} else if (line.StartsWith("//")) {
// Comment, ignore it
} else if (line.StartsWith("Total increase") || line.StartsWith("Total decrease")) {
// Ignore this too
} else if (line.StartsWith(" ") && (line.EndsWith(".pdb"))) {
// Symbol path for a module, ignore it
} else {
Console.WriteLine("Unrecognized diff content: {0}", line.ToString());
}
}
var result = new HeapDiff(
filename, moduleNames, functionNames, deltas, tracebacks
);
yield return new Result(result);
}
