private async Task <Analysis> AnalyseForSmallTalksAndCurseWords(string input, SmallTalksPreProcessingConfiguration configuration)
{
InputProcess preProcess = PreProcessInput(input, configuration);
await Init();
var analysis = new Analysis
{
Input = preProcess.Output,
Matches = new List <MatchData>()
};
DectectorData.SmallTalksIntents = DectectorData.SmallTalksIntents.OrderBy(i => i.Priority).ToList();
var parsedInput = input;
var haveCursedWords = false;
(parsedInput, haveCursedWords) = await _curseWordsDetector.ReplaceWordsAsync(parsedInput, InputProcess.Placeholder);
analysis.HaveCursedWords = haveCursedWords;
parsedInput = ParseInputSearchingForSmallTalks(configuration, analysis, parsedInput);
await FillHigherInformationLevel(configuration, analysis, parsedInput);
return(analysis);
}
public static InputProcess RemoveAccentuation(this InputProcess inputProcess)
{
var input = inputProcess.Data;
inputProcess.Output = input.RemoveDiacritics();
return(inputProcess);
}
public static InputProcess TranslateTextNumberToNumeric(this InputProcess inputProcess)
{
var input = inputProcess.Data;
inputProcess.Output = input.ParseIntegersFromRaw();
return(inputProcess);
}
public static InputProcess ToLower(this InputProcess inputProcess)
{
var input = inputProcess.Data;
inputProcess.Output = input.ToLowerInvariant();
return(inputProcess);
}
public static InputProcess RemoveRepeteadChars(this InputProcess inputProcess)
{
var input = inputProcess.Data;
var builder = new StringBuilder();
char firstLastChar = (char)0;
char secondLastChar = (char)0;
char currentChar = (char)0;
for (int i = 0; i < input.Length; i++)
{
secondLastChar = firstLastChar;
firstLastChar = currentChar;
currentChar = input[i];
if (MatchesSingleCharRule(currentChar) && currentChar == firstLastChar)
{
continue;
}
if (currentChar != secondLastChar)
{
builder.Append(currentChar);
}
}
inputProcess.Output = builder.ToString();
return(inputProcess);
}
public static InputProcess RemovePunctuation(this InputProcess inputProcess)
{
var input = inputProcess.Data;
inputProcess.Output = PunctuationRegex.Replace(input, string.Empty);
return(inputProcess);
}
public static InputProcess RemovePlaceholder(this InputProcess inputProcess)
{
var input = inputProcess.Data;
inputProcess.Output = input.Replace(InputProcess.Placeholder.ToString(), string.Empty).Trim();
return(inputProcess);
}
private async Task FillHigherInformationLevel(SmallTalksPreProcessingConfiguration configuration, Analysis analysis, string parsedInput)
{
if (configuration.InformationLevel >= InformationLevel.NORMAL)
{
var parsedInputProcess = InputProcess.FromString(parsedInput);
if (configuration.UnicodeNormalization)
{
parsedInputProcess = parsedInputProcess.RemoveAccentuation();
}
analysis.MarkedInput = parsedInputProcess.Output;
analysis.CleanedInput = parsedInputProcess
.RemovePunctuation()
.RemoveRepeteadChars()
.RemovePlaceholder()
.Output;
analysis.CleanedInputRatio = analysis.CleanedInput.Length / (float)analysis.Input.Length;
analysis.UseCleanedInput = analysis.CleanedInputRatio >= 0.5f;
if (configuration.InformationLevel >= InformationLevel.FULL)
{
analysis.RelevantInput = InputProcess
.FromString(await _stopWordsDetector.RemoveWordsAsync(analysis.CleanedInput))
.RemoveRepeteadChars()
.Output;
}
}
}
public void When_PreProcess_RepeteadChar_ShouldRemoveThen(string input, string expectedOutput)
{
var preprocessor = new InputProcess
{
Input = input
}
.RemoveRepeteadChars();
Assert.AreEqual(expectedOutput, preprocessor.Output);
}
private static InputProcess PreProcessInput(string input, SmallTalksPreProcessingConfiguration configuration)
{
var preProcess = new InputProcess
{
Input = input
}
.RemoveRepeteadChars();
if (configuration.ToLower)
{
preProcess = preProcess.ToLower();
}
return(preProcess);
}
public Analysis Detect(string input)
{
var preProcess = new InputProcess
{
Input = input
}
.RemoveRepeteadChars()
.ToLower();
Init();
var analysis = new Analysis
{
Input = preProcess.Output,
Matches = new List <MatchData>()
};
DectectorData.SmallTalksIntents = DectectorData.SmallTalksIntents.OrderBy(i => i.Priority).ToList();
var parsedInput = input;
foreach (var intent in DectectorData.SmallTalksIntents)
{
var matches = intent.Regex.Matches(parsedInput);
if (matches.Count > 0)
{
foreach (Match m in matches)
{
parsedInput = parsedInput.Replace(index: m.Index, length: m.Length, replacement: InputProcess.Placeholder);
analysis.Matches.Add(new MatchData
{
SmallTalk = intent.Name,
Value = m.Value,
Index = m.Index,
Lenght = m.Length,
});
}
}
}
analysis.AnalysedInput = parsedInput;
analysis.CleanedInput = InputProcess.FromString(parsedInput)
.RemovePlaceholder()
.RemovePunctuation()
.Output;
analysis.RelevantInput = _stopWordsDetector.RemoveStopWords(analysis.CleanedInput);
return(analysis);
}
public List <EvaluatedRule> GetMatches(Grammar grammar, string input)
{
var matches = new List <EvaluatedRule>();
var preProcess = new InputProcess
{
Input = input
}
.RemoveAccentuation()
.ToLower()
.RemoveRepeteadChars()
.TranslateTextNumberToNumeric();
var postProcessedInput = preProcess.Output;
ExpandAllRules(grammar);
bool atLeastOneMatch = true;
while (atLeastOneMatch)
{
atLeastOneMatch = false;
foreach (var rule in grammar.StartRules)
{
var match = rule.Match(postProcessedInput);
if (match != null)
{
atLeastOneMatch = true;
postProcessedInput = postProcessedInput.Replace(index: match.Index, length: match.Length, replacement: '_');
matches.Add(new EvaluatedRule
{
Rule = rule,
Input = input,
Value = match.Value
});
}
}
}
return(matches);
}
protected override IEnumerable <IRow> EvaluateImpl(Stopwatch netTimeStopwatch)
{
var group = new List <IReadOnlySlimRow>();
string lastKey = null;
netTimeStopwatch.Stop();
var enumerator = InputProcess.Evaluate(this).TakeRowsAndTransferOwnership().GetEnumerator();
netTimeStopwatch.Start();
var success = true;
var rowCount = 0;
var aggregateCount = 0;
while (!Context.CancellationTokenSource.IsCancellationRequested)
{
netTimeStopwatch.Stop();
var finished = !enumerator.MoveNext();
netTimeStopwatch.Start();
if (finished)
{
break;
}
var row = enumerator.Current;
rowCount++;
var key = KeyGenerator.Invoke(row);
if (key != lastKey)
{
lastKey = key;
if (group.Count > 0)
{
var aggregates = new List <SlimRow>();
try
{
Operation.TransformGroup(group, () =>
{
var aggregate = new SlimRow();
if (FixColumns != null)
{
foreach (var column in FixColumns)
{
aggregate.SetValue(column.ToColumn, group[0][column.FromColumn]);
}
}
aggregates.Add(aggregate);
return(aggregate);
});
}
catch (Exception ex)
{
var exception = new MemoryAggregationException(this, Operation, group, ex);
Context.AddException(this, exception);
success = false;
break;
}
foreach (var groupRow in group)
{
Context.SetRowOwner(groupRow as IRow, null);
}
group.Clear();
foreach (var aggregate in aggregates)
{
aggregateCount++;
var aggregateRow = Context.CreateRow(this, aggregate);
netTimeStopwatch.Stop();
yield return(aggregateRow);
netTimeStopwatch.Start();
}
}
}
group.Add(row);
}
if (success && group.Count > 0)
{
var aggregates = new List <SlimRow>();
try
{
Operation.TransformGroup(group, () =>
{
var aggregate = new SlimRow();
if (FixColumns != null)
{
foreach (var column in FixColumns)
{
aggregate.SetValue(column.ToColumn, group[0][column.FromColumn]);
}
}
aggregates.Add(aggregate);
return(aggregate);
});
}
catch (Exception ex)
{
var exception = new MemoryAggregationException(this, Operation, group, ex);
Context.AddException(this, exception);
success = false;
}
foreach (var groupRow in group)
{
Context.SetRowOwner(groupRow as IRow, null);
}
group.Clear();
if (success)
{
foreach (var aggregate in aggregates)
{
aggregateCount++;
var aggregateRow = Context.CreateRow(this, aggregate);
netTimeStopwatch.Stop();
yield return(aggregateRow);
netTimeStopwatch.Start();
}
}
}
Context.Log(LogSeverity.Debug, this, "evaluated {RowCount} input rows and created {GroupCount} groups in {Elapsed}",
rowCount, group.Count, InvocationInfo.LastInvocationStarted.Elapsed);
netTimeStopwatch.Stop();
Context.Log(LogSeverity.Debug, this, "created {AggregateCount} aggregates in {Elapsed}/{ElapsedWallClock}",
aggregateCount, InvocationInfo.LastInvocationStarted.Elapsed, netTimeStopwatch.Elapsed);
Context.RegisterProcessInvocationEnd(this, netTimeStopwatch.ElapsedMilliseconds);
}
protected override IEnumerable <IRow> EvaluateImpl(Stopwatch netTimeStopwatch)
{
var groupRows = new List <IReadOnlySlimRow>();
string lastKey = null;
netTimeStopwatch.Stop();
var enumerator = InputProcess.Evaluate(this).TakeRowsAndTransferOwnership().GetEnumerator();
netTimeStopwatch.Start();
var success = true;
var rowCount = 0;
var ignoredRowCount = 0;
var groupCount = 0;
var aggregateCount = 0;
while (!Context.CancellationTokenSource.IsCancellationRequested)
{
netTimeStopwatch.Stop();
var finished = !enumerator.MoveNext();
netTimeStopwatch.Start();
if (finished)
{
break;
}
var row = enumerator.Current;
var apply = false;
if (RowFilter != null)
{
try
{
apply = RowFilter.Invoke(row);
}
catch (Exception ex)
{
Context.AddException(this, ProcessExecutionException.Wrap(this, row, ex));
break;
}
if (!apply)
{
ignoredRowCount++;
netTimeStopwatch.Stop();
yield return(row);
netTimeStopwatch.Start();
continue;
}
}
if (RowTagFilter != null)
{
try
{
apply = RowTagFilter.Invoke(row.Tag);
}
catch (Exception ex)
{
Context.AddException(this, ProcessExecutionException.Wrap(this, row, ex));
break;
}
if (!apply)
{
ignoredRowCount++;
netTimeStopwatch.Stop();
yield return(row);
netTimeStopwatch.Start();
continue;
}
}
rowCount++;
var key = KeyGenerator.Invoke(row);
if (key != lastKey)
{
lastKey = key;
if (groupRows.Count > 0)
{
var aggregates = new List <SlimRow>();
groupCount++;
try
{
Operation.TransformGroup(groupRows, () =>
{
var aggregate = new SlimRow
{
Tag = groupRows[0].Tag
};
if (FixColumns != null)
{
foreach (var column in FixColumns)
{
aggregate[column.Key] = groupRows[0][column.Value ?? column.Key];
}
}
aggregates.Add(aggregate);
return(aggregate);
});
}
catch (Exception ex)
{
var exception = new MemoryAggregationException(this, Operation, groupRows, ex);
Context.AddException(this, exception);
success = false;
break;
}
foreach (var groupRow in groupRows)
{
Context.SetRowOwner(groupRow as IRow, null);
}
groupRows.Clear();
foreach (var aggregate in aggregates)
{
aggregateCount++;
var aggregateRow = Context.CreateRow(this, aggregate);
netTimeStopwatch.Stop();
yield return(aggregateRow);
netTimeStopwatch.Start();
}
}
}
groupRows.Add(row);
}
if (success && groupRows.Count > 0)
{
var aggregates = new List <SlimRow>();
groupCount++;
try
{
Operation.TransformGroup(groupRows, () =>
{
var aggregate = new SlimRow();
if (FixColumns != null)
{
foreach (var col in FixColumns)
{
aggregate[col.Key] = groupRows[0][col.Value ?? col.Key];
}
}
aggregates.Add(aggregate);
return(aggregate);
});
}
catch (Exception ex)
{
var exception = new MemoryAggregationException(this, Operation, groupRows, ex);
Context.AddException(this, exception);
success = false;
}
foreach (var groupRow in groupRows)
{
Context.SetRowOwner(groupRow as IRow, null);
}
groupRows.Clear();
if (success)
{
foreach (var aggregate in aggregates)
{
aggregateCount++;
var aggregateRow = Context.CreateRow(this, aggregate);
netTimeStopwatch.Stop();
yield return(aggregateRow);
netTimeStopwatch.Start();
}
}
}
netTimeStopwatch.Stop();
Context.Log(LogSeverity.Debug, this, "evaluated {RowCount} input rows, created {GroupCount} groups and created {AggregateCount} aggregates in {Elapsed}/{ElapsedWallClock}, ignored: {IgnoredRowCount}",
rowCount, groupCount, aggregateCount, InvocationInfo.LastInvocationStarted.Elapsed, netTimeStopwatch.Elapsed, ignoredRowCount);
Context.RegisterProcessInvocationEnd(this, netTimeStopwatch.ElapsedMilliseconds);
}
protected sealed override IEnumerable <IRow> EvaluateImpl(Stopwatch netTimeStopwatch)
{
try
{
StartMutator();
}
catch (Exception ex)
{
Context.AddException(this, ProcessExecutionException.Wrap(this, ex));
netTimeStopwatch.Stop();
Context.RegisterProcessInvocationEnd(this, netTimeStopwatch.ElapsedMilliseconds);
yield break;
}
var mutatedRows = new List <IRow>();
netTimeStopwatch.Stop();
var enumerator = InputProcess.Evaluate(this).TakeRowsAndTransferOwnership().GetEnumerator();
netTimeStopwatch.Start();
var mutatedRowCount = 0;
var ignoredRowCount = 0;
while (!Context.CancellationTokenSource.IsCancellationRequested)
{
netTimeStopwatch.Stop();
var finished = !enumerator.MoveNext();
netTimeStopwatch.Start();
if (finished)
{
break;
}
var row = enumerator.Current;
var apply = false;
if (RowFilter != null)
{
try
{
apply = RowFilter.Invoke(row);
}
catch (Exception ex)
{
Context.AddException(this, ProcessExecutionException.Wrap(this, row, ex));
break;
}
if (!apply)
{
ignoredRowCount++;
netTimeStopwatch.Stop();
yield return(row);
netTimeStopwatch.Start();
continue;
}
}
if (RowTagFilter != null)
{
try
{
apply = RowTagFilter.Invoke(row.Tag);
}
catch (Exception ex)
{
Context.AddException(this, ProcessExecutionException.Wrap(this, row, ex));
break;
}
if (!apply)
{
ignoredRowCount++;
netTimeStopwatch.Stop();
yield return(row);
netTimeStopwatch.Start();
continue;
}
}
mutatedRowCount++;
var kept = false;
try
{
foreach (var mutatedRow in MutateRow(row))
{
if (mutatedRow == row)
{
kept = true;
}
if (mutatedRow.CurrentProcess != this)
{
Context.AddException(this, new ProcessExecutionException(this, mutatedRow, "mutator returned a row without proper ownership"));
break;
}
mutatedRows.Add(mutatedRow);
}
}
catch (Exception ex)
{
Context.AddException(this, ProcessExecutionException.Wrap(this, row, ex));
break;
}
if (!kept)
{
Context.SetRowOwner(row, null);
}
netTimeStopwatch.Stop();
foreach (var mutatedRow in mutatedRows)
{
yield return(mutatedRow);
}
netTimeStopwatch.Start();
mutatedRows.Clear();
}
try
{
CloseMutator();
}
catch (Exception ex)
{
Context.AddException(this, ProcessExecutionException.Wrap(this, ex));
netTimeStopwatch.Stop();
Context.RegisterProcessInvocationEnd(this, netTimeStopwatch.ElapsedMilliseconds);
yield break;
}
netTimeStopwatch.Stop();
if (mutatedRowCount + ignoredRowCount > 0)
{
Context.Log(LogSeverity.Debug, this, "mutated {MutatedRowCount} of {TotalRowCount} rows in {Elapsed}/{ElapsedWallClock}",
mutatedRowCount, mutatedRowCount + ignoredRowCount, InvocationInfo.LastInvocationStarted.Elapsed, netTimeStopwatch.Elapsed);
}
Context.RegisterProcessInvocationEnd(this, netTimeStopwatch.ElapsedMilliseconds);
}
protected override IEnumerable <IRow> EvaluateImpl(Stopwatch netTimeStopwatch)
{
var groups = new Dictionary <string, List <IReadOnlySlimRow> >();
netTimeStopwatch.Stop();
var enumerator = InputProcess.Evaluate(this).TakeRowsAndTransferOwnership().GetEnumerator();
netTimeStopwatch.Start();
var rowCount = 0;
var ignoredRowCount = 0;
while (!Context.CancellationTokenSource.IsCancellationRequested)
{
netTimeStopwatch.Stop();
var finished = !enumerator.MoveNext();
netTimeStopwatch.Start();
if (finished)
{
break;
}
var row = enumerator.Current;
var apply = false;
try
{
apply = If?.Invoke(row) != false;
}
catch (Exception ex)
{
Context.AddException(this, ProcessExecutionException.Wrap(this, row, ex));
break;
}
if (!apply)
{
ignoredRowCount++;
netTimeStopwatch.Stop();
yield return(row);
netTimeStopwatch.Start();
continue;
}
rowCount++;
var key = KeyGenerator.Invoke(row);
if (!groups.TryGetValue(key, out var list))
{
list = new List <IReadOnlySlimRow>();
groups.Add(key, list);
}
list.Add(row);
}
Context.Log(LogSeverity.Debug, this, "evaluated {RowCount} input rows and created {GroupCount} groups in {Elapsed}",
rowCount, groups.Count, InvocationInfo.LastInvocationStarted.Elapsed);
var aggregateCount = 0;
var aggregates = new List <SlimRow>();
foreach (var group in groups.Values)
{
if (Context.CancellationTokenSource.IsCancellationRequested)
{
break;
}
try
{
Operation.TransformGroup(group, () =>
{
var aggregate = new SlimRow();
if (FixColumns != null)
{
foreach (var column in FixColumns)
{
aggregate.SetValue(column.ToColumn, group[0][column.FromColumn]);
}
}
aggregates.Add(aggregate);
return(aggregate);
});
}
catch (Exception ex)
{
var exception = new MemoryAggregationException(this, Operation, group, ex);
Context.AddException(this, exception);
break;
}
foreach (var row in group)
{
Context.SetRowOwner(row as IRow, null);
}
foreach (var aggregate in aggregates)
{
aggregateCount++;
var aggregateRow = Context.CreateRow(this, aggregate);
netTimeStopwatch.Stop();
yield return(aggregateRow);
netTimeStopwatch.Start();
}
group.Clear();
aggregates.Clear();
}
groups.Clear();
netTimeStopwatch.Stop();
Context.Log(LogSeverity.Debug, this, "created {AggregateCount} aggregates in {Elapsed}/{ElapsedWallClock}, ignored: {IgnoredRowCount}",
aggregateCount, InvocationInfo.LastInvocationStarted.Elapsed, netTimeStopwatch.Elapsed, ignoredRowCount);
Context.RegisterProcessInvocationEnd(this, netTimeStopwatch.ElapsedMilliseconds);
}
protected virtual void OnProcess(string numer)
{
InputProcess?.Invoke(numer);
}
// Use this for initialization
void Start()
{
data = mgr.GetComponent <InputProcess> ();
}
// Use this for initialization
void Start()
{
offsizeArr = off_obj.Length;
onsizeArr = on_obj.Length;
data = GameObject.FindGameObjectWithTag("GameManager").GetComponent <InputProcess> ();
}
// Use this for initialization
void Start()
{
data = GameObject.FindGameObjectWithTag("GameManager").GetComponent <InputProcess> ();
}
void Start()
{
data = GameObject.FindGameObjectWithTag("GameManager").GetComponent <InputProcess> ();
anim = GetComponent <Animation> ();
anim.Play("Looking_around");
}
