private IEnumerable <Word> ParallelSynthesize(IEnumerable <Word> analyses)
{
var validWordsStack = new ConcurrentStack <Word>();
Exception exception = null;
Parallel.ForEach(analyses, (analysisWord, state) =>
{
try
{
foreach (Word synthesisWord in LexicalLookup(analysisWord))
{
Word[] valid = _synthesisRule.Apply(synthesisWord).Where(IsWordValid).ToArray();
if (valid.Length > 0)
{
validWordsStack.PushRange(valid);
}
}
}
catch (Exception e)
{
state.Stop();
exception = e;
}
});
if (exception != null)
{
throw exception;
}
return(validWordsStack.Distinct(FreezableEqualityComparer <Word> .Default));
}
public int SortAndWriteToFile()
{
var sortedCollection = _collection.Distinct().OrderBy(x => x.Key);
File.WriteAllLines(Common.Common.OutputFileName, sortedCollection.Select(s => s.Value.ToString()).ToList());
return(sortedCollection.Count());
}
public void MtTest()
{
var gen = new UlidGenTS();
var cl = new ConcurrentStack <Ulid>();
const int c = 10_000_000;
var t1 = new Thread(
() =>
{
var l = new List <Ulid>();
for (var i = 0; i < c; i++)
{
l.Add(gen.Generate());
}
cl.PushRange(l.ToArray());
}
);
var t2 = new Thread(
() =>
{
var l = new List <Ulid>();
for (var i = 0; i < c; i++)
{
l.Add(gen.Generate());
}
cl.PushRange(l.ToArray());
}
);
var t3 = new Thread(
() =>
{
var l = new List <Ulid>();
for (var i = 0; i < c; i++)
{
l.Add(gen.Generate());
}
cl.PushRange(l.ToArray());
}
);
t1.Start();
t2.Start();
t3.Start();
t1.Join();
t2.Join();
t3.Join();
cl.Count
.ShouldBe(c * 3, "Threads PushRange failed. Maybe some threads failed execution");
cl.Distinct()
.Count()
.ShouldBe(c * 3, "Some id's duplicated!");
}
public override IEnumerable <TData> Apply(TData input)
{
var output = new ConcurrentStack <TData>();
Parallel.ForEach(Rules, rule =>
{
TData[] outData = rule.Apply(input).ToArray();
if (outData.Length > 0)
{
output.PushRange(outData);
}
});
return(output.Distinct(Comparer));
}
/// <summary>
/// Verifies received messages against expected messages.
/// </summary>
/// <param name="cluster">Cluster.</param>
/// <param name="expectedMessages">Expected messages.</param>
/// <param name="resultFunc">Result transform function.</param>
/// <param name="expectedRepeat">Expected repeat count.</param>
public static void VerifyReceive(IClusterGroup cluster, IEnumerable <string> expectedMessages,
Func <IEnumerable <string>, IEnumerable <string> > resultFunc, int expectedRepeat)
{
// check if expected message count has been received; Wait returns false if there were none.
Assert.IsTrue(ReceivedEvent.Wait(MessageTimeout));
expectedMessages = expectedMessages.SelectMany(x => Enumerable.Repeat(x, expectedRepeat));
Assert.AreEqual(expectedMessages, resultFunc(ReceivedMessages));
// check that all messages came from local node.
var localNodeId = cluster.Ignite.GetCluster().GetLocalNode().Id;
Assert.AreEqual(localNodeId, LastNodeIds.Distinct().Single());
AssertFailures();
}
public override IEnumerable <TData> Apply(TData input)
{
var output = new ConcurrentStack <TData>();
var from = new ConcurrentStack <Tuple <TData, HashSet <int> > >();
from.Push(Tuple.Create(input, !MultipleApplication ? new HashSet <int>() : null));
var to = new ConcurrentStack <Tuple <TData, HashSet <int> > >();
while (!from.IsEmpty)
{
to.Clear();
Parallel.ForEach(from, work =>
{
for (int i = 0; i < Rules.Count; i++)
{
if ((work.Item2 == null || !work.Item2.Contains(i)))
{
TData[] results = ApplyRule(Rules[i], i, work.Item1).ToArray();
if (results.Length > 0)
{
output.PushRange(results);
Tuple <TData, HashSet <int> >[] workItems = results.Where(res => !Comparer.Equals(work.Item1, res))
.Select(res => Tuple.Create(res, work.Item2 == null ? null : new HashSet <int>(work.Item2)
{
i
})).ToArray();
if (workItems.Length > 0)
{
to.PushRange(workItems);
}
}
}
}
});
ConcurrentStack <Tuple <TData, HashSet <int> > > temp = from;
from = to;
to = temp;
}
return(output.Distinct(Comparer));
}
/// <summary>
/// Process all data with selected technique from data.txt, transform it with transform.dll and save results time to results.txt
/// </summary>
private static void ProcessFile()
{
var stopwatch = Stopwatch.StartNew();
Console.Write("Processing time: ");
var finalCollection = new Dictionary <int, char>();
try
{
var data = File.ReadAllLines(DataFileName);
switch (_technique)
{
case 1:
{
var numbersFromFile = new ConcurrentStack <uint>();
Parallel.ForEach(data, (line) =>
{
if (UInt32.TryParse(line, out uint uintFromString))
{
numbersFromFile.Push(uintFromString);
}
});
var distinctedNumbersFromFile = numbersFromFile.Distinct().ToList();
_threadCountFinal = _maxRealThreadCount = 0;
// Wait for all tasks to complete.
var tasks = new Task[distinctedNumbersFromFile.Count()];
for (int i = 0; i < distinctedNumbersFromFile.Count(); i++)
{
var line = distinctedNumbersFromFile[i];
tasks[i] = Task.Factory.StartNew(() =>
{
ProcessLine(line, finalCollection);
});
}
Task.WaitAll(tasks);
break;
};
case 2:
{
Parallel.ForEach(data, (line) =>
{
if (UInt32.TryParse(line, out uint uintFromString))
{
ProcessLine(uintFromString, finalCollection);
}
});
break;
}
default:
break;
}
File.WriteAllText(OutputFileName, string.Join("", finalCollection.OrderBy(x => x.Key).Select(s => s.Value)));
stopwatch.Stop();
var processingTime = $"{stopwatch.Elapsed.ToString(@"mm\:ss\:fffffff")}";
if (!File.Exists(ResultsFileName))
{
File.Create(ResultsFileName).Close();
File.AppendAllText(ResultsFileName, $"{nameof(_techniqueName)}, {nameof(_processorCount)}, {nameof(_currentprocessesCount)}, {nameof(_currentThreadsCount)}, {nameof(_minThreadCount)}, {nameof(_maxRealThreadCount)}, {nameof(processingTime)}" + Environment.NewLine);
}
File.AppendAllText(ResultsFileName, $"{_techniqueName}, {_processorCount}, {_currentprocessesCount}, {_currentThreadsCount}, {_minThreadCount}, {_maxRealThreadCount}, {stopwatch.Elapsed.TotalMilliseconds}" + Environment.NewLine);
Console.WriteLine(processingTime);
Console.WriteLine();
Console.WriteLine($"Count of output letters: {finalCollection.Count}");
Console.WriteLine($"Max thread count: {_maxRealThreadCount}");
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
Console.WriteLine(ex.ToString());
Console.WriteLine("EXCEPTION. Couldn't perform this test.");
Console.WriteLine(ex);
throw;
}
finally
{
finalCollection.Clear();
finalCollection = null;
}
}
public IEnumerable <uint> Distinct()
{
return(_collection.Distinct());
}
private IEnumerable<Word> ParallelSynthesize(IEnumerable<Word> analyses)
{
var validWordsStack = new ConcurrentStack<Word>();
Exception exception = null;
Parallel.ForEach(analyses, (analysisWord, state) =>
{
try
{
foreach (Word synthesisWord in LexicalLookup(analysisWord))
{
Word[] valid = _synthesisRule.Apply(synthesisWord).Where(IsWordValid).ToArray();
if (valid.Length > 0)
validWordsStack.PushRange(valid);
}
}
catch (Exception e)
{
state.Stop();
exception = e;
}
});
if (exception != null)
throw exception;
return validWordsStack.Distinct(FreezableEqualityComparer<Word>.Default);
}
public IEnumerable <string> GenerateWords(LexEntry rootEntry, IEnumerable <Morpheme> otherMorphemes,
FeatureStruct realizationalFS, out object trace)
{
Stack <Tuple <IMorphologicalRule, RootAllomorph> >[] rulePermutations = PermuteRules(otherMorphemes.ToArray())
.ToArray();
object rootTrace = _traceManager.IsTracing ? _traceManager.GenerateWords(_lang) : null;
trace = rootTrace;
var validWordsStack = new ConcurrentStack <Word>();
Exception exception = null;
Parallel.ForEach(rootEntry.Allomorphs.SelectMany(a => rulePermutations,
(a, p) => new { Allomorph = a, RulePermutation = p }), (synthesisInfo, state) =>
{
try
{
var synthesisWord = new Word(synthesisInfo.Allomorph, realizationalFS);
foreach (Tuple <IMorphologicalRule, RootAllomorph> rule in synthesisInfo.RulePermutation)
{
synthesisWord.MorphologicalRuleUnapplied(rule.Item1);
if (rule.Item2 != null)
{
synthesisWord.NonHeadUnapplied(new Word(rule.Item2, new FeatureStruct()));
}
}
synthesisWord.CurrentTrace = rootTrace;
if (_traceManager.IsTracing)
{
_traceManager.SynthesizeWord(_lang, synthesisWord);
}
synthesisWord.Freeze();
Word[] valid = _synthesisRule.Apply(synthesisWord).Where(IsWordValid).ToArray();
if (valid.Length > 0)
{
validWordsStack.PushRange(valid);
}
}
catch (Exception e)
{
state.Stop();
exception = e;
}
});
if (exception != null)
{
throw exception;
}
var words = new List <string>();
foreach (Word w in CheckDisjunction(validWordsStack.Distinct(FreezableEqualityComparer <Word> .Default)))
{
if (_traceManager.IsTracing)
{
_traceManager.Successful(_lang, w);
}
words.Add(w.Shape.ToString(_lang.SurfaceStratum.CharacterDefinitionTable, false));
}
return(words);
}