public void OrderedParallelPipelineTest()
{
Random r = new Random((int)((System.Diagnostics.Stopwatch.GetTimestamp() >> 3) & 0x7FFFFFFF));
int COUNT = 64;
int[] delays = new int[COUNT];
int[] delays2 = new int[COUNT];
foreach (int i in Enumerable.Range(0, COUNT))
{
delays[i] = 50 + r.Next(200) + (r.Next(5) == 0 ? 500 : 0);
delays2[i] = 50 + r.Next(200) + (r.Next(5) == 0 ? 500 : 0);
}
Func <int, Task <long> > proc = async delegate(int w)
{
System.Diagnostics.Debug.WriteLine($"proc begin wait for {w}");
await Task.Delay(delays[w]);
System.Diagnostics.Debug.WriteLine($"proc end wait for {w}");
return((long)w);
};
Func <long, Task <bool> > predicate = async delegate(long l)
{
System.Diagnostics.Debug.WriteLine($"predicate begin wait for {l}");
await Task.Delay(delays[(int)l]);
System.Diagnostics.Debug.WriteLine($"predicate end wait for {l}");
return((l & 2L) == 0L);
};
ParallelWorker pWorker = new ParallelWorker(3);
var x = Enumerable.Range(0, COUNT)
.AsQueueSource(5)
.OrderedParallelSelect(pWorker, proc, 5)
.OrderedParallelWhere(pWorker, predicate, 5)
.AsEnumerable();
int actualCount = 0;
long?oldI = null;
foreach (long i in x)
{
++actualCount;
System.Diagnostics.Debug.WriteLine($"Received {i}");
Assert.IsTrue(!oldI.HasValue || oldI.Value < i);
oldI = i;
}
Assert.AreEqual(COUNT / 2, actualCount);
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="physicalMapSize">Physical map size in meters</param>
/// <param name="holeMapSize">Hole map size in pixels</param>
/// <param name="obstacleMapSize">Obstacle map size in pixels</param>
/// <param name="startPose">Start pose</param>
/// <param name="sigmaXY">Sigma XY in meters</param>
/// <param name="sigmaTheta">Sigma theta in radians</param>
/// <param name="iterationsPerThread">Search iterations per thread</param>
/// <param name="numSearchThreads">Number of search threads (1 or less for no threading)</param>
public CoreSLAMProcessor(float physicalMapSize, int holeMapSize, int obstacleMapSize, Vector3 startPose,
float sigmaXY, float sigmaTheta, int iterationsPerThread, int numSearchThreads)
{
// Set properties
PhysicalMapSize = physicalMapSize;
this.startPose = startPose;
SigmaXY = sigmaXY;
SigmaTheta = sigmaTheta;
SearchIterationsPerThread = iterationsPerThread;
NumSearchThreads = numSearchThreads;
// Create maps
HoleMap = new HoleMap(holeMapSize, physicalMapSize);
ObstacleMap = new ObstacleMap(obstacleMapSize, physicalMapSize);
noHitMap = new bool[obstacleMapSize, obstacleMapSize];
// Use 3rd party library for fast normal distribution random number generator
samplerXY = new ZigguratGaussianSampler(0.0f, sigmaXY);
samplerTheta = new ZigguratGaussianSampler(0.0f, sigmaTheta);
// Reset everything
Reset();
// Threaded mode or not ?
if (numSearchThreads <= 0)
{
worker = null;
}
else
{
worker = new ParallelWorker(NumSearchThreads, "CoreSLAM search");
randomQueuesXY = new Queue <float> [NumSearchThreads];
randomQueuesTheta = new Queue <float> [NumSearchThreads];
// Create random number queues and fill them up
for (int i = 0; i < NumSearchThreads; i++)
{
randomQueuesXY[i] = new Queue <float>();
randomQueuesTheta[i] = new Queue <float>();
FillRandomQueues(i);
}
}
}
public override void onMetadata(Metadata metadata)
{
_userGroupId = -1;
ParallelWorker worker = new ParallelWorker(8);
bool found = false;
for (int it = 0; it < _statistics.Count && !found; it++)
{
int i = it;
worker.Queue(delegate
{
if (_statistics[i].users.Contains(metadata.userId))
{
_userGroupId = i;
found = true;
}
});
}
metadata.WriteToStream(_writer);
worker.Wait();
}
public void ParallelMultiSourceForEachExceptionTest()
{
object rSyncRoot = new object();
Random r = new Random((int)((System.Diagnostics.Stopwatch.GetTimestamp() >> 3) & 0x7FFFFFFF));
ExceptionCollector ec = new ExceptionCollector();
AsyncQueue <int> q1 = new AsyncQueue <int>(5);
Func <Task> t1 = async delegate()
{
for (int i = 0; i < 100; ++i)
{
await q1.Enqueue(i, ec.CancellationToken);
}
q1.WriteEof();
};
AsyncQueue <int> q2 = new AsyncQueue <int>(5);
Func <Task> t2 = async delegate()
{
for (int i = 100; i < 200; ++i)
{
await q2.Enqueue(i, ec.CancellationToken);
}
q2.WriteEof();
};
AsyncQueue <int> q3 = new AsyncQueue <int>(5);
ParallelWorker pw = new ParallelWorker(8);
Func <Task> t3 = WorkerTask.ParallelForEach
(
new IQueueSource <int>[] { q1, q2 },
InputPriorities.RoundRobin,
pw,
async fei =>
{
await q3.Enqueue(fei.Item, fei.CancellationToken);
if (fei.Item == 133)
{
throw new InvalidOperationException("test");
}
int time;
lock (rSyncRoot)
{
time = 100 + r.Next(800);
}
await Task.Delay(time);
},
() =>
{
q3.WriteEof();
return(Task.CompletedTask);
},
ec
);
Func <Task> t4 = WorkerTask.ForEach
(
q3,
fei =>
{
System.Diagnostics.Debug.WriteLine(fei.Item);
return(Task.CompletedTask);
},
null,
ec
);
Task T1 = Task.Run(t1);
Task T2 = Task.Run(t2);
Task T3 = Task.Run(t3);
Task T4 = Task.Run(t4);
try
{
ec.WaitAll(T1, T2, T3, T4);
}
catch (Exception exc)
{
System.Diagnostics.Debug.WriteLine(exc);
}
}
public void ParallelForEachTest()
{
object rSyncRoot = new object();
Random r = new Random((int)((System.Diagnostics.Stopwatch.GetTimestamp() >> 3) & 0x7FFFFFFF));
ExceptionCollector ec = new ExceptionCollector();
AsyncQueue <int> q1 = new AsyncQueue <int>(5);
Func <Task> t1 = async delegate()
{
for (int i = 0; i < 100; ++i)
{
await q1.Enqueue(r.Next(1000), ec.CancellationToken);
}
q1.WriteEof();
};
AsyncQueue <int> q2 = new AsyncQueue <int>(5);
ParallelWorker pw = new ParallelWorker(8);
Func <Task> t2 = WorkerTask.ParallelForEach
(
q1,
pw,
async delegate(ForEachInfo <int> fi)
{
await q2.Enqueue(fi.Item * 100, fi.CancellationToken);
int time;
lock (rSyncRoot)
{
time = 100 + r.Next(800);
}
await Task.Delay(time);
},
delegate()
{
q2.WriteEof();
return(Task.CompletedTask);
},
ec
);
List <int> items = new List <int>();
Func <Task> t3 = WorkerTask.ForEach
(
q2,
delegate(ForEachInfo <int> fi)
{
items.Add(fi.Item);
System.Diagnostics.Debug.WriteLine(fi.Item);
return(Task.CompletedTask);
},
delegate()
{
return(Task.CompletedTask);
},
ec
);
Task T1 = Task.Run(t1);
Task T2 = Task.Run(t2);
Task T3 = Task.Run(t3);
Task.WaitAll(T1, T2, T3);
Assert.AreEqual(0, ec.Exceptions.Count);
Assert.AreEqual(100, items.Count);
}
public static void DoKMeans(String filename, String output)
{
var watch = Stopwatch.StartNew();
var matrix = MatrixReader.GetMatrix(PathResolver.UserMatrix);
watch.Stop();
Console.WriteLine("Matrix loading:\t" + watch.Elapsed);
var r = new Random();
watch = Stopwatch.StartNew();
List <User> allUsers = User.GetUsers(matrix);
//allUsers.Sort((u1, u2) => { return u1.terms.Count - u2.terms.Count; });
watch.Stop();
Console.WriteLine("Getting users:\t" + watch.Elapsed);
matrix = null;
GC.Collect();
watch = Stopwatch.StartNew();
List <List <User> > boxes = new List <List <User> >();
boxes = getBoxes(allUsers);
watch.Stop();
Console.WriteLine("Putting in boxes:\t" + watch.Elapsed);
#region WYLICZANIE STATYSTYK
{
float avgCount = 0;
float dev = 0;
for (int b = 0; b < boxes.Count; b++)
{
avgCount += boxes[b].Count;
}
avgCount /= boxes.Count;
for (int b = 0; b < boxes.Count; b++)
{
dev += (float)Math.Pow(boxes[b].Count - avgCount, 2);
}
dev = (float)Math.Sqrt(dev / boxes.Count);
Console.WriteLine("Stats of {0} boxes:", boxes.Count);
Console.WriteLine(" AVG(Count):\t" + avgCount + "\tDev:\t" + dev);
}
#endregion WYLICZANIE STATYSTYK
ParallelWorker worker = new ParallelWorker(8);
#region ZAPISYWANIE GRUPY DO PLIKU
using (var writer = new StreamWriter(output))
{
for (int j = 0; j < boxes.Count - 1; j++)
{
foreach (User user in boxes[j])
{
writer.WriteLine(user.userId);
}
writer.WriteLine();
}
}
#endregion ZAPISYWANIE GRUPY DO PLIKU
}
private static List <List <User> > getBoxes(List <User> allUsers2)
{
List <List <User> > allBoxes = new List <List <User> >();
const int N = 100;
for (int b = 0; b < N; b++)
{
int min = b * allUsers2.Count / N;
int max = (b + 1) * allUsers2.Count / N;
var users = allUsers2.GetRange(min, max - min);
var boxes2 = getSmallBoxes(users);
const float MIN_SIM = 0.5f;
ParallelWorker tmpWorker = new ParallelWorker(8);
for (int it = 0; it < boxes2.Count; it++)
{
int i = it;
tmpWorker.Queue(delegate
{
float bestSim = 0;
int bestBox = -1;
int nBoxes;
lock (allBoxes)
{
nBoxes = allBoxes.Count;
}
for (int k = 0; k < nBoxes; k++)
{
float sim = boxes2[i][0].GetSim(allBoxes[k][0]);
if (bestSim < sim)
{
bestSim = sim;
bestBox = k;
}
}
lock (allBoxes)
{
for (int k = nBoxes; k < allBoxes.Count; k++)
{
float sim = boxes2[i][0].GetSim(allBoxes[k][0]);
if (bestSim < sim)
{
bestSim = sim;
bestBox = k;
}
}
if (bestSim < MIN_SIM)
{
bestBox = allBoxes.Count;
allBoxes.Add(new List <User>());
}
allBoxes[bestBox].AddRange(boxes2[i]);
}
});
}
tmpWorker.Wait();
Console.WriteLine("Done {0}%", 100.0f * b / N);
}
return(allBoxes);
}
private static List <List <User> > getSmallBoxes(List <User> allUsers)
{
allUsers.Sort((u1, u2) => { return(u1.terms.Count - u2.terms.Count); });
List <List <User> > boxes = new List <List <User> >();
ParallelWorker tmpWorker = new ParallelWorker(8);
const float MIN_SIM = 0.7f;
for (int it = 0; it < allUsers.Count; it++)
{
int i = it;
tmpWorker.Queue(delegate
{
float bestSim = 0;
int bestBox = -1;
int nBoxes;
lock (boxes)
{
nBoxes = boxes.Count;
}
for (int k = 0; k < nBoxes; k++)
{
float sim = allUsers[i].GetSim(boxes[k][0]);
if (bestSim < sim)
{
bestSim = sim;
bestBox = k;
}
}
lock (boxes)
{
for (int k = nBoxes; k < boxes.Count; k++)
{
float sim = allUsers[i].GetSim(boxes[k][0]);
if (bestSim < sim)
{
bestSim = sim;
bestBox = k;
}
}
if (bestSim < MIN_SIM)
{
bestBox = boxes.Count;
boxes.Add(new List <User>());
}
boxes[bestBox].Add(allUsers[i]);
}
});
if (it % 1000 == 0)
{
Console.Write("Users: {0:0.00}%\t({1})\r", 100.0f * it / allUsers.Count, boxes.Count);
}
}
tmpWorker.Wait();
return(boxes);
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="numThreads">Number of calculation threads to use</param>
/// <param name="logger">Logging interface</param>
public ScanMatcher(int numThreads, ILogger logger = null)
{
this.logger = logger;
worker = new ParallelWorker(numThreads, "HectorSLAM estimator");
}
