private static void RunTest(ConnectionManager connectionManager, ThreadNumberScheduler scheduler, Options options)
{
Log.TraceMessage(Log.Activity.Custom, "Creating data");
var initializeDataThread = DistributedThread.Create(
new Func<int, string, IBluepathCommunicationFramework, int>(
(dataSize, key, bluepath) =>
{
var data = new List<string>(dataSize);
var list = new DistributedList<string>(bluepath.Storage as IExtendedStorage, key);
if (list.Count != dataSize)
{
list.Clear();
}
else
{
return dataSize;
}
for (int i = 0; i < dataSize; i++)
{
int nextSourceDocument = i % SourceDocuments.Documents.Count;
data.Add(SourceDocuments.Documents[nextSourceDocument]);
}
Console.WriteLine("Start saving data to redis");
list.AddRange(data);
return dataSize;
}),
connectionManager, scheduler, DistributedThread.ExecutorSelectionMode.LocalOnly);
var inputDataKey = "distributedPrefixData";
var numberOfElements = options.NoOfElements;
initializeDataThread.Start(numberOfElements, inputDataKey);
initializeDataThread.Join();
var expectedSum = (int)initializeDataThread.Result;
var numberOfShards = options.NoOfShards;
var elementsPerShard = numberOfElements / numberOfShards;
var threads = new List<DistributedThread>();
Log.TraceMessage(Log.Activity.Custom, "Running test");
var sw = new Stopwatch();
sw.Start();
for (int i = 0; i < numberOfShards; i++)
{
int startIndex = i * elementsPerShard;
int endIndex;
if (i == numberOfShards - 1)
{
// last element
endIndex = numberOfElements;
}
else
{
endIndex = startIndex + elementsPerShard;
}
var thread = DistributedThread.Create(
new Func<bool, string, int, int, IBluepathCommunicationFramework, byte[]>(
(returnPrefixes, inputKey, indexStart, indexEnd, bluepath)
=>
{
var inputList = new DistributedList<string>(bluepath.Storage as IExtendedStorage, inputKey);
var inputToProcess = new string[indexEnd - indexStart];
inputList.CopyPartTo(indexStart, inputToProcess.Length, inputToProcess);
List<string> results = new List<string>();
for (int x = indexStart; x < indexEnd; x++)
{
var documentLine = inputToProcess[x - indexStart];
var words = documentLine.Split(' ');
var partialResult = new List<string>();
foreach (var word in words)
{
var stringToProcess = word;
for (int si = 0; si < stringToProcess.Length; si++)
{
string res = "";
if (si == stringToProcess.Length - 1)
{
res = stringToProcess;
}
else
{
res = stringToProcess.Substring(0, si + 1);
}
partialResult.Add(res);
}
}
results.AddRange(partialResult);
if (results.Count > 100000)
{
results.Clear();
}
}
if (returnPrefixes)
{
return new PrefixesResult()
{
Prefixes = results
}.Serialize();
}
else
{
return new PrefixesResult().Serialize();
}
}),
connectionManager,
scheduler
);
thread.Start(options.ReturnPrefixes == 1, inputDataKey, startIndex, endIndex);
threads.Add(thread);
}
foreach (var thread in threads)
{
thread.Join();
if (thread.State == Executor.ExecutorState.Faulted)
{
Console.WriteLine("Err");
}
// Could process prefixes
}
sw.Stop();
//var clearDataThread = DistributedThread.Create(
// new Func<string, IBluepathCommunicationFramework, int>(
// (key, bluepath) =>
// {
// var list = new DistributedList<int>(bluepath.Storage as IExtendedStorage, key);
// list.Clear();
// return 0;
// }),
// connectionManager, scheduler, DistributedThread.ExecutorSelectionMode.LocalOnly);
//clearDataThread.Start(inputDataKey);
//clearDataThread.Join();
Console.WriteLine(sw.ElapsedMilliseconds);
}
public void DistributedListsPerformsCopyPartToArray()
{
var storage = new RedisStorage(Host);
var key = Guid.NewGuid().ToString();
var list1 = new DistributedList<int>(storage, key);
int elementCount = 10;
for (int i = 0; i < elementCount; i++)
{
list1.Add(i);
}
list1.Count.ShouldBe(elementCount);
int startIndex = 3;
int count = 3;
var destinationArray = new int[count];
list1.CopyPartTo(startIndex, count, destinationArray);
for (int i = 0; i < destinationArray.Length; i++)
{
destinationArray[i].ShouldBe(i + startIndex);
}
}
private static int LoadDocuments(string inputKey, string counterKey, int chunkSize, IBluepathCommunicationFramework bluepath)
{
var inputList = new DistributedList<string>(bluepath.Storage as IExtendedStorage, inputKey);
var inputCount = inputList.Count;
var counter = new DistributedCounter(bluepath.Storage as IExtendedStorage, counterKey);
int indexEnd = 0;
do
{
int noOfElements = chunkSize;
int indexStart = counter.GetAndIncrease(chunkSize);
if (indexStart >= inputCount)
{
break;
}
indexEnd = indexStart + noOfElements;
if (indexEnd > inputCount)
{
indexEnd = inputCount;
noOfElements = indexEnd - indexStart;
}
var inputDocuments = new string[noOfElements];
inputList.CopyPartTo(indexStart, noOfElements, inputDocuments);
foreach (var document in inputDocuments)
{
var words = document.Split(' ');
foreach (var word in words)
{
if (word.Length == 0)
{
continue;
}
var stringToProcess = word;
var partialResult = new List<string>(stringToProcess.Length);
for (int si = 0; si < stringToProcess.Length; si++)
{
string res = "";
if (si == stringToProcess.Length - 1)
{
res = stringToProcess;
}
else
{
res = stringToProcess.Substring(0, si + 1);
}
partialResult.Add(res);
}
lock (Program.prefixesDictionaryLock)
{
foreach (var prefix in partialResult)
{
if (Program.Prefixes.ContainsKey(prefix))
{
if (!Program.Prefixes[prefix].Contains(word))
{
Program.Prefixes[prefix].Add(word);
}
}
else
{
Program.Prefixes.Add(prefix, new List<string>() { word });
}
}
}
}
}
} while (indexEnd <= inputCount);
return 0;
}