public void DistributedListHandlesBigCollections()
{
var storage = new RedisStorage(Host);
var key = Guid.NewGuid().ToString();
var list = new DistributedList<int>(storage, key);
int amount = 100000;
var localList = new List<int>(amount);
for (int i = 0; i < amount;i++ )
{
localList.Add(i);
}
list.AddRange(localList);
list.Count.ShouldBe(amount);
list.Clear();
list.Count.ShouldBe(0);
}
private static void RunTest(ConnectionManager connectionManager, ThreadNumberScheduler scheduler, Options options)
{
Log.TraceMessage(Log.Activity.Custom, "Creating data");
var initializeDataThread = DistributedThread.Create(
new Func<int, string, IBluepathCommunicationFramework, int>(
(dataSize, key, bluepath) =>
{
var data = new List<string>(dataSize);
var list = new DistributedList<string>(bluepath.Storage as IExtendedStorage, key);
if (list.Count != dataSize)
{
list.Clear();
}
else
{
return dataSize;
}
for (int i = 0; i < dataSize; i++)
{
int nextSourceDocument = i % SourceDocuments.Documents.Count;
data.Add(SourceDocuments.Documents[nextSourceDocument]);
}
Console.WriteLine("Start saving data to redis");
list.AddRange(data);
return dataSize;
}),
connectionManager, scheduler, DistributedThread.ExecutorSelectionMode.LocalOnly);
var inputDataKey = "distributedPrefixData";
var numberOfElements = options.NoOfElements;
initializeDataThread.Start(numberOfElements, inputDataKey);
initializeDataThread.Join();
var expectedSum = (int)initializeDataThread.Result;
var numberOfShards = options.NoOfShards;
var elementsPerShard = numberOfElements / numberOfShards;
var threads = new List<DistributedThread>();
Log.TraceMessage(Log.Activity.Custom, "Running test");
var sw = new Stopwatch();
sw.Start();
for (int i = 0; i < numberOfShards; i++)
{
int startIndex = i * elementsPerShard;
int endIndex;
if (i == numberOfShards - 1)
{
// last element
endIndex = numberOfElements;
}
else
{
endIndex = startIndex + elementsPerShard;
}
var thread = DistributedThread.Create(
new Func<bool, string, int, int, IBluepathCommunicationFramework, byte[]>(
(returnPrefixes, inputKey, indexStart, indexEnd, bluepath)
=>
{
var inputList = new DistributedList<string>(bluepath.Storage as IExtendedStorage, inputKey);
var inputToProcess = new string[indexEnd - indexStart];
inputList.CopyPartTo(indexStart, inputToProcess.Length, inputToProcess);
List<string> results = new List<string>();
for (int x = indexStart; x < indexEnd; x++)
{
var documentLine = inputToProcess[x - indexStart];
var words = documentLine.Split(' ');
var partialResult = new List<string>();
foreach (var word in words)
{
var stringToProcess = word;
for (int si = 0; si < stringToProcess.Length; si++)
{
string res = "";
if (si == stringToProcess.Length - 1)
{
res = stringToProcess;
}
else
{
res = stringToProcess.Substring(0, si + 1);
}
partialResult.Add(res);
}
}
results.AddRange(partialResult);
if (results.Count > 100000)
{
results.Clear();
}
}
if (returnPrefixes)
{
return new PrefixesResult()
{
Prefixes = results
}.Serialize();
}
else
{
return new PrefixesResult().Serialize();
}
}),
connectionManager,
scheduler
);
thread.Start(options.ReturnPrefixes == 1, inputDataKey, startIndex, endIndex);
threads.Add(thread);
}
foreach (var thread in threads)
{
thread.Join();
if (thread.State == Executor.ExecutorState.Faulted)
{
Console.WriteLine("Err");
}
// Could process prefixes
}
sw.Stop();
//var clearDataThread = DistributedThread.Create(
// new Func<string, IBluepathCommunicationFramework, int>(
// (key, bluepath) =>
// {
// var list = new DistributedList<int>(bluepath.Storage as IExtendedStorage, key);
// list.Clear();
// return 0;
// }),
// connectionManager, scheduler, DistributedThread.ExecutorSelectionMode.LocalOnly);
//clearDataThread.Start(inputDataKey);
//clearDataThread.Join();
Console.WriteLine(sw.ElapsedMilliseconds);
}
private static void RunTest(ConnectionManager connectionManager, ThreadNumberScheduler scheduler, Options options)
{
Log.TraceMessage(Log.Activity.Custom, "Creating data");
var initializeDataThread = DistributedThread.Create(
new Func<int, string, IBluepathCommunicationFramework, int>(
(dataSize, key, bluepath) =>
{
var data = new List<int>(dataSize);
for (int i = 0; i < dataSize; i++)
{
data.Add(1);
}
var list = new DistributedList<int>(bluepath.Storage as IExtendedStorage, key);
if (list.Count != dataSize)
{
list.Clear();
list.AddRange(data);
}
return dataSize;
}),
connectionManager, scheduler, DistributedThread.ExecutorSelectionMode.LocalOnly);
var inputDataKey = "inputData";
var numberOfElements = options.NoOfElements;
initializeDataThread.Start(numberOfElements, inputDataKey);
initializeDataThread.Join();
var expectedSum = (int)initializeDataThread.Result;
var numberOfShards = options.NoOfShards;
var elementsPerShard = numberOfElements / numberOfShards;
var threads = new List<DistributedThread>();
Log.TraceMessage(Log.Activity.Custom, "Running test");
var sw = new Stopwatch();
sw.Start();
for (int i = 0; i < numberOfShards; i++)
{
int startIndex = i * elementsPerShard;
int endIndex;
if (i == numberOfShards - 1)
{
// last element
endIndex = numberOfElements;
}
else
{
endIndex = startIndex + elementsPerShard;
}
var thread = DistributedThread.Create(
new Func<string, int, int, IBluepathCommunicationFramework, int>(
(inputKey, indexStart, indexEnd, bluepath)
=>
{
var inputList = new DistributedList<int>(bluepath.Storage as IExtendedStorage, inputKey);
int partialSum = 0;
for (int x = indexStart; x < indexEnd; x++)
{
partialSum += inputList[x];
//System.Threading.Thread.Sleep(1000);
}
return partialSum;
}),
connectionManager,
scheduler
);
thread.Start(inputDataKey, startIndex, endIndex);
threads.Add(thread);
}
int overallSum = 0;
foreach (var thread in threads)
{
thread.Join();
if (thread.State == Executor.ExecutorState.Faulted)
{
Console.WriteLine("Err");
}
overallSum += (int)thread.Result;
}
sw.Stop();
//var clearDataThread = DistributedThread.Create(
// new Func<string, IBluepathCommunicationFramework, int>(
// (key, bluepath) =>
// {
// var list = new DistributedList<int>(bluepath.Storage as IExtendedStorage, key);
// list.Clear();
// return 0;
// }),
// connectionManager, scheduler, DistributedThread.ExecutorSelectionMode.LocalOnly);
//clearDataThread.Start(inputDataKey);
//clearDataThread.Join();
Console.WriteLine(overallSum);
Console.WriteLine(sw.ElapsedMilliseconds);
}
public static void SaveXes(string fileName, string caseName = null, bool clearListAfterSave = false)
{
var storage = new RedisStorage(DistributedMemoryHost);
var list = new DistributedList<EventType>(storage, XesLogKey);
var @case = new TraceType(caseName ?? Guid.NewGuid().ToString(), list);
LogType log = null;
if (!File.Exists(fileName))
{
log = LogType.Create(new[] { @case });
}
else
{
using (StreamReader reader = new StreamReader(fileName))
{
log = LogType.Deserialize(reader.BaseStream);
}
var tracesList = log.trace.ToList();
tracesList.Add(@case);
log.trace = tracesList.ToArray();
}
using (System.IO.StreamWriter file = new System.IO.StreamWriter(fileName))
{
log.Serialize(file.BaseStream);
}
if (clearListAfterSave)
{
list.Clear();
}
}
public static void ClearXes()
{
var storage = new RedisStorage(DistributedMemoryHost);
var list = new DistributedList<EventType>(storage, XesLogKey);
list.Clear();
}
static void Main(string[] args)
{
Program.Prefixes = new Dictionary<string, List<string>>();
var options = new Options();
if (CommandLine.Parser.Default.ParseArguments(args, options))
{
Bluepath.Log.DistributedMemoryHost = options.RedisHost;
Bluepath.Log.WriteInfoToConsole = false;
Bluepath.Log.WriteToDistributedMemory = false;
var bluepathListener = Bluepath.Services.BluepathListener.InitializeDefaultListener(options.Ip, options.Port);
using (var serviceDiscoveryClient = new CentralizedDiscovery.Client.CentralizedDiscovery(
new ServiceUri(options.CentralizedDiscoveryURI, BindingType.BasicHttpBinding),
bluepathListener
))
{
using (var connectionManager = new ConnectionManager(
remoteService: null,
listener: bluepathListener,
serviceDiscovery: serviceDiscoveryClient,
serviceDiscoveryPeriod: TimeSpan.FromSeconds(1)))
{
System.Threading.Thread.Sleep(1500);
string command = string.Empty;
string sharedStorageKey = "loadlist";
var sharedCounterKey = "counter";
Console.WriteLine("Initializing Redis");
var storage = new RedisStorage(options.RedisHost);
var list = new DistributedList<string>(storage, sharedStorageKey);
Console.WriteLine("List count: {0}", list.Count);
var commands = new Dictionary<string, Action>()
{
{"LOAD", ()=>
{
var sw = new Stopwatch();
sw.Start();
var scheduler = new RoundRobinLocalScheduler(connectionManager.RemoteServices.Select(s=>s.Key).ToArray());
list.Clear();
var counter = new DistributedCounter(storage, sharedCounterKey);
counter.SetValue(0);
var localList = new List<string>();
foreach(var file in Directory.EnumerateFiles(options.InputFolder))
{
var fileContent = File.ReadAllText(file);
localList.Add(fileContent);
}
list.AddRange(localList);
localList.Clear();
var servicesCount = connectionManager.RemoteServices.Count;
if(servicesCount==0)
{
servicesCount = 1;
}
var calculatedChunkSize = (int)Math.Floor(Convert.ToDouble(list.Count) / servicesCount);
if(calculatedChunkSize==0)
{
calculatedChunkSize = 1;
}
var threads = new List<DistributedThread>();
for(int i=0;i<servicesCount;i++)
{
var loadThread = DistributedThread.Create(
new Func<string, string, int, IBluepathCommunicationFramework, int>(
(inputKey, counterKey, chunkSize, bluepath) =>
{
return LoadDocuments(inputKey, counterKey, chunkSize, bluepath);
}), connectionManager, scheduler);
loadThread.Start(sharedStorageKey, sharedCounterKey, calculatedChunkSize);
threads.Add(loadThread);
}
foreach (var thread in threads)
{
thread.Join();
}
list.Clear();
sw.Stop();
Console.WriteLine("Loaded in {0}ms", sw.ElapsedMilliseconds);
}},
{"SEARCH", ()=>
{
var services = connectionManager.RemoteServices.Select(s => s.Key).ToArray();
var scheduler = new RoundRobinLocalScheduler(services);
var threads = new List<DistributedThread>();
Console.Write("Word part: ");
var query = Console.ReadLine();
var sw = new Stopwatch();
sw.Start();
for(int i = 0;(i<services.Length) || i < 1;i++)
{
var searchThread = DistributedThread.Create(
new Func<string,string[]>((searchPhraze) =>
{
List<string> result = new List<string>();
lock(Program.prefixesDictionaryLock)
{
if(Program.Prefixes.ContainsKey(searchPhraze))
{
result = Program.Prefixes[searchPhraze];
}
}
return result.ToArray();
}), null, scheduler);
searchThread.Start(query);
threads.Add(searchThread);
}
var joinedResult = new List<string>();
foreach (var thread in threads)
{
thread.Join();
joinedResult.AddRange(thread.Result as string[]);
}
var endResult = joinedResult.Distinct().ToArray();
sw.Stop();
Console.WriteLine();
Console.WriteLine(string.Join("||", endResult));
Console.WriteLine("Found in {0}ms", sw.ElapsedMilliseconds);
}},
{"CLEAN", ()=>
{
var services = connectionManager.RemoteServices.Select(s => s.Key).ToArray();
var scheduler = new RoundRobinLocalScheduler(services);
var threads = new List<DistributedThread>();
for(int i = 0;(i<services.Length) || i < 1;i++)
{
var searchThread = DistributedThread.Create(
new Func<int>(() =>
{
List<string> result = null;
lock(Program.prefixesDictionaryLock)
{
Program.Prefixes.Clear();
}
return 0;
}), null, scheduler);
searchThread.Start();
threads.Add(searchThread);
}
foreach (var thread in threads)
{
thread.Join();
}
}}
};
do
{
Console.WriteLine("Available commands:");
foreach (var key in commands.Keys)
{
Console.WriteLine(key);
}
Console.Write("Command['q' to exit]: ");
command = Console.ReadLine().ToUpper();
if (commands.ContainsKey(command))
{
commands[command]();
}
} while (command != "Q");
}
}
}
}