private static void RunRedisTest(Options options)
{
var storage = new Bluepath.Storage.Redis.RedisStorage(options.RedisHost);
var list = new DistributedList<int>(storage, Guid.NewGuid().ToString());
var localList = new List<int>();
var rSw = new Stopwatch();
int amount = 1000;
rSw.Start();
for (int i = 0; i < amount; i++)
{
localList.Add(1);
}
list.AddRange(localList);
rSw.Stop();
var wSw = new Stopwatch();
int aa = 0;
wSw.Start();
for (int i = 0; i < amount; i++)
{
aa = list[i];
}
wSw.Stop();
Log.TraceMessage(Log.Activity.Custom, string.Format("Write: {0}; Read: {1}; Value {2}; Amount: {3}", rSw.ElapsedMilliseconds, wSw.ElapsedMilliseconds, aa, amount));
}
public void OpenXesSerializationTest()
{
var storage = new RedisStorage(Host);
var key = Guid.NewGuid().ToString();
var list = new DistributedList<EventType>(storage, key);
list.Add(new EventType("Start", "Start", DateTime.Now, EventType.Transition.Start));
list.Add(new EventType("Start", "Start", DateTime.Now.AddSeconds(1), EventType.Transition.Complete));
list.Add(new EventType("Phone Call", "Helen", DateTime.Now.AddSeconds(2), EventType.Transition.Start));
list.Add(new EventType("Phone Call", "Helen", DateTime.Now.AddSeconds(3), EventType.Transition.Complete));
list.Add(new EventType("End", "End", DateTime.Now.AddSeconds(4), EventType.Transition.Start));
list.Add(new EventType("End", "End", DateTime.Now.AddSeconds(5), EventType.Transition.Complete));
storage = new RedisStorage(Host);
list = new DistributedList<EventType>(storage, key);
var localList = list.ToList();
localList.Count.ShouldBe(6);
localList.Where(e => e != null).Count().ShouldBe(6);
var @case = new TraceType(Guid.NewGuid().ToString(), list);
var log = LogType.Create(new[] { @case });
var xml = log.Serialize();
Assert.IsTrue(xml.Contains("Phone Call"));
}
public void DistributedListAllowsAddingEnumerablesInOneStep()
{
var storage = new RedisStorage(Host);
var key = Guid.NewGuid().ToString();
var list = new DistributedList<int>(storage, key);
var localList = new List<int>();
for (int i = 0; i < 10; i++)
{
localList.Add(i);
}
list.AddRange(localList);
list.Count.ShouldBe(10);
}
public void DistributedListHandlesBigCollections()
{
var storage = new RedisStorage(Host);
var key = Guid.NewGuid().ToString();
var list = new DistributedList<int>(storage, key);
int amount = 100000;
var localList = new List<int>(amount);
for (int i = 0; i < amount;i++ )
{
localList.Add(i);
}
list.AddRange(localList);
list.Count.ShouldBe(amount);
list.Clear();
list.Count.ShouldBe(0);
}
public void DistributedListAllowsAddingAndRemovingItems()
{
var storage = new RedisStorage(Host);
var key = Guid.NewGuid().ToString();
var list = new DistributedList<int>(storage, key);
for (int i = 0; i < 10; i++)
{
list.Add(i);
}
list.Count.ShouldBe(10);
list.Remove(5);
list.Count.ShouldBe(9);
foreach (var item in list)
{
item.ShouldNotBe(5);
}
}
public void DistributedListsWorksWithClasses()
{
var storage = new RedisStorage(Host);
var key = Guid.NewGuid().ToString();
var list1 = new DistributedList<Bluepath.Tests.Integration.DistributedThread.ComplexParameter>(storage, key);
for (int i = 0; i < 10; i++)
{
list1.Add(new Bluepath.Tests.Integration.DistributedThread.ComplexParameter()
{
AnotherProperty = i,
SomeProperty = string.Format("ala{0}", i)
});
}
list1.Count.ShouldBe(10);
var list2 = new DistributedList<Bluepath.Tests.Integration.DistributedThread.ComplexParameter>(storage, key);
for (int i = 0; i < list2.Count; i++)
{
list2[i].AnotherProperty.ShouldBe(i);
}
}
public void DistributedListsWithTheSameKeyShareState()
{
var storage = new RedisStorage(Host);
var key = Guid.NewGuid().ToString();
var list1 = new DistributedList<int>(storage, key);
for (int i = 0; i < 10; i++)
{
list1.Add(i);
}
list1.Count.ShouldBe(10);
var list2 = new DistributedList<int>(storage, key);
list2.Count.ShouldBe(10);
}
public void DistributedListsPostponeRemovingItemsUntilEnumarationIsFinished()
{
var storage = new RedisStorage(Host);
var key = Guid.NewGuid().ToString();
var list1 = new DistributedList<int>(storage, key);
var synchLock = new object();
bool finishedRemoving = false;
bool isEnumerating = false;
int count = 0;
for (int i = 0; i < 10; i++)
{
list1.Add(i);
}
var enumerationThread = new Thread(() =>
{
var storage1 = new RedisStorage(Host);
var list = new DistributedList<int>(storage1, key);
bool isFirstTurn = true;
int localCount = 0;
foreach (var item in list)
{
localCount++;
isEnumerating = true;
if (isFirstTurn)
{
isFirstTurn = false;
lock (synchLock)
{
Monitor.Wait(synchLock);
}
}
}
count = localCount;
});
var removeItemThread = new Thread(() =>
{
var storage2 = new RedisStorage(Host);
var list = new DistributedList<int>(storage2, key);
list.RemoveAt(4);
finishedRemoving = true;
});
enumerationThread.Start();
TestHelpers.RepeatUntilTrue(() => isEnumerating, times: 5);
removeItemThread.Start();
Thread.Sleep(10);
list1.Count.ShouldBe(10);
finishedRemoving.ShouldBe(false);
lock (synchLock)
{
Monitor.Pulse(synchLock);
}
enumerationThread.Join();
removeItemThread.Join();
finishedRemoving.ShouldBe(true);
list1.Count.ShouldBe(9);
count.ShouldBe(10);
}
private static int LoadDocuments(string inputKey, string counterKey, int chunkSize, IBluepathCommunicationFramework bluepath)
{
var inputList = new DistributedList<string>(bluepath.Storage as IExtendedStorage, inputKey);
var inputCount = inputList.Count;
var counter = new DistributedCounter(bluepath.Storage as IExtendedStorage, counterKey);
int indexEnd = 0;
do
{
int noOfElements = chunkSize;
int indexStart = counter.GetAndIncrease(chunkSize);
if (indexStart >= inputCount)
{
break;
}
indexEnd = indexStart + noOfElements;
if (indexEnd > inputCount)
{
indexEnd = inputCount;
noOfElements = indexEnd - indexStart;
}
var inputDocuments = new string[noOfElements];
inputList.CopyPartTo(indexStart, noOfElements, inputDocuments);
foreach (var document in inputDocuments)
{
var words = document.Split(' ');
foreach (var word in words)
{
if (word.Length == 0)
{
continue;
}
var stringToProcess = word;
var partialResult = new List<string>(stringToProcess.Length);
for (int si = 0; si < stringToProcess.Length; si++)
{
string res = "";
if (si == stringToProcess.Length - 1)
{
res = stringToProcess;
}
else
{
res = stringToProcess.Substring(0, si + 1);
}
partialResult.Add(res);
}
lock (Program.prefixesDictionaryLock)
{
foreach (var prefix in partialResult)
{
if (Program.Prefixes.ContainsKey(prefix))
{
if (!Program.Prefixes[prefix].Contains(word))
{
Program.Prefixes[prefix].Add(word);
}
}
else
{
Program.Prefixes.Add(prefix, new List<string>() { word });
}
}
}
}
}
} while (indexEnd <= inputCount);
return 0;
}
private static void RunTest(ConnectionManager connectionManager, ThreadNumberScheduler scheduler, Options options)
{
Log.TraceMessage(Log.Activity.Custom, "Creating data");
var initializeDataThread = DistributedThread.Create(
new Func<int, string, IBluepathCommunicationFramework, int>(
(dataSize, key, bluepath) =>
{
var data = new List<string>(dataSize);
var list = new DistributedList<string>(bluepath.Storage as IExtendedStorage, key);
if (list.Count != dataSize)
{
list.Clear();
}
else
{
return dataSize;
}
for (int i = 0; i < dataSize; i++)
{
int nextSourceDocument = i % SourceDocuments.Documents.Count;
data.Add(SourceDocuments.Documents[nextSourceDocument]);
}
Console.WriteLine("Start saving data to redis");
list.AddRange(data);
return dataSize;
}),
connectionManager, scheduler, DistributedThread.ExecutorSelectionMode.LocalOnly);
var inputDataKey = "distributedPrefixData";
var numberOfElements = options.NoOfElements;
initializeDataThread.Start(numberOfElements, inputDataKey);
initializeDataThread.Join();
var expectedSum = (int)initializeDataThread.Result;
var numberOfShards = options.NoOfShards;
var elementsPerShard = numberOfElements / numberOfShards;
var threads = new List<DistributedThread>();
Log.TraceMessage(Log.Activity.Custom, "Running test");
var sw = new Stopwatch();
sw.Start();
for (int i = 0; i < numberOfShards; i++)
{
int startIndex = i * elementsPerShard;
int endIndex;
if (i == numberOfShards - 1)
{
// last element
endIndex = numberOfElements;
}
else
{
endIndex = startIndex + elementsPerShard;
}
var thread = DistributedThread.Create(
new Func<bool, string, int, int, IBluepathCommunicationFramework, byte[]>(
(returnPrefixes, inputKey, indexStart, indexEnd, bluepath)
=>
{
var inputList = new DistributedList<string>(bluepath.Storage as IExtendedStorage, inputKey);
var inputToProcess = new string[indexEnd - indexStart];
inputList.CopyPartTo(indexStart, inputToProcess.Length, inputToProcess);
List<string> results = new List<string>();
for (int x = indexStart; x < indexEnd; x++)
{
var documentLine = inputToProcess[x - indexStart];
var words = documentLine.Split(' ');
var partialResult = new List<string>();
foreach (var word in words)
{
var stringToProcess = word;
for (int si = 0; si < stringToProcess.Length; si++)
{
string res = "";
if (si == stringToProcess.Length - 1)
{
res = stringToProcess;
}
else
{
res = stringToProcess.Substring(0, si + 1);
}
partialResult.Add(res);
}
}
results.AddRange(partialResult);
if (results.Count > 100000)
{
results.Clear();
}
}
if (returnPrefixes)
{
return new PrefixesResult()
{
Prefixes = results
}.Serialize();
}
else
{
return new PrefixesResult().Serialize();
}
}),
connectionManager,
scheduler
);
thread.Start(options.ReturnPrefixes == 1, inputDataKey, startIndex, endIndex);
threads.Add(thread);
}
foreach (var thread in threads)
{
thread.Join();
if (thread.State == Executor.ExecutorState.Faulted)
{
Console.WriteLine("Err");
}
// Could process prefixes
}
sw.Stop();
//var clearDataThread = DistributedThread.Create(
// new Func<string, IBluepathCommunicationFramework, int>(
// (key, bluepath) =>
// {
// var list = new DistributedList<int>(bluepath.Storage as IExtendedStorage, key);
// list.Clear();
// return 0;
// }),
// connectionManager, scheduler, DistributedThread.ExecutorSelectionMode.LocalOnly);
//clearDataThread.Start(inputDataKey);
//clearDataThread.Join();
Console.WriteLine(sw.ElapsedMilliseconds);
}
private static void RunTest(ConnectionManager connectionManager, ThreadNumberScheduler scheduler, Options options)
{
Log.TraceMessage(Log.Activity.Custom, "Creating data");
var initializeDataThread = DistributedThread.Create(
new Func<int, string, IBluepathCommunicationFramework, int>(
(dataSize, key, bluepath) =>
{
var data = new List<int>(dataSize);
for (int i = 0; i < dataSize; i++)
{
data.Add(1);
}
var list = new DistributedList<int>(bluepath.Storage as IExtendedStorage, key);
if (list.Count != dataSize)
{
list.Clear();
list.AddRange(data);
}
return dataSize;
}),
connectionManager, scheduler, DistributedThread.ExecutorSelectionMode.LocalOnly);
var inputDataKey = "inputData";
var numberOfElements = options.NoOfElements;
initializeDataThread.Start(numberOfElements, inputDataKey);
initializeDataThread.Join();
var expectedSum = (int)initializeDataThread.Result;
var numberOfShards = options.NoOfShards;
var elementsPerShard = numberOfElements / numberOfShards;
var threads = new List<DistributedThread>();
Log.TraceMessage(Log.Activity.Custom, "Running test");
var sw = new Stopwatch();
sw.Start();
for (int i = 0; i < numberOfShards; i++)
{
int startIndex = i * elementsPerShard;
int endIndex;
if (i == numberOfShards - 1)
{
// last element
endIndex = numberOfElements;
}
else
{
endIndex = startIndex + elementsPerShard;
}
var thread = DistributedThread.Create(
new Func<string, int, int, IBluepathCommunicationFramework, int>(
(inputKey, indexStart, indexEnd, bluepath)
=>
{
var inputList = new DistributedList<int>(bluepath.Storage as IExtendedStorage, inputKey);
int partialSum = 0;
for (int x = indexStart; x < indexEnd; x++)
{
partialSum += inputList[x];
//System.Threading.Thread.Sleep(1000);
}
return partialSum;
}),
connectionManager,
scheduler
);
thread.Start(inputDataKey, startIndex, endIndex);
threads.Add(thread);
}
int overallSum = 0;
foreach (var thread in threads)
{
thread.Join();
if (thread.State == Executor.ExecutorState.Faulted)
{
Console.WriteLine("Err");
}
overallSum += (int)thread.Result;
}
sw.Stop();
//var clearDataThread = DistributedThread.Create(
// new Func<string, IBluepathCommunicationFramework, int>(
// (key, bluepath) =>
// {
// var list = new DistributedList<int>(bluepath.Storage as IExtendedStorage, key);
// list.Clear();
// return 0;
// }),
// connectionManager, scheduler, DistributedThread.ExecutorSelectionMode.LocalOnly);
//clearDataThread.Start(inputDataKey);
//clearDataThread.Join();
Console.WriteLine(overallSum);
Console.WriteLine(sw.ElapsedMilliseconds);
}
private static void WriteToStorageList(Activity activity, string message)
{
if(!Log.WriteToDistributedMemory)
{
return;
}
var resource = BluepathListener.NodeGuid.ToString();
if (activity != Activity.Custom)
{
message = activity.ToString().Replace('_', ' ');
}
try
{
var logThread = new System.Threading.Thread(() =>
{
try
{
int retryCount = 5;
int retryNo = 0;
// jest zahardcodowany new RedisStorage!! (??)
var storage = new RedisStorage(DistributedMemoryHost);
var dateTime = DateTime.Now;
if (monotonicallyIncreasingLogTime)
{
dateTime = AssureTimeMonotonicity(dateTime, storage, string.Format("{0}_time", XesLogKey));
}
var list = new DistributedList<EventType>(storage, XesLogKey);
while (retryNo < retryCount)
{
try
{
list.Add(new EventType(message, resource, dateTime, EventType.Transition.Start));
break;
}
catch (Exception ex)
{
ExceptionMessage(ex, Log.Activity.Info, string.Format("Error on saving log to Redis[{0}]. {1}", retryNo, ex.StackTrace), logLocallyOnly: true);
retryNo++;
}
}
retryNo = 0;
while (retryNo < retryCount)
{
try
{
list.Add(new EventType(message, resource, dateTime, EventType.Transition.Complete));
break;
}
catch (Exception ex)
{
ExceptionMessage(ex, Log.Activity.Info, string.Format("Error on saving log to Redis[{0}]. {1}", retryNo, ex.StackTrace), logLocallyOnly: true);
retryNo++;
}
}
}
catch
{
}
});
logThread.IsBackground = false;
logThread.Start();
}
catch (NotSupportedException ex)
{
ExceptionMessage(ex, Log.Activity.Info, "Exception on saving log to DistributedList", logLocallyOnly: true);
}
}
public static void SaveXes(string fileName, string caseName = null, bool clearListAfterSave = false)
{
var storage = new RedisStorage(DistributedMemoryHost);
var list = new DistributedList<EventType>(storage, XesLogKey);
var @case = new TraceType(caseName ?? Guid.NewGuid().ToString(), list);
LogType log = null;
if (!File.Exists(fileName))
{
log = LogType.Create(new[] { @case });
}
else
{
using (StreamReader reader = new StreamReader(fileName))
{
log = LogType.Deserialize(reader.BaseStream);
}
var tracesList = log.trace.ToList();
tracesList.Add(@case);
log.trace = tracesList.ToArray();
}
using (System.IO.StreamWriter file = new System.IO.StreamWriter(fileName))
{
log.Serialize(file.BaseStream);
}
if (clearListAfterSave)
{
list.Clear();
}
}
public static void ClearXes()
{
var storage = new RedisStorage(DistributedMemoryHost);
var list = new DistributedList<EventType>(storage, XesLogKey);
list.Clear();
}
public void DistributedListsPerformsCopyPartToArray()
{
var storage = new RedisStorage(Host);
var key = Guid.NewGuid().ToString();
var list1 = new DistributedList<int>(storage, key);
int elementCount = 10;
for (int i = 0; i < elementCount; i++)
{
list1.Add(i);
}
list1.Count.ShouldBe(elementCount);
int startIndex = 3;
int count = 3;
var destinationArray = new int[count];
list1.CopyPartTo(startIndex, count, destinationArray);
for (int i = 0; i < destinationArray.Length; i++)
{
destinationArray[i].ShouldBe(i + startIndex);
}
}
public void DistributedListsPerformsCopyToArray()
{
var storage = new RedisStorage(Host);
var key = Guid.NewGuid().ToString();
var list1 = new DistributedList<int>(storage, key);
int elementCount = 10;
for (int i = 0; i < elementCount; i++)
{
list1.Add(i);
}
list1.Count.ShouldBe(elementCount);
var destinationArray = new int[elementCount];
list1.CopyTo(destinationArray, 0);
for (int i = 0; i < elementCount; i++)
{
destinationArray[i].ShouldBe(i);
}
}
static void Main(string[] args)
{
Program.Prefixes = new Dictionary<string, List<string>>();
var options = new Options();
if (CommandLine.Parser.Default.ParseArguments(args, options))
{
Bluepath.Log.DistributedMemoryHost = options.RedisHost;
Bluepath.Log.WriteInfoToConsole = false;
Bluepath.Log.WriteToDistributedMemory = false;
var bluepathListener = Bluepath.Services.BluepathListener.InitializeDefaultListener(options.Ip, options.Port);
using (var serviceDiscoveryClient = new CentralizedDiscovery.Client.CentralizedDiscovery(
new ServiceUri(options.CentralizedDiscoveryURI, BindingType.BasicHttpBinding),
bluepathListener
))
{
using (var connectionManager = new ConnectionManager(
remoteService: null,
listener: bluepathListener,
serviceDiscovery: serviceDiscoveryClient,
serviceDiscoveryPeriod: TimeSpan.FromSeconds(1)))
{
System.Threading.Thread.Sleep(1500);
string command = string.Empty;
string sharedStorageKey = "loadlist";
var sharedCounterKey = "counter";
Console.WriteLine("Initializing Redis");
var storage = new RedisStorage(options.RedisHost);
var list = new DistributedList<string>(storage, sharedStorageKey);
Console.WriteLine("List count: {0}", list.Count);
var commands = new Dictionary<string, Action>()
{
{"LOAD", ()=>
{
var sw = new Stopwatch();
sw.Start();
var scheduler = new RoundRobinLocalScheduler(connectionManager.RemoteServices.Select(s=>s.Key).ToArray());
list.Clear();
var counter = new DistributedCounter(storage, sharedCounterKey);
counter.SetValue(0);
var localList = new List<string>();
foreach(var file in Directory.EnumerateFiles(options.InputFolder))
{
var fileContent = File.ReadAllText(file);
localList.Add(fileContent);
}
list.AddRange(localList);
localList.Clear();
var servicesCount = connectionManager.RemoteServices.Count;
if(servicesCount==0)
{
servicesCount = 1;
}
var calculatedChunkSize = (int)Math.Floor(Convert.ToDouble(list.Count) / servicesCount);
if(calculatedChunkSize==0)
{
calculatedChunkSize = 1;
}
var threads = new List<DistributedThread>();
for(int i=0;i<servicesCount;i++)
{
var loadThread = DistributedThread.Create(
new Func<string, string, int, IBluepathCommunicationFramework, int>(
(inputKey, counterKey, chunkSize, bluepath) =>
{
return LoadDocuments(inputKey, counterKey, chunkSize, bluepath);
}), connectionManager, scheduler);
loadThread.Start(sharedStorageKey, sharedCounterKey, calculatedChunkSize);
threads.Add(loadThread);
}
foreach (var thread in threads)
{
thread.Join();
}
list.Clear();
sw.Stop();
Console.WriteLine("Loaded in {0}ms", sw.ElapsedMilliseconds);
}},
{"SEARCH", ()=>
{
var services = connectionManager.RemoteServices.Select(s => s.Key).ToArray();
var scheduler = new RoundRobinLocalScheduler(services);
var threads = new List<DistributedThread>();
Console.Write("Word part: ");
var query = Console.ReadLine();
var sw = new Stopwatch();
sw.Start();
for(int i = 0;(i<services.Length) || i < 1;i++)
{
var searchThread = DistributedThread.Create(
new Func<string,string[]>((searchPhraze) =>
{
List<string> result = new List<string>();
lock(Program.prefixesDictionaryLock)
{
if(Program.Prefixes.ContainsKey(searchPhraze))
{
result = Program.Prefixes[searchPhraze];
}
}
return result.ToArray();
}), null, scheduler);
searchThread.Start(query);
threads.Add(searchThread);
}
var joinedResult = new List<string>();
foreach (var thread in threads)
{
thread.Join();
joinedResult.AddRange(thread.Result as string[]);
}
var endResult = joinedResult.Distinct().ToArray();
sw.Stop();
Console.WriteLine();
Console.WriteLine(string.Join("||", endResult));
Console.WriteLine("Found in {0}ms", sw.ElapsedMilliseconds);
}},
{"CLEAN", ()=>
{
var services = connectionManager.RemoteServices.Select(s => s.Key).ToArray();
var scheduler = new RoundRobinLocalScheduler(services);
var threads = new List<DistributedThread>();
for(int i = 0;(i<services.Length) || i < 1;i++)
{
var searchThread = DistributedThread.Create(
new Func<int>(() =>
{
List<string> result = null;
lock(Program.prefixesDictionaryLock)
{
Program.Prefixes.Clear();
}
return 0;
}), null, scheduler);
searchThread.Start();
threads.Add(searchThread);
}
foreach (var thread in threads)
{
thread.Join();
}
}}
};
do
{
Console.WriteLine("Available commands:");
foreach (var key in commands.Keys)
{
Console.WriteLine(key);
}
Console.Write("Command['q' to exit]: ");
command = Console.ReadLine().ToUpper();
if (commands.ContainsKey(command))
{
commands[command]();
}
} while (command != "Q");
}
}
}
}