private static void RunRedisTest(Options options) { var storage = new Bluepath.Storage.Redis.RedisStorage(options.RedisHost); var list = new DistributedList<int>(storage, Guid.NewGuid().ToString()); var localList = new List<int>(); var rSw = new Stopwatch(); int amount = 1000; rSw.Start(); for (int i = 0; i < amount; i++) { localList.Add(1); } list.AddRange(localList); rSw.Stop(); var wSw = new Stopwatch(); int aa = 0; wSw.Start(); for (int i = 0; i < amount; i++) { aa = list[i]; } wSw.Stop(); Log.TraceMessage(Log.Activity.Custom, string.Format("Write: {0}; Read: {1}; Value {2}; Amount: {3}", rSw.ElapsedMilliseconds, wSw.ElapsedMilliseconds, aa, amount)); }
public void OpenXesSerializationTest() { var storage = new RedisStorage(Host); var key = Guid.NewGuid().ToString(); var list = new DistributedList<EventType>(storage, key); list.Add(new EventType("Start", "Start", DateTime.Now, EventType.Transition.Start)); list.Add(new EventType("Start", "Start", DateTime.Now.AddSeconds(1), EventType.Transition.Complete)); list.Add(new EventType("Phone Call", "Helen", DateTime.Now.AddSeconds(2), EventType.Transition.Start)); list.Add(new EventType("Phone Call", "Helen", DateTime.Now.AddSeconds(3), EventType.Transition.Complete)); list.Add(new EventType("End", "End", DateTime.Now.AddSeconds(4), EventType.Transition.Start)); list.Add(new EventType("End", "End", DateTime.Now.AddSeconds(5), EventType.Transition.Complete)); storage = new RedisStorage(Host); list = new DistributedList<EventType>(storage, key); var localList = list.ToList(); localList.Count.ShouldBe(6); localList.Where(e => e != null).Count().ShouldBe(6); var @case = new TraceType(Guid.NewGuid().ToString(), list); var log = LogType.Create(new[] { @case }); var xml = log.Serialize(); Assert.IsTrue(xml.Contains("Phone Call")); }
public void DistributedListAllowsAddingEnumerablesInOneStep() { var storage = new RedisStorage(Host); var key = Guid.NewGuid().ToString(); var list = new DistributedList<int>(storage, key); var localList = new List<int>(); for (int i = 0; i < 10; i++) { localList.Add(i); } list.AddRange(localList); list.Count.ShouldBe(10); }
public void DistributedListHandlesBigCollections() { var storage = new RedisStorage(Host); var key = Guid.NewGuid().ToString(); var list = new DistributedList<int>(storage, key); int amount = 100000; var localList = new List<int>(amount); for (int i = 0; i < amount;i++ ) { localList.Add(i); } list.AddRange(localList); list.Count.ShouldBe(amount); list.Clear(); list.Count.ShouldBe(0); }
public void DistributedListAllowsAddingAndRemovingItems() { var storage = new RedisStorage(Host); var key = Guid.NewGuid().ToString(); var list = new DistributedList<int>(storage, key); for (int i = 0; i < 10; i++) { list.Add(i); } list.Count.ShouldBe(10); list.Remove(5); list.Count.ShouldBe(9); foreach (var item in list) { item.ShouldNotBe(5); } }
public void DistributedListsWorksWithClasses() { var storage = new RedisStorage(Host); var key = Guid.NewGuid().ToString(); var list1 = new DistributedList<Bluepath.Tests.Integration.DistributedThread.ComplexParameter>(storage, key); for (int i = 0; i < 10; i++) { list1.Add(new Bluepath.Tests.Integration.DistributedThread.ComplexParameter() { AnotherProperty = i, SomeProperty = string.Format("ala{0}", i) }); } list1.Count.ShouldBe(10); var list2 = new DistributedList<Bluepath.Tests.Integration.DistributedThread.ComplexParameter>(storage, key); for (int i = 0; i < list2.Count; i++) { list2[i].AnotherProperty.ShouldBe(i); } }
public void DistributedListsWithTheSameKeyShareState() { var storage = new RedisStorage(Host); var key = Guid.NewGuid().ToString(); var list1 = new DistributedList<int>(storage, key); for (int i = 0; i < 10; i++) { list1.Add(i); } list1.Count.ShouldBe(10); var list2 = new DistributedList<int>(storage, key); list2.Count.ShouldBe(10); }
public void DistributedListsPostponeRemovingItemsUntilEnumarationIsFinished() { var storage = new RedisStorage(Host); var key = Guid.NewGuid().ToString(); var list1 = new DistributedList<int>(storage, key); var synchLock = new object(); bool finishedRemoving = false; bool isEnumerating = false; int count = 0; for (int i = 0; i < 10; i++) { list1.Add(i); } var enumerationThread = new Thread(() => { var storage1 = new RedisStorage(Host); var list = new DistributedList<int>(storage1, key); bool isFirstTurn = true; int localCount = 0; foreach (var item in list) { localCount++; isEnumerating = true; if (isFirstTurn) { isFirstTurn = false; lock (synchLock) { Monitor.Wait(synchLock); } } } count = localCount; }); var removeItemThread = new Thread(() => { var storage2 = new RedisStorage(Host); var list = new DistributedList<int>(storage2, key); list.RemoveAt(4); finishedRemoving = true; }); enumerationThread.Start(); TestHelpers.RepeatUntilTrue(() => isEnumerating, times: 5); removeItemThread.Start(); Thread.Sleep(10); list1.Count.ShouldBe(10); finishedRemoving.ShouldBe(false); lock (synchLock) { Monitor.Pulse(synchLock); } enumerationThread.Join(); removeItemThread.Join(); finishedRemoving.ShouldBe(true); list1.Count.ShouldBe(9); count.ShouldBe(10); }
private static int LoadDocuments(string inputKey, string counterKey, int chunkSize, IBluepathCommunicationFramework bluepath) { var inputList = new DistributedList<string>(bluepath.Storage as IExtendedStorage, inputKey); var inputCount = inputList.Count; var counter = new DistributedCounter(bluepath.Storage as IExtendedStorage, counterKey); int indexEnd = 0; do { int noOfElements = chunkSize; int indexStart = counter.GetAndIncrease(chunkSize); if (indexStart >= inputCount) { break; } indexEnd = indexStart + noOfElements; if (indexEnd > inputCount) { indexEnd = inputCount; noOfElements = indexEnd - indexStart; } var inputDocuments = new string[noOfElements]; inputList.CopyPartTo(indexStart, noOfElements, inputDocuments); foreach (var document in inputDocuments) { var words = document.Split(' '); foreach (var word in words) { if (word.Length == 0) { continue; } var stringToProcess = word; var partialResult = new List<string>(stringToProcess.Length); for (int si = 0; si < stringToProcess.Length; si++) { string res = ""; if (si == stringToProcess.Length - 1) { res = stringToProcess; } else { res = stringToProcess.Substring(0, si + 1); } partialResult.Add(res); } lock (Program.prefixesDictionaryLock) { foreach (var prefix in partialResult) { if (Program.Prefixes.ContainsKey(prefix)) { if (!Program.Prefixes[prefix].Contains(word)) { Program.Prefixes[prefix].Add(word); } } else { Program.Prefixes.Add(prefix, new List<string>() { word }); } } } } } } while (indexEnd <= inputCount); return 0; }
private static void RunTest(ConnectionManager connectionManager, ThreadNumberScheduler scheduler, Options options) { Log.TraceMessage(Log.Activity.Custom, "Creating data"); var initializeDataThread = DistributedThread.Create( new Func<int, string, IBluepathCommunicationFramework, int>( (dataSize, key, bluepath) => { var data = new List<string>(dataSize); var list = new DistributedList<string>(bluepath.Storage as IExtendedStorage, key); if (list.Count != dataSize) { list.Clear(); } else { return dataSize; } for (int i = 0; i < dataSize; i++) { int nextSourceDocument = i % SourceDocuments.Documents.Count; data.Add(SourceDocuments.Documents[nextSourceDocument]); } Console.WriteLine("Start saving data to redis"); list.AddRange(data); return dataSize; }), connectionManager, scheduler, DistributedThread.ExecutorSelectionMode.LocalOnly); var inputDataKey = "distributedPrefixData"; var numberOfElements = options.NoOfElements; initializeDataThread.Start(numberOfElements, inputDataKey); initializeDataThread.Join(); var expectedSum = (int)initializeDataThread.Result; var numberOfShards = options.NoOfShards; var elementsPerShard = numberOfElements / numberOfShards; var threads = new List<DistributedThread>(); Log.TraceMessage(Log.Activity.Custom, "Running test"); var sw = new Stopwatch(); sw.Start(); for (int i = 0; i < numberOfShards; i++) { int startIndex = i * elementsPerShard; int endIndex; if (i == numberOfShards - 1) { // last element endIndex = numberOfElements; } else { endIndex = startIndex + elementsPerShard; } var thread = DistributedThread.Create( new Func<bool, string, int, int, IBluepathCommunicationFramework, byte[]>( (returnPrefixes, inputKey, indexStart, indexEnd, bluepath) => { var inputList = new DistributedList<string>(bluepath.Storage as IExtendedStorage, inputKey); var inputToProcess = new string[indexEnd - indexStart]; inputList.CopyPartTo(indexStart, inputToProcess.Length, inputToProcess); List<string> results = new List<string>(); for (int x = indexStart; x < indexEnd; x++) { var documentLine = inputToProcess[x - indexStart]; var words = documentLine.Split(' '); var partialResult = new List<string>(); foreach (var word in words) { var stringToProcess = word; for (int si = 0; si < stringToProcess.Length; si++) { string res = ""; if (si == stringToProcess.Length - 1) { res = stringToProcess; } else { res = stringToProcess.Substring(0, si + 1); } partialResult.Add(res); } } results.AddRange(partialResult); if (results.Count > 100000) { results.Clear(); } } if (returnPrefixes) { return new PrefixesResult() { Prefixes = results }.Serialize(); } else { return new PrefixesResult().Serialize(); } }), connectionManager, scheduler ); thread.Start(options.ReturnPrefixes == 1, inputDataKey, startIndex, endIndex); threads.Add(thread); } foreach (var thread in threads) { thread.Join(); if (thread.State == Executor.ExecutorState.Faulted) { Console.WriteLine("Err"); } // Could process prefixes } sw.Stop(); //var clearDataThread = DistributedThread.Create( // new Func<string, IBluepathCommunicationFramework, int>( // (key, bluepath) => // { // var list = new DistributedList<int>(bluepath.Storage as IExtendedStorage, key); // list.Clear(); // return 0; // }), // connectionManager, scheduler, DistributedThread.ExecutorSelectionMode.LocalOnly); //clearDataThread.Start(inputDataKey); //clearDataThread.Join(); Console.WriteLine(sw.ElapsedMilliseconds); }
private static void RunTest(ConnectionManager connectionManager, ThreadNumberScheduler scheduler, Options options) { Log.TraceMessage(Log.Activity.Custom, "Creating data"); var initializeDataThread = DistributedThread.Create( new Func<int, string, IBluepathCommunicationFramework, int>( (dataSize, key, bluepath) => { var data = new List<int>(dataSize); for (int i = 0; i < dataSize; i++) { data.Add(1); } var list = new DistributedList<int>(bluepath.Storage as IExtendedStorage, key); if (list.Count != dataSize) { list.Clear(); list.AddRange(data); } return dataSize; }), connectionManager, scheduler, DistributedThread.ExecutorSelectionMode.LocalOnly); var inputDataKey = "inputData"; var numberOfElements = options.NoOfElements; initializeDataThread.Start(numberOfElements, inputDataKey); initializeDataThread.Join(); var expectedSum = (int)initializeDataThread.Result; var numberOfShards = options.NoOfShards; var elementsPerShard = numberOfElements / numberOfShards; var threads = new List<DistributedThread>(); Log.TraceMessage(Log.Activity.Custom, "Running test"); var sw = new Stopwatch(); sw.Start(); for (int i = 0; i < numberOfShards; i++) { int startIndex = i * elementsPerShard; int endIndex; if (i == numberOfShards - 1) { // last element endIndex = numberOfElements; } else { endIndex = startIndex + elementsPerShard; } var thread = DistributedThread.Create( new Func<string, int, int, IBluepathCommunicationFramework, int>( (inputKey, indexStart, indexEnd, bluepath) => { var inputList = new DistributedList<int>(bluepath.Storage as IExtendedStorage, inputKey); int partialSum = 0; for (int x = indexStart; x < indexEnd; x++) { partialSum += inputList[x]; //System.Threading.Thread.Sleep(1000); } return partialSum; }), connectionManager, scheduler ); thread.Start(inputDataKey, startIndex, endIndex); threads.Add(thread); } int overallSum = 0; foreach (var thread in threads) { thread.Join(); if (thread.State == Executor.ExecutorState.Faulted) { Console.WriteLine("Err"); } overallSum += (int)thread.Result; } sw.Stop(); //var clearDataThread = DistributedThread.Create( // new Func<string, IBluepathCommunicationFramework, int>( // (key, bluepath) => // { // var list = new DistributedList<int>(bluepath.Storage as IExtendedStorage, key); // list.Clear(); // return 0; // }), // connectionManager, scheduler, DistributedThread.ExecutorSelectionMode.LocalOnly); //clearDataThread.Start(inputDataKey); //clearDataThread.Join(); Console.WriteLine(overallSum); Console.WriteLine(sw.ElapsedMilliseconds); }
private static void WriteToStorageList(Activity activity, string message) { if(!Log.WriteToDistributedMemory) { return; } var resource = BluepathListener.NodeGuid.ToString(); if (activity != Activity.Custom) { message = activity.ToString().Replace('_', ' '); } try { var logThread = new System.Threading.Thread(() => { try { int retryCount = 5; int retryNo = 0; // jest zahardcodowany new RedisStorage!! (??) var storage = new RedisStorage(DistributedMemoryHost); var dateTime = DateTime.Now; if (monotonicallyIncreasingLogTime) { dateTime = AssureTimeMonotonicity(dateTime, storage, string.Format("{0}_time", XesLogKey)); } var list = new DistributedList<EventType>(storage, XesLogKey); while (retryNo < retryCount) { try { list.Add(new EventType(message, resource, dateTime, EventType.Transition.Start)); break; } catch (Exception ex) { ExceptionMessage(ex, Log.Activity.Info, string.Format("Error on saving log to Redis[{0}]. {1}", retryNo, ex.StackTrace), logLocallyOnly: true); retryNo++; } } retryNo = 0; while (retryNo < retryCount) { try { list.Add(new EventType(message, resource, dateTime, EventType.Transition.Complete)); break; } catch (Exception ex) { ExceptionMessage(ex, Log.Activity.Info, string.Format("Error on saving log to Redis[{0}]. {1}", retryNo, ex.StackTrace), logLocallyOnly: true); retryNo++; } } } catch { } }); logThread.IsBackground = false; logThread.Start(); } catch (NotSupportedException ex) { ExceptionMessage(ex, Log.Activity.Info, "Exception on saving log to DistributedList", logLocallyOnly: true); } }
public static void SaveXes(string fileName, string caseName = null, bool clearListAfterSave = false) { var storage = new RedisStorage(DistributedMemoryHost); var list = new DistributedList<EventType>(storage, XesLogKey); var @case = new TraceType(caseName ?? Guid.NewGuid().ToString(), list); LogType log = null; if (!File.Exists(fileName)) { log = LogType.Create(new[] { @case }); } else { using (StreamReader reader = new StreamReader(fileName)) { log = LogType.Deserialize(reader.BaseStream); } var tracesList = log.trace.ToList(); tracesList.Add(@case); log.trace = tracesList.ToArray(); } using (System.IO.StreamWriter file = new System.IO.StreamWriter(fileName)) { log.Serialize(file.BaseStream); } if (clearListAfterSave) { list.Clear(); } }
public static void ClearXes() { var storage = new RedisStorage(DistributedMemoryHost); var list = new DistributedList<EventType>(storage, XesLogKey); list.Clear(); }
public void DistributedListsPerformsCopyPartToArray() { var storage = new RedisStorage(Host); var key = Guid.NewGuid().ToString(); var list1 = new DistributedList<int>(storage, key); int elementCount = 10; for (int i = 0; i < elementCount; i++) { list1.Add(i); } list1.Count.ShouldBe(elementCount); int startIndex = 3; int count = 3; var destinationArray = new int[count]; list1.CopyPartTo(startIndex, count, destinationArray); for (int i = 0; i < destinationArray.Length; i++) { destinationArray[i].ShouldBe(i + startIndex); } }
public void DistributedListsPerformsCopyToArray() { var storage = new RedisStorage(Host); var key = Guid.NewGuid().ToString(); var list1 = new DistributedList<int>(storage, key); int elementCount = 10; for (int i = 0; i < elementCount; i++) { list1.Add(i); } list1.Count.ShouldBe(elementCount); var destinationArray = new int[elementCount]; list1.CopyTo(destinationArray, 0); for (int i = 0; i < elementCount; i++) { destinationArray[i].ShouldBe(i); } }
static void Main(string[] args) { Program.Prefixes = new Dictionary<string, List<string>>(); var options = new Options(); if (CommandLine.Parser.Default.ParseArguments(args, options)) { Bluepath.Log.DistributedMemoryHost = options.RedisHost; Bluepath.Log.WriteInfoToConsole = false; Bluepath.Log.WriteToDistributedMemory = false; var bluepathListener = Bluepath.Services.BluepathListener.InitializeDefaultListener(options.Ip, options.Port); using (var serviceDiscoveryClient = new CentralizedDiscovery.Client.CentralizedDiscovery( new ServiceUri(options.CentralizedDiscoveryURI, BindingType.BasicHttpBinding), bluepathListener )) { using (var connectionManager = new ConnectionManager( remoteService: null, listener: bluepathListener, serviceDiscovery: serviceDiscoveryClient, serviceDiscoveryPeriod: TimeSpan.FromSeconds(1))) { System.Threading.Thread.Sleep(1500); string command = string.Empty; string sharedStorageKey = "loadlist"; var sharedCounterKey = "counter"; Console.WriteLine("Initializing Redis"); var storage = new RedisStorage(options.RedisHost); var list = new DistributedList<string>(storage, sharedStorageKey); Console.WriteLine("List count: {0}", list.Count); var commands = new Dictionary<string, Action>() { {"LOAD", ()=> { var sw = new Stopwatch(); sw.Start(); var scheduler = new RoundRobinLocalScheduler(connectionManager.RemoteServices.Select(s=>s.Key).ToArray()); list.Clear(); var counter = new DistributedCounter(storage, sharedCounterKey); counter.SetValue(0); var localList = new List<string>(); foreach(var file in Directory.EnumerateFiles(options.InputFolder)) { var fileContent = File.ReadAllText(file); localList.Add(fileContent); } list.AddRange(localList); localList.Clear(); var servicesCount = connectionManager.RemoteServices.Count; if(servicesCount==0) { servicesCount = 1; } var calculatedChunkSize = (int)Math.Floor(Convert.ToDouble(list.Count) / servicesCount); if(calculatedChunkSize==0) { calculatedChunkSize = 1; } var threads = new List<DistributedThread>(); for(int i=0;i<servicesCount;i++) { var loadThread = DistributedThread.Create( new Func<string, string, int, IBluepathCommunicationFramework, int>( (inputKey, counterKey, chunkSize, bluepath) => { return LoadDocuments(inputKey, counterKey, chunkSize, bluepath); }), connectionManager, scheduler); loadThread.Start(sharedStorageKey, sharedCounterKey, calculatedChunkSize); threads.Add(loadThread); } foreach (var thread in threads) { thread.Join(); } list.Clear(); sw.Stop(); Console.WriteLine("Loaded in {0}ms", sw.ElapsedMilliseconds); }}, {"SEARCH", ()=> { var services = connectionManager.RemoteServices.Select(s => s.Key).ToArray(); var scheduler = new RoundRobinLocalScheduler(services); var threads = new List<DistributedThread>(); Console.Write("Word part: "); var query = Console.ReadLine(); var sw = new Stopwatch(); sw.Start(); for(int i = 0;(i<services.Length) || i < 1;i++) { var searchThread = DistributedThread.Create( new Func<string,string[]>((searchPhraze) => { List<string> result = new List<string>(); lock(Program.prefixesDictionaryLock) { if(Program.Prefixes.ContainsKey(searchPhraze)) { result = Program.Prefixes[searchPhraze]; } } return result.ToArray(); }), null, scheduler); searchThread.Start(query); threads.Add(searchThread); } var joinedResult = new List<string>(); foreach (var thread in threads) { thread.Join(); joinedResult.AddRange(thread.Result as string[]); } var endResult = joinedResult.Distinct().ToArray(); sw.Stop(); Console.WriteLine(); Console.WriteLine(string.Join("||", endResult)); Console.WriteLine("Found in {0}ms", sw.ElapsedMilliseconds); }}, {"CLEAN", ()=> { var services = connectionManager.RemoteServices.Select(s => s.Key).ToArray(); var scheduler = new RoundRobinLocalScheduler(services); var threads = new List<DistributedThread>(); for(int i = 0;(i<services.Length) || i < 1;i++) { var searchThread = DistributedThread.Create( new Func<int>(() => { List<string> result = null; lock(Program.prefixesDictionaryLock) { Program.Prefixes.Clear(); } return 0; }), null, scheduler); searchThread.Start(); threads.Add(searchThread); } foreach (var thread in threads) { thread.Join(); } }} }; do { Console.WriteLine("Available commands:"); foreach (var key in commands.Keys) { Console.WriteLine(key); } Console.Write("Command['q' to exit]: "); command = Console.ReadLine().ToUpper(); if (commands.ContainsKey(command)) { commands[command](); } } while (command != "Q"); } } } }