static void Main(string[] args) { var options = new Options(); if (CommandLine.Parser.Default.ParseArguments(args, options)) { Bluepath.Log.DistributedMemoryHost = options.RedisHost; Bluepath.Log.WriteInfoToConsole = false; var bluepathListener = new BluepathListener(options.Ip, options.Port); using (var serviceDiscoveryClient = new CentralizedDiscovery.Client.CentralizedDiscovery( new ServiceUri(options.CentralizedDiscoveryURI, BindingType.BasicHttpBinding), bluepathListener ) ) { using (var connectionManager = new ConnectionManager( remoteService: null, listener: bluepathListener, serviceDiscovery: serviceDiscoveryClient, serviceDiscoveryPeriod: TimeSpan.FromSeconds(30))) { System.Threading.Thread.Sleep(1500); var scheduler = new ThreadNumberScheduler(connectionManager); if (options.IsSlave == 0) { Log.TraceMessage(Log.Activity.Custom, "Running master"); RunTest(connectionManager, scheduler, options); } else { Log.TraceMessage(Log.Activity.Custom, "Running slave"); } Console.WriteLine("Press <Enter> to stop the service."); Console.ReadLine(); } } } }
private static void RunTest(ConnectionManager connectionManager, ThreadNumberScheduler scheduler, Options options) { Log.TraceMessage(Log.Activity.Custom, "Creating data"); var initializeDataThread = DistributedThread.Create( new Func<int, string, IBluepathCommunicationFramework, int>( (dataSize, key, bluepath) => { var data = new List<string>(dataSize); var list = new DistributedList<string>(bluepath.Storage as IExtendedStorage, key); if (list.Count != dataSize) { list.Clear(); } else { return dataSize; } for (int i = 0; i < dataSize; i++) { int nextSourceDocument = i % SourceDocuments.Documents.Count; data.Add(SourceDocuments.Documents[nextSourceDocument]); } Console.WriteLine("Start saving data to redis"); list.AddRange(data); return dataSize; }), connectionManager, scheduler, DistributedThread.ExecutorSelectionMode.LocalOnly); var inputDataKey = "distributedPrefixData"; var numberOfElements = options.NoOfElements; initializeDataThread.Start(numberOfElements, inputDataKey); initializeDataThread.Join(); var expectedSum = (int)initializeDataThread.Result; var numberOfShards = options.NoOfShards; var elementsPerShard = numberOfElements / numberOfShards; var threads = new List<DistributedThread>(); Log.TraceMessage(Log.Activity.Custom, "Running test"); var sw = new Stopwatch(); sw.Start(); for (int i = 0; i < numberOfShards; i++) { int startIndex = i * elementsPerShard; int endIndex; if (i == numberOfShards - 1) { // last element endIndex = numberOfElements; } else { endIndex = startIndex + elementsPerShard; } var thread = DistributedThread.Create( new Func<bool, string, int, int, IBluepathCommunicationFramework, byte[]>( (returnPrefixes, inputKey, indexStart, indexEnd, bluepath) => { var inputList = new DistributedList<string>(bluepath.Storage as IExtendedStorage, inputKey); var inputToProcess = new string[indexEnd - indexStart]; inputList.CopyPartTo(indexStart, inputToProcess.Length, inputToProcess); List<string> results = new List<string>(); for (int x = indexStart; x < indexEnd; x++) { var documentLine = inputToProcess[x - indexStart]; var words = documentLine.Split(' '); var partialResult = new List<string>(); foreach (var word in words) { var stringToProcess = word; for (int si = 0; si < stringToProcess.Length; si++) { string res = ""; if (si == stringToProcess.Length - 1) { res = stringToProcess; } else { res = stringToProcess.Substring(0, si + 1); } partialResult.Add(res); } } results.AddRange(partialResult); if (results.Count > 100000) { results.Clear(); } } if (returnPrefixes) { return new PrefixesResult() { Prefixes = results }.Serialize(); } else { return new PrefixesResult().Serialize(); } }), connectionManager, scheduler ); thread.Start(options.ReturnPrefixes == 1, inputDataKey, startIndex, endIndex); threads.Add(thread); } foreach (var thread in threads) { thread.Join(); if (thread.State == Executor.ExecutorState.Faulted) { Console.WriteLine("Err"); } // Could process prefixes } sw.Stop(); //var clearDataThread = DistributedThread.Create( // new Func<string, IBluepathCommunicationFramework, int>( // (key, bluepath) => // { // var list = new DistributedList<int>(bluepath.Storage as IExtendedStorage, key); // list.Clear(); // return 0; // }), // connectionManager, scheduler, DistributedThread.ExecutorSelectionMode.LocalOnly); //clearDataThread.Start(inputDataKey); //clearDataThread.Join(); Console.WriteLine(sw.ElapsedMilliseconds); }