static void Main(string[] args)
{
var options = new Options();
if (CommandLine.Parser.Default.ParseArguments(args, options))
{
Bluepath.Log.DistributedMemoryHost = options.RedisHost;
Bluepath.Log.WriteInfoToConsole = false;
var bluepathListener = new BluepathListener(options.Ip, options.Port);
using (var serviceDiscoveryClient
= new CentralizedDiscovery.Client.CentralizedDiscovery(
new ServiceUri(options.CentralizedDiscoveryURI, BindingType.BasicHttpBinding),
bluepathListener
)
)
{
using (var connectionManager = new ConnectionManager(
remoteService: null,
listener: bluepathListener,
serviceDiscovery: serviceDiscoveryClient,
serviceDiscoveryPeriod: TimeSpan.FromSeconds(30)))
{
System.Threading.Thread.Sleep(1500);
var scheduler = new ThreadNumberScheduler(connectionManager);
if (options.IsSlave == 0)
{
Log.TraceMessage(Log.Activity.Custom, "Running master");
//RunRedisTest(options);
RunTest(connectionManager, scheduler, options);
}
else
{
Log.TraceMessage(Log.Activity.Custom, "Running slave");
}
Console.WriteLine("Press <Enter> to stop the service.");
Console.ReadLine();
}
}
}
}
public void ConnectionManagerAddsNewServicesFromRemoteServiceDiscovery()
{
var serviceDiscoveryHost = new CentralizedDiscovery.CentralizedDiscoveryListener("localhost", 30000);
var bluepathListener1 = new Bluepath.Services.BluepathListener("localhost", 30001);
var bluepathListener2 = new Bluepath.Services.BluepathListener("localhost", 30002);
var bluepathListener3 = new Bluepath.Services.BluepathListener("localhost", 30003);
try
{
using (var serviceDiscoveryClient1
= new CentralizedDiscovery.Client.CentralizedDiscovery(serviceDiscoveryHost.MasterUri, bluepathListener1))
{
using (var serviceDiscoveryClient2
= new CentralizedDiscovery.Client.CentralizedDiscovery(serviceDiscoveryHost.MasterUri, bluepathListener2))
{
var connectionManager = new Bluepath.Services.ConnectionManager(remoteService: null,
listener: bluepathListener1,
serviceDiscovery: serviceDiscoveryClient1,
serviceDiscoveryPeriod: new TimeSpan(days: 0, hours: 0, minutes: 0, seconds: 0, milliseconds: 100));
TestHelpers.RepeatUntilTrue(() => connectionManager.RemoteServices.Count() == 1, times: 10);
connectionManager.RemoteServices.Count().ShouldBe(1);
using (var serviceDiscoveryClient3
= new CentralizedDiscovery.Client.CentralizedDiscovery(serviceDiscoveryHost.MasterUri, bluepathListener3))
{
TestHelpers.RepeatUntilTrue(() => connectionManager.RemoteServices.Count() == 2, times: 10);
connectionManager.RemoteServices.Count().ShouldBe(2);
}
}
}
}
finally
{
serviceDiscoveryHost.Stop();
bluepathListener1.Stop();
bluepathListener2.Stop();
bluepathListener3.Stop();
}
}
public void CentralizedDiscoveryProvidesInformationAboutLoadOfTheNodes()
{
var serviceDiscoveryHost = new CentralizedDiscovery.CentralizedDiscoveryListener("localhost", 41000);
var bluepathListener1 = new Bluepath.Services.BluepathListener("localhost", 41001);
var bluepathListener2 = new Bluepath.Services.BluepathListener("localhost", 41002);
try
{
using (var serviceDiscoveryClient1
= new CentralizedDiscovery.Client.CentralizedDiscovery(serviceDiscoveryHost.MasterUri, bluepathListener1))
{
using (var serviceDiscoveryClient2
= new CentralizedDiscovery.Client.CentralizedDiscovery(serviceDiscoveryHost.MasterUri, bluepathListener2))
{
var connectionManager = new Bluepath.Services.ConnectionManager(
remoteService: null,
listener: bluepathListener1,
serviceDiscovery: serviceDiscoveryClient1);
var scheduler = new ThreadNumberScheduler(connectionManager);
TestHelpers.RepeatUntilTrue(() => connectionManager.RemoteServices.Count() == 1, times: 10);
connectionManager.RemoteServices.Count().ShouldBe(1);
// TODO: Review this test
var testMethod = new Func<int, int, int>(
(a, b) =>
{
Thread.Sleep(100);
return a + b;
});
var thread = DistributedThread.Create(
testMethod,
connectionManager,
scheduler,
DistributedThread.ExecutorSelectionMode.RemoteOnly
);
thread.Start(4, 5);
var performanceStatistics = serviceDiscoveryClient1.GetPerformanceStatistics();
performanceStatistics.Count.ShouldBe(2);
performanceStatistics.ElementAt(0).Key.Address.ShouldNotBeSameAs(performanceStatistics.ElementAt(1).Key.Address);
performanceStatistics.ElementAt(0).Value.NumberOfTasks[ExecutorState.NotStarted].ShouldBe(0);
performanceStatistics.ElementAt(0).Value.NumberOfTasks[ExecutorState.Running].ShouldBeLessThanOrEqualTo(1);
performanceStatistics.ElementAt(1).Value.NumberOfTasks[ExecutorState.Running].ShouldBeLessThanOrEqualTo(1);
if (performanceStatistics.ElementAt(0).Value.NumberOfTasks[ExecutorState.Running] > 0
&& performanceStatistics.ElementAt(1).Value.NumberOfTasks[ExecutorState.Running] > 0)
{
Assert.Fail("One task was scheduled but two are reported to be running.");
}
(performanceStatistics.ElementAt(0).Value.NumberOfTasks[ExecutorState.Running]
+ performanceStatistics.ElementAt(1).Value.NumberOfTasks[ExecutorState.Running]).ShouldBe(1);
performanceStatistics.ElementAt(0).Value.NumberOfTasks[ExecutorState.Finished].ShouldBe(0);
performanceStatistics.ElementAt(0).Value.NumberOfTasks[ExecutorState.Faulted].ShouldBe(0);
thread.Join();
}
}
}
finally
{
serviceDiscoveryHost.Stop();
bluepathListener1.Stop();
bluepathListener2.Stop();
}
}
public void ConnectionManagerFetchesServicesFromRemoteServiceDiscovery()
{
var serviceDiscoveryHost = new CentralizedDiscovery.CentralizedDiscoveryListener("localhost", 20000);
var bluepathListener1 = new Bluepath.Services.BluepathListener("localhost", 20001);
var bluepathListener2 = new Bluepath.Services.BluepathListener("localhost", 20002);
try
{
using (var serviceDiscoveryClient1
= new CentralizedDiscovery.Client.CentralizedDiscovery(serviceDiscoveryHost.MasterUri, bluepathListener1))
{
using (var serviceDiscoveryClient2
= new CentralizedDiscovery.Client.CentralizedDiscovery(serviceDiscoveryHost.MasterUri, bluepathListener2))
{
var connectionManager = new Bluepath.Services.ConnectionManager(remoteService: null,
listener: bluepathListener1,
serviceDiscovery: serviceDiscoveryClient1);
TestHelpers.RepeatUntilTrue(() => connectionManager.RemoteServices.Count() == 1, times: 10);
connectionManager.RemoteServices.Count().ShouldBe(1);
}
}
}
finally
{
serviceDiscoveryHost.Stop();
bluepathListener1.Stop();
bluepathListener2.Stop();
}
}
static void Main(string[] args)
{
Program.Prefixes = new Dictionary<string, List<string>>();
var options = new Options();
if (CommandLine.Parser.Default.ParseArguments(args, options))
{
Bluepath.Log.DistributedMemoryHost = options.RedisHost;
Bluepath.Log.WriteInfoToConsole = false;
Bluepath.Log.WriteToDistributedMemory = false;
var bluepathListener = Bluepath.Services.BluepathListener.InitializeDefaultListener(options.Ip, options.Port);
using (var serviceDiscoveryClient = new CentralizedDiscovery.Client.CentralizedDiscovery(
new ServiceUri(options.CentralizedDiscoveryURI, BindingType.BasicHttpBinding),
bluepathListener
))
{
using (var connectionManager = new ConnectionManager(
remoteService: null,
listener: bluepathListener,
serviceDiscovery: serviceDiscoveryClient,
serviceDiscoveryPeriod: TimeSpan.FromSeconds(1)))
{
System.Threading.Thread.Sleep(1500);
string command = string.Empty;
string sharedStorageKey = "loadlist";
var sharedCounterKey = "counter";
Console.WriteLine("Initializing Redis");
var storage = new RedisStorage(options.RedisHost);
var list = new DistributedList<string>(storage, sharedStorageKey);
Console.WriteLine("List count: {0}", list.Count);
var commands = new Dictionary<string, Action>()
{
{"LOAD", ()=>
{
var sw = new Stopwatch();
sw.Start();
var scheduler = new RoundRobinLocalScheduler(connectionManager.RemoteServices.Select(s=>s.Key).ToArray());
list.Clear();
var counter = new DistributedCounter(storage, sharedCounterKey);
counter.SetValue(0);
var localList = new List<string>();
foreach(var file in Directory.EnumerateFiles(options.InputFolder))
{
var fileContent = File.ReadAllText(file);
localList.Add(fileContent);
}
list.AddRange(localList);
localList.Clear();
var servicesCount = connectionManager.RemoteServices.Count;
if(servicesCount==0)
{
servicesCount = 1;
}
var calculatedChunkSize = (int)Math.Floor(Convert.ToDouble(list.Count) / servicesCount);
if(calculatedChunkSize==0)
{
calculatedChunkSize = 1;
}
var threads = new List<DistributedThread>();
for(int i=0;i<servicesCount;i++)
{
var loadThread = DistributedThread.Create(
new Func<string, string, int, IBluepathCommunicationFramework, int>(
(inputKey, counterKey, chunkSize, bluepath) =>
{
return LoadDocuments(inputKey, counterKey, chunkSize, bluepath);
}), connectionManager, scheduler);
loadThread.Start(sharedStorageKey, sharedCounterKey, calculatedChunkSize);
threads.Add(loadThread);
}
foreach (var thread in threads)
{
thread.Join();
}
list.Clear();
sw.Stop();
Console.WriteLine("Loaded in {0}ms", sw.ElapsedMilliseconds);
}},
{"SEARCH", ()=>
{
var services = connectionManager.RemoteServices.Select(s => s.Key).ToArray();
var scheduler = new RoundRobinLocalScheduler(services);
var threads = new List<DistributedThread>();
Console.Write("Word part: ");
var query = Console.ReadLine();
var sw = new Stopwatch();
sw.Start();
for(int i = 0;(i<services.Length) || i < 1;i++)
{
var searchThread = DistributedThread.Create(
new Func<string,string[]>((searchPhraze) =>
{
List<string> result = new List<string>();
lock(Program.prefixesDictionaryLock)
{
if(Program.Prefixes.ContainsKey(searchPhraze))
{
result = Program.Prefixes[searchPhraze];
}
}
return result.ToArray();
}), null, scheduler);
searchThread.Start(query);
threads.Add(searchThread);
}
var joinedResult = new List<string>();
foreach (var thread in threads)
{
thread.Join();
joinedResult.AddRange(thread.Result as string[]);
}
var endResult = joinedResult.Distinct().ToArray();
sw.Stop();
Console.WriteLine();
Console.WriteLine(string.Join("||", endResult));
Console.WriteLine("Found in {0}ms", sw.ElapsedMilliseconds);
}},
{"CLEAN", ()=>
{
var services = connectionManager.RemoteServices.Select(s => s.Key).ToArray();
var scheduler = new RoundRobinLocalScheduler(services);
var threads = new List<DistributedThread>();
for(int i = 0;(i<services.Length) || i < 1;i++)
{
var searchThread = DistributedThread.Create(
new Func<int>(() =>
{
List<string> result = null;
lock(Program.prefixesDictionaryLock)
{
Program.Prefixes.Clear();
}
return 0;
}), null, scheduler);
searchThread.Start();
threads.Add(searchThread);
}
foreach (var thread in threads)
{
thread.Join();
}
}}
};
do
{
Console.WriteLine("Available commands:");
foreach (var key in commands.Keys)
{
Console.WriteLine(key);
}
Console.Write("Command['q' to exit]: ");
command = Console.ReadLine().ToUpper();
if (commands.ContainsKey(command))
{
commands[command]();
}
} while (command != "Q");
}
}
}
}
