public void TestReduce()
{
var data = new DataForProcessing();
var dataForReduce = new List <List <KeyValuePair <string, int> > >
{
new List <KeyValuePair <string, int> >
{
new KeyValuePair <string, int>("Hello", 1),
new KeyValuePair <string, int>("Hello", 1)
},
new List <KeyValuePair <string, int> >
{
new KeyValuePair <string, int>("world", 1)
},
new List <KeyValuePair <string, int> >
{
new KeyValuePair <string, int>("my", 1)
},
new List <KeyValuePair <string, int> >
{
new KeyValuePair <string, int>("friend", 1)
}
};
var result = new List <KeyValuePair <string, int> >
{
new KeyValuePair <string, int>("Hello", 2),
new KeyValuePair <string, int>("world", 1),
new KeyValuePair <string, int>("my", 1),
new KeyValuePair <string, int>("friend", 1)
};
var actual = data.Reduce(dataForReduce);
CollectionAssert.AreEqual(result, actual);
}
public void SendData(DataForProcessing data)
{
lock (LockObject)
{
var connection = OperationContext.Current.GetCallbackChannel <IClient>();
var mapResult = new List <KeyValuePair <string, int> >();
var reduceResult = new List <KeyValuePair <string, int> >();
var numMappers = Workers.Count(x => x.Value.Contains("mapper"));
var numReducers = Workers.Count(x => x.Value.Contains("reducer"));
if (numMappers > 0 && numReducers > 0)
{
var dataForMappers = SplitDataForChunks(data.FileList, numMappers).ToList();
SendDataForMappers(data, mapResult, numMappers, dataForMappers);
var groupedCustomerList = mapResult.GroupBy(u => u.Key).Select(grp => grp.ToList()).ToList();
var dataForReducers = SplitDataForChunks(groupedCustomerList, numMappers).ToList();
SendDataForReducers(data, reduceResult, dataForReducers);
connection.ReceiveData(reduceResult);
Thread.Sleep(5000);
}
else
{
connection.SystemMessage("Server doesn't have enough workers for for calculations.\nPlease try again later");
}
}
}
public List <KeyValuePair <string, int> > ReceiveDataForReduce(DataForProcessing testData, List <List <KeyValuePair <string, int> > > dataAfterMap, string type)
{
var res = new List <KeyValuePair <string, int> >();
if (type == "reducer")
{
res = testData.Reduce(dataAfterMap);
}
Console.WriteLine("'Reduce' operation has finished.");
return(res);
}
private void SendDataForReducers(DataForProcessing data, List <KeyValuePair <string, int> > reduceResult, List <List <List <KeyValuePair <string, int> > > > dataForReducers)
{
foreach (var worker in Workers)
{
if (worker.Value != "reducer")
{
continue;
}
foreach (var list in dataForReducers)
{
reduceResult.AddRange(worker.Key.ReceiveDataForReduce(data, list, worker.Value));
}
}
}
public void TestMap()
{
var data = new DataForProcessing();
var result = new List <KeyValuePair <string, int> >
{
new KeyValuePair <string, int>("Hello", 1),
new KeyValuePair <string, int>("world", 1),
new KeyValuePair <string, int>("Hello", 1),
new KeyValuePair <string, int>("my", 1),
new KeyValuePair <string, int>("friend", 1)
};
var actual = data.Map("Hello world! Hello my friend.");
CollectionAssert.AreEqual(result, actual);
}
public List <KeyValuePair <string, int> > ReceiveDataForMap(DataForProcessing testData, List <FileToProcessing> files, string type)
{
var res = new List <KeyValuePair <string, int> >();
string result = null;
foreach (var file in files)
{
result += '\n' + Encoding.UTF8.GetString(file.Content);
}
if (type == "mapper")
{
res = testData.Map(result);
}
Console.WriteLine("'Map' operation has finished");
return(res);
}
private void SendDataForMappers(DataForProcessing data, List <KeyValuePair <string, int> > mapResult, int numMappers, List <List <FileToProcessing> > dataForMappers)
{
for (var i = 0; i < Workers.Count; i++)
{
if (Workers.ElementAt(i).Value != "mapper")
{
continue;
}
foreach (var list in dataForMappers)
{
mapResult.AddRange(Workers.ElementAt(i).Key.ReceiveDataForMap(data, list, Workers.ElementAt(i).Value));
}
if (dataForMappers.Count < numMappers)
{
i++;
}
}
}
static void Main(string[] args)
{
try
{
Console.SetWindowSize(Math.Min(85, Console.LargestWindowWidth), Math.Min(15, Console.LargestWindowHeight));
//Path = ConfigurationManager.AppSettings["pathFolder"];
var address = new Uri("http://localhost:4000/IDistrCalcService");
var binding = new WSDualHttpBinding
{
MaxBufferPoolSize = 2147483647,
MaxReceivedMessageSize = 2147483647,
ReaderQuotas = new System.Xml.XmlDictionaryReaderQuotas
{
MaxDepth = 2147483647,
MaxStringContentLength = 2147483647,
MaxArrayLength = 2147483647,
MaxBytesPerRead = 2147483647,
MaxNameTableCharCount = 2147483647
},
SendTimeout = new TimeSpan(0, 0, 1, 0),
ReceiveTimeout = new TimeSpan(0, 0, 20, 0)
};
var endpoint = new EndpointAddress(address);
var instanceContext = new InstanceContext(new ClientNotifyHandler());
_channelFactory = new DuplexChannelFactory <IDistrCalcService>(instanceContext, binding);
_clientService = _channelFactory.CreateChannel(endpoint);
Console.WriteLine("Input files path:");
Path = Console.ReadLine();
var files = LoadFile(Path);
if (files == null)
{
throw new ArgumentNullException(nameof(files));
}
Console.WriteLine("Input your name:");
Name = Console.ReadLine();
if (!string.IsNullOrWhiteSpace(Name))
{
Console.WriteLine("If the server does not respond wait for a minute...");
_clientService.SubscribeClient(Name);
var data = new DataForProcessing {
FileList = files
};
_clientService.SendData(data);
}
else
{
Console.WriteLine("Incorrect input name.");
}
Console.WriteLine("Press the Enter key to exit the program.");
Console.ReadKey();
_clientService.UnSubscribeClient();
_channelFactory.Close();
}
catch (DirectoryNotFoundException)
{
Console.WriteLine("You did not input a valid path");
Console.WriteLine("Press the Enter key to exit the program.");
Console.ReadKey();
}
catch (EndpointNotFoundException)
{
Console.WriteLine("Server isn't available. Please try later...");
Console.ReadKey();
}
catch (Exception exception) { Console.WriteLine(exception.Message); Console.ReadKey(); }
}
