static void Main(string[] args)
{
using (new MPI.Environment(ref args))
{
Intracommunicator icomm = Communicator.world;
//tags
const int master = 0;
const int requestURL = 1;
const int passURL = 2;
const int headings = 3;
const int recivedHeading = 4;
const int killYourself = 5;
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
if (icomm.Rank == 0) // Master
{
int slavesNumber = icomm.Size - 1;
List <string> addressList = new List <string>();
List <string> resultsList = new List <string>();
if (slavesNumber < 1)
{
Console.WriteLine("MASTER has no SLAVES to work...");
foreach (var item in resultsList)
{
Console.WriteLine(item);
}
return;
}
//open file and add all lines to: List<string> addressList
try
{
using (StreamReader streamreader = new StreamReader("domains.txt"))
{
string line = streamreader.ReadLine();
while (line != null)
{
addressList.Add(line);
line = streamreader.ReadLine();
}
}
}
catch
{
Console.WriteLine("domains.txt - unable to open file or file doesn't exist");
return;
}
// Master work loop
while (true)
{
int tag = icomm.Probe(Communicator.anySource, Communicator.anyTag).Tag; //get incoming tag
int slaveRank = icomm.Probe(Communicator.anySource, Communicator.anyTag).Source; //get icoming rank
switch (tag)
{
case requestURL:
icomm.Receive <string>(Communicator.anySource, Communicator.anyTag); // recive request
Console.WriteLine("\nMASTER: recived URL request from SLAVE " + slaveRank);
if (addressList.Count > 0)
{
Console.WriteLine("MASTER: is sending URL to SLAVE " + slaveRank);
icomm.Send <string>(addressList[0], slaveRank, passURL);
Console.WriteLine("MASTER: has just sent URL " + addressList[0] + " to SLAVE " + slaveRank);
addressList.RemoveAt(0); //remove sent URL from: List<string> addressList
}
else
{
Console.WriteLine("-----No more work... MASTER is sending KILL TAG to SLAVE " + slaveRank + "-----");
icomm.Send <string>(String.Empty, slaveRank, killYourself);
slavesNumber--;
}
break;
case headings:
Console.WriteLine("MASTER: is receiving output from SLAVE " + slaveRank);
string[] incomingArr = icomm.Receive <string[]>(Communicator.anySource, Communicator.anyTag);
Console.WriteLine("MASTER: has just received output from SLAVE " + slaveRank);
//adding results to list
foreach (var item in incomingArr)
{
resultsList.Add(item);
}
Console.WriteLine("MASTER: is sending confirmation message to SLAVE " + slaveRank);
icomm.Send <string>(String.Empty, slaveRank, recivedHeading);
break;
default:
break;
}
if (slavesNumber < 1)
{
Console.WriteLine();
Thread.Sleep(2800);
Console.WriteLine("-------------Program output-------------");
Console.WriteLine();
foreach (var item in resultsList)
{
Console.WriteLine(item);
}
return;
}
}
}
else // Slave work loop
{
Console.WriteLine("SLAVE " + icomm.Rank + ": sends URL request to MASTER");
icomm.Send <string>(String.Empty, master, requestURL);
while (true)
{
Console.WriteLine("SLAVE " + icomm.Rank + ": is waiting for a response from MASTER");
int incomingTag = icomm.Probe(master, Communicator.anyTag).Tag;
string message = icomm.Receive <string>(master, Communicator.anyTag);
switch (incomingTag)
{
case passURL:
Console.WriteLine("\nSLAVE " + icomm.Rank + " : received URL:");
List <string> allResults = new List <string>();
using (WebClient client = new WebClient())
{
client.Encoding = System.Text.Encoding.UTF8;
Console.WriteLine("SLAVE is downloading: " + message);
string htmlCode = client.DownloadString(message);
string pattern = "(<h1>(.*?)</h1>)|(<h2>(.*?)</h2>)";
MatchCollection matches;
Console.WriteLine("SLAVE " + icomm.Rank + ": is working now....");
matches = Regex.Matches(htmlCode, pattern, RegexOptions.IgnoreCase);
foreach (Match mt in matches)
{
string substring = mt.Value.Substring(4, mt.Value.Length - 9);
while (substring.Contains(">") && substring.Contains("<"))
{
string temp = Regex.Replace(substring, "<.*?>", String.Empty);
substring = temp;
}
allResults.Add(substring);
}
string[] resultArr = allResults.ToArray();
Console.WriteLine("SLAVE " + icomm.Rank + ": is sending result to MASTER....");
icomm.Send <string[]>(resultArr, master, headings);
}
break;
case recivedHeading:
Console.WriteLine("\nSLAVE " + icomm.Rank + ": has received delivery confirmation and is requesting another URL now...");
icomm.Send <string>(String.Empty, master, requestURL);
break;
case killYourself:
Console.WriteLine("SLAVE " + icomm.Rank + ": KILLED");
return;
default:
break;
}
}
}
}
}