private async Task ProcessApartmentMessagesAsync(Message message, CancellationToken token)
{
String messageContent = Encoding.UTF8.GetString(message.Body);
Console.WriteLine("Received message " + messageContent);
WebScraperModularized.data.Message apartmentMessage =
JsonConvert.DeserializeObject <WebScraperModularized.data.Message>(
messageContent);
_apartmentService.processAndSaveApartmentsFromPropertyPage(apartmentMessage);
await apartmentQueueClient.CompleteAsync(message.SystemProperties.LockToken);
}
private async Task ProcessPropertyMessagesAsync(Message message, CancellationToken token)
{
String messageContent = Encoding.UTF8.GetString(message.Body);
Console.WriteLine("Received message " + messageContent);
WebScraperModularized.data.Message propertyMessage =
JsonConvert.DeserializeObject <WebScraperModularized.data.Message>(
messageContent);
List <WebScraperModularized.data.Message> listOfPropertyUrl =
_propertyService.getPropertyUrlsFromPropertyListPage(propertyMessage);
await sendListOfMessagesToQueue(listOfPropertyUrl, "apartmentqueue");
await propertyQueueClient.CompleteAsync(message.SystemProperties.LockToken);
}
private async Task ProcessZipcodeMessagesAsync(Message message, CancellationToken token)
{
String messageContent = Encoding.UTF8.GetString(message.Body);
Console.WriteLine("Received message " + messageContent);
WebScraperModularized.data.Message zipcodeMessage =
JsonConvert.DeserializeObject <WebScraperModularized.data.Message>(
messageContent);
List <WebScraperModularized.data.Message> listOfUrlsToBeParsedForTheGivenPostcode =
_zipcodeService.ProcessZipcodeMessagesAsync(zipcodeMessage.zipcode);
await sendListOfMessagesToQueue(listOfUrlsToBeParsedForTheGivenPostcode, "propertyqueue");
await zipcodeQueueClient.CompleteAsync(message.SystemProperties.LockToken);
}
public List <WebScraperModularized.data.Message> ProcessZipcodeMessagesAsync(String zipcode)
{
String baseurl = SEED_URL + zipcode;
var response = client.GetAsync(baseurl).Result; //make an HTTP call and get the html for this URL
string content = response.Content.ReadAsStringAsync().Result; //save HTML into string
//parse the baseurl for that postcode to get the total number of pages for that url
ZipcodePropertyUrlParser zipcodePropertyUrlParser = new ZipcodePropertyUrlParser(content, baseurl);
zipcodePropertyUrlParser.parse();
List <WebScraperModularized.data.Message> messages = new List <Message>();
foreach (var each in zipcodePropertyUrlParser.propertyUrls)
{
Message message = new Message();
message.url = each;
message.zipcode = zipcode;
messages.Add(message);
}
//send the total urls to the propertyQueue so that it can be processed by the Property Parser
return(messages);
}
public async Task sendMessageToQueue(WebScraperModularized.data.Message message, String queueName)
{
var azureMessage = new Message(Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(message)));
await SendMessageAsync((IQueueClient)mapOfQueues[queueName], azureMessage);
}
