public List<Message> getPropertyUrlsFromPropertyListPage(WebScraperModularized.data.Message propertyPageMessage)
{
URL myUrl = new URL();
myUrl.url = propertyPageMessage.url;
myUrl.zip_code = int.Parse(propertyPageMessage.zipcode);
var response = client.GetAsync(myUrl.url).Result; //make an HTTP call and get the html for this URL
string content = response.Content.ReadAsStringAsync().Result; //save HTML into string
PropertyParser parser = new PropertyParser(content, myUrl);
//parse the html
PropertyData propData = parser.parse();
//insert into DB
DBHelper.insertParsedProperties(propData);
Console.WriteLine("Stored {0} properties",
(propData != null && propData.urlList != null) ? propData.urlList.Count : 0);
List<Message> listOfPropertyUrl = new List<Message>();
foreach (var each in propData.urlList)
{
each.properties.ForEach(eachProperty =>
{
Message message = new Message();
message.url = eachProperty.url.url;
message.id = eachProperty.id;
listOfPropertyUrl.Add(message);
});
}
return listOfPropertyUrl;
}
public void processAndSaveApartmentsFromPropertyPage(WebScraperModularized.data.Message propertyMessage)
{
URL myUrl = new URL();
myUrl.url = propertyMessage.url;
myUrl.property = propertyMessage.id;
var response = client.GetAsync(myUrl.url).Result; //make an HTTP call and get the html for this URL
string content = response.Content.ReadAsStringAsync().Result; //save HTML into string
ApartmentParser parser = new ApartmentParser(content, myUrl);
//call the parse method
ApartmentData apartmentData = parser.parse();
DBHelper.insertParsedApartment(apartmentData);
Console.WriteLine("Stored data for property id {0}!", myUrl.property);
}
static void Main(string[] args)
{
doDapperMapping();
IQueueService queueService = AzureServiceBusService.Instance;
List <string> zipcodes = File.ReadAllLines("zipcode.dat").ToList();
List <Message> messages = new List <Message>();
foreach (var zipcode in zipcodes)
{
Message message = new Message();
message.zipcode = zipcode;
Console.WriteLine("Going to send zipcode " + zipcode + " to zipcode queue");
messages.Add(message);
}
queueService.sendListOfMessagesToQueue(messages, "zipcodequeue");
String input = "process";
while (!input.SequenceEqual("exit"))
{
Console.WriteLine("Type exit to exit processing");
input = Console.ReadLine();
}
}