public List<Message> getPropertyUrlsFromPropertyListPage(WebScraperModularized.data.Message propertyPageMessage) { URL myUrl = new URL(); myUrl.url = propertyPageMessage.url; myUrl.zip_code = int.Parse(propertyPageMessage.zipcode); var response = client.GetAsync(myUrl.url).Result; //make an HTTP call and get the html for this URL string content = response.Content.ReadAsStringAsync().Result; //save HTML into string PropertyParser parser = new PropertyParser(content, myUrl); //parse the html PropertyData propData = parser.parse(); //insert into DB DBHelper.insertParsedProperties(propData); Console.WriteLine("Stored {0} properties", (propData != null && propData.urlList != null) ? propData.urlList.Count : 0); List<Message> listOfPropertyUrl = new List<Message>(); foreach (var each in propData.urlList) { each.properties.ForEach(eachProperty => { Message message = new Message(); message.url = eachProperty.url.url; message.id = eachProperty.id; listOfPropertyUrl.Add(message); }); } return listOfPropertyUrl; }
public void processAndSaveApartmentsFromPropertyPage(WebScraperModularized.data.Message propertyMessage) { URL myUrl = new URL(); myUrl.url = propertyMessage.url; myUrl.property = propertyMessage.id; var response = client.GetAsync(myUrl.url).Result; //make an HTTP call and get the html for this URL string content = response.Content.ReadAsStringAsync().Result; //save HTML into string ApartmentParser parser = new ApartmentParser(content, myUrl); //call the parse method ApartmentData apartmentData = parser.parse(); DBHelper.insertParsedApartment(apartmentData); Console.WriteLine("Stored data for property id {0}!", myUrl.property); }
static void Main(string[] args) { doDapperMapping(); IQueueService queueService = AzureServiceBusService.Instance; List <string> zipcodes = File.ReadAllLines("zipcode.dat").ToList(); List <Message> messages = new List <Message>(); foreach (var zipcode in zipcodes) { Message message = new Message(); message.zipcode = zipcode; Console.WriteLine("Going to send zipcode " + zipcode + " to zipcode queue"); messages.Add(message); } queueService.sendListOfMessagesToQueue(messages, "zipcodequeue"); String input = "process"; while (!input.SequenceEqual("exit")) { Console.WriteLine("Type exit to exit processing"); input = Console.ReadLine(); } }