public string DoTheThing()
{
var watch = Stopwatch.StartNew();
SQLClass SQLClass = new SQLClass();
HTMLCriteriaClass HTMLCriteriaClass = new HTMLCriteriaClass();
string siteContent = string.Empty;
DataTable dtSearchMaster = SQLClass.GetDataTable("SELECT ID, ADVERTTYPEID FROM TABLE_SEARCH_MASTER (NOLOCK) WHERE ISACTIVE = 1", out string Error);
int searchMasterID;
foreach (DataRow item in dtSearchMaster.Rows)
{
searchMasterID = Convert.ToInt32(item["ID"]);
DataTable dtAdvert = SQLClass.GetDataTable("SELECT AdvertID FROM TABLE_ADVERT (NOLOCK) WHERE SearchMasterID = " + searchMasterID, out Error);
List <int> advertDBList = DataTabletoIntList(dtAdvert);
List <int> advertWebList = new List <int>();
int advertTypeID = Convert.ToInt32(item["ADVERTTYPEID"]);
bool contiuneOnNextPage = true;
int currentPage = 1;
string siteAddress;
while (contiuneOnNextPage)
{
List <int> advertWebList_ = new List <int>();
siteAddress = SQLClass.GetSingleCellDataComplex("SP_GETSEARCHURL " + searchMasterID.ToString() + ", " + currentPage.ToString());
using (HttpClient client = new HttpClient())
{
client.DefaultRequestHeaders.Add("user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36");
using HttpResponseMessage response = client.GetAsync(siteAddress).Result;
using HttpContent content = response.Content;
siteContent = content.ReadAsStringAsync().Result;
}
if (siteContent.Contains("too-many-requests"))
{
LogWriter("We are banned :)");
return("");
}
string trimmedSiteContent = TrimHelper(HTMLCriteriaClass.AdvertTrimCriteria, siteContent);
string cleanedSiteContent = WebUtility.HtmlDecode(ReplaceNonAnsiChars(CleanData(trimmedSiteContent)));
List <string> splittedInput = SplitDivisionHelper(HTMLCriteriaClass.AdvertSplitDivisionCriteria, cleanedSiteContent, false);
List <ResultModel> ResultModelList = PopulateResultModel(splittedInput, advertTypeID, searchMasterID, advertDBList, out advertWebList_);
using (DataTable dataTable = ConvertListToDataTable(ResultModelList))
SQLClass.BulkInsert(dataTable, "TABLE_ADVERT");
if (splittedInput.Count < 20)
{
contiuneOnNextPage = false;
}
currentPage++;
advertWebList.AddRange(advertWebList_);
}
if (advertWebList.Count > 0)
{
MarkAsDeleted(advertDBList, advertWebList);
}
SendNotification(searchMasterID);
}
watch.Stop();
LogWriter("Done in " + (watch.ElapsedMilliseconds / 1000).ToString() + " seconds.");
return(Error);
}
public HelperClass()
{
SQLClass = new SQLClass();
HTMLCriteriaClass = new HTMLCriteriaClass();
}
