public static void AddSite(
// the incoming queue
[QueueTrigger(SiteMonitRConfiguration.QUEUE_NAME_NEW_SITE)] string url,
// the list of all sites
[Table(SiteMonitRConfiguration.TABLE_NAME_SITES)]
IQueryable <SiteRecord> listOfSiteRecords,
// the table into which sites should be saved
[Table(SiteMonitRConfiguration.TABLE_NAME_SITES)]
CloudTable table
)
{
var cleansedUrl = SiteMonitRConfiguration.CleanUrlForRowKey(url);
var siteRecord = new SiteRecord();
siteRecord.RowKey = SiteMonitRConfiguration.GetPartitionKey();
siteRecord.PartitionKey = cleansedUrl;
siteRecord.Uri = url;
if (!listOfSiteRecords.ToList().Any(entity => entity.PartitionKey == siteRecord.PartitionKey))
{
table.Execute(TableOperation.InsertOrReplace(siteRecord));
}
}
//
// Event Methods
//
private void OnPatchCreated(Patch patch, string siteName)
{
#if SAFETY_CHECK
if (patch.Level < 0 || patch.Level >= levels.Length)
{
Debug.LogError("Patch " + patch.Filename + " has invalid level: " + patch.Level);
return;
}
#endif
var level = levels[patch.Level];
SiteRecord siteRecord = null;
foreach (var layerSite in level.layerSites)
{
if (siteName == layerSite.Site.Name)
{
siteRecord = layerSite.Add(patch);
break;
}
}
if (siteRecord == null)
{
var site = dataManager.GetOrAddSite(siteName);
var layerSite = level.AddSite(site, patch);
siteRecord = layerSite.LastRecord;
}
patch.SetSiteRecord(siteRecord);
}
//
// Event Methods
//
private void OnPatchCreated(Patch patch)
{
#if SAFETY_CHECK
if (patch.level < 0 || patch.level >= levels.Length)
{
Debug.LogError("Patch " + patch.name + "(" + patch.Filename + ") has invalid level: " + patch.level);
return;
}
#endif
var level = levels[patch.level];
SiteRecord siteRecord = null;
foreach (var layerSite in level.layerSites)
{
if (patch.name.Equals(layerSite.name))
{
siteRecord = layerSite.Add(patch);
break;
}
}
if (siteRecord == null)
{
var site = level.AddSite(patch.name, patch);
siteRecord = site.lastRecord;
}
patch.siteRecord = siteRecord;
}
public LayerSite(Level level, string name, Patch patch)
{
this.level = level;
this.name = name;
lastRecord = new SiteRecord(this, patch);
records.Add(lastRecord.year, lastRecord);
}
private void Add(SiteRecord record)
{
records.Add(record.Year, record);
if (LastRecord == null || record.Year > LastRecord.Year)
{
LastRecord = record;
}
}
public void Add(SiteRecord record)
{
records.Add(record.year, record);
if (lastRecord == null || record.year > lastRecord.year )
{
lastRecord = record;
}
}
public SiteRecord Add(Patch patch)
{
if (records.TryGetValue(patch.Year, out SiteRecord siteRecord))
{
siteRecord.Add(patch);
}
else
{
siteRecord = new SiteRecord(this, patch);
Add(siteRecord);
}
return(siteRecord);
}
public bool Post(SiteRecord site)
{
try
{
_helper.QueueNewTrackedSite(new SiteRecordTableEntity
{
Uri = site.Uri
});
return(true);
}
catch
{
return(false);
}
}
public SiteRecord Add(Patch patch)
{
SiteRecord siteRecord;
if (records.ContainsKey(patch.year))
{
siteRecord = records[patch.year];
siteRecord.patches.Add(patch);
}
else
{
siteRecord = new SiteRecord(this, patch);
Add(siteRecord);
}
return siteRecord;
}
public void ChangeYear(int oldYear, int newYear)
{
if (!records.TryGetValue(oldYear, out SiteRecord record))
{
return;
}
records.Remove(oldYear);
records.Add(newYear, record);
record.ChangeYear(newYear);
// Update last record
foreach (var pair in records)
{
if (pair.Key > LastRecord.Year)
{
LastRecord = pair.Value;
}
}
}
private static SiteRecord Crawl(SiteRecord record)
{
const int MIN_P_LENGTH = 15;
const int MAX_P_LENGTH = 157;
try
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(record.url);
request.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.83 Safari/535.11";
var doc = new HtmlDocument();
doc.Load(request.GetResponse().GetResponseStream());
record.title = doc.DocumentNode.SelectSingleNode("//title").InnerText;
if (record.title.IsNullEmptyOrWhitespace())
{
return null;
}
foreach (var pElem in doc.DocumentNode.SelectNodes("//p"))
{
record.textPreview = pElem.InnerText.Trim();
if (record.textPreview.Length < MIN_P_LENGTH)
{
continue;
}
else
{
if (record.textPreview.Length > MAX_P_LENGTH + 3)
{
record.textPreview = record.textPreview.Substring(0, MAX_P_LENGTH) + "...";
}
break;
}
}
if (record.textPreview.Length < MIN_P_LENGTH)
{
return null;
}
}
catch { }
return record;
}
public static void Add(SiteRecord record)
{
// TODO: implement
}
public void SetSiteRecord(SiteRecord siteRecord)
{
SiteRecord = siteRecord;
}
protected virtual void AddNewRecords()
{
ArrayList newRecordList = new ArrayList();
System.Collections.Generic.List<Hashtable> newUIDataList = new System.Collections.Generic.List<Hashtable>();
// Loop though all the record controls and if the record control
// does not have a unique record id set, then create a record
// and add to the list.
if (!this.ResetData)
{
System.Web.UI.WebControls.Repeater rep = (System.Web.UI.WebControls.Repeater)(BaseClasses.Utils.MiscUtils.FindControlRecursively(this, "SiteTableControlRepeater"));
if (rep == null){return;}
foreach (System.Web.UI.WebControls.RepeaterItem repItem in rep.Items)
{
// Loop through all rows in the table, set its DataSource and call DataBind().
SiteTableControlRow recControl = (SiteTableControlRow)(repItem.FindControl("SiteTableControlRow"));
if (recControl.Visible && recControl.IsNewRecord) {
SiteRecord rec = new SiteRecord();
if (recControl.SiteName.Text != "") {
rec.Parse(recControl.SiteName.Text, SiteTable.SiteName);
}
if (recControl.SiteNo.Text != "") {
rec.Parse(recControl.SiteNo.Text, SiteTable.SiteNo);
}
if (recControl.Status.Text != "") {
rec.Parse(recControl.Status.Text, SiteTable.Status);
}
newUIDataList.Add(recControl.PreservedUIData());
newRecordList.Add(rec);
}
}
}
// Add any new record to the list.
for (int count = 1; count <= this.AddNewRecord; count++) {
newRecordList.Insert(0, new SiteRecord());
newUIDataList.Insert(0, new Hashtable());
}
this.AddNewRecord = 0;
// Finally, add any new records to the DataSource.
if (newRecordList.Count > 0) {
ArrayList finalList = new ArrayList(this.DataSource);
finalList.InsertRange(0, newRecordList);
Type myrec = typeof(FPCEstimate.Business.SiteRecord);
this.DataSource = (SiteRecord[])(finalList.ToArray(myrec));
}
// Add the existing UI data to this hash table
if (newUIDataList.Count > 0)
this.UIData.InsertRange(0, newUIDataList);
}
