/// <summary> /// Check whether the specified url contents have changed since last time we collected keywords /// </summary> public async Task <bool> PageUpdated(string pageUrl, DateTime dateCreated, DateTime?dateUpdated) { if (dateCreated == DateTime.MinValue && (dateUpdated == null || dateUpdated == DateTime.MinValue))// if no one supplied a date, we don't know { return(true); } using (SearchDataUrlDataProvider searchUrlDP = new SearchDataUrlDataProvider()) { SearchDataUrl searchUrl = await searchUrlDP.GetItemByUrlAsync(pageUrl); if (searchUrl == null) { return(true); } DateTime dataAge = searchUrl.DatePageCreated; if (searchUrl.DatePageUpdated != null && (DateTime)searchUrl.DatePageUpdated < dataAge) { dataAge = (DateTime)searchUrl.DatePageUpdated; } DateTime newAge = dateCreated; if (dateUpdated != null && (DateTime)dateUpdated > newAge) { newAge = (DateTime)dateUpdated; } if (dataAge.AddSeconds(1) < newAge) // allow for slight difference in timestamps { return(true); } // update the dateadded datetimes for all search terms on this page to reflect that we didn't search and just accept them again await MarkUpdatedAsync(searchUrl.SearchDataUrlId); } return(false); }
public async Task <bool> AddItemsAsync(List <SearchData> list, string pageUrl, PageDefinition.PageSecurityType pageSecurity, string pageDescription, string pageSummary, DateTime pageCreated, DateTime?pageUpdated, DateTime searchStarted, string customData) { if (!IsUsable) { return(false); } bool status = false; using (ILockObject lockObject = await YetaWF.Core.IO.Caching.LockProvider.LockResourceAsync($"{AreaRegistration.CurrentPackage.AreaName}_{nameof(SearchDataProvider)}")) { if (pageUpdated != null && (DateTime)pageUpdated < pageCreated) { pageCreated = (DateTime)pageUpdated; } using (SearchDataUrlDataProvider searchUrlDP = new SearchDataUrlDataProvider()) { SearchDataUrl searchUrl = await searchUrlDP.GetItemByUrlAsync(pageUrl); if (searchUrl == null) { searchUrl = new SearchDataUrl { DatePageCreated = pageCreated, DatePageUpdated = pageUpdated, PageTitle = pageDescription.Truncate(SearchDataUrl.MaxTitle), PageUrl = pageUrl, PageSecurity = pageSecurity, PageSummary = pageSummary.Truncate(SearchDataUrl.MaxSummary), CustomData = customData, }; if (!await searchUrlDP.AddItemAsync(searchUrl)) { throw new InternalError("Unexpected error adding SearchDataUrl for url {0}", pageUrl); } } else { searchUrl.PageTitle = pageDescription.Truncate(SearchDataUrl.MaxTitle); searchUrl.PageSummary = pageSummary.Truncate(SearchDataUrl.MaxSummary); searchUrl.DatePageCreated = pageCreated; searchUrl.DatePageUpdated = pageUpdated ?? pageCreated; searchUrl.PageSecurity = pageSecurity; searchUrl.CustomData = customData; UpdateStatusEnum updStatus = await searchUrlDP.UpdateItemAsync(searchUrl); if (updStatus != UpdateStatusEnum.OK) { throw new InternalError("Unexpected error updating SearchDataUrl for url {0} - {1}", pageUrl, updStatus); } } foreach (SearchData data in list) { data.SearchDataUrlId = searchUrl.SearchDataUrlId; data.DateAdded = searchStarted; await AddItemAsync(data); } } status = true; await lockObject.UnlockAsync(); } return(status); }
public async Task <bool> AddItemAsync(SearchDataUrl data) { if (!SearchDataProvider.IsUsable) { return(false); } return(await DataProvider.AddAsync(data)); }
public async Task <UpdateStatusEnum> UpdateItemAsync(SearchDataUrl data) { if (!SearchDataProvider.IsUsable) { return(UpdateStatusEnum.RecordDeleted); } return(await DataProvider.UpdateByIdentityAsync(data.SearchDataUrlId, data)); }
internal async Task <SearchDataUrl> GetItemByUrlAsync(string pageUrl) { if (!SearchDataProvider.IsUsable) { return(null); } List <DataProviderFilterInfo> filters = null; filters = DataProviderFilterInfo.Join(filters, new DataProviderFilterInfo { Field = nameof(SearchDataUrl.PageUrl), Operator = "==", Value = pageUrl }); SearchDataUrl searchUrl = await DataProvider.GetOneRecordAsync(filters); return(searchUrl); }