/// <summary> /// Check whether the specified url contents have changed since last time we collected keywords /// </summary> public async Task <bool> PageUpdated(string pageUrl, DateTime dateCreated, DateTime?dateUpdated) { if (dateCreated == DateTime.MinValue && (dateUpdated == null || dateUpdated == DateTime.MinValue))// if no one supplied a date, we don't know { return(true); } using (SearchDataUrlDataProvider searchUrlDP = new SearchDataUrlDataProvider()) { SearchDataUrl searchUrl = await searchUrlDP.GetItemByUrlAsync(pageUrl); if (searchUrl == null) { return(true); } DateTime dataAge = searchUrl.DatePageCreated; if (searchUrl.DatePageUpdated != null && (DateTime)searchUrl.DatePageUpdated < dataAge) { dataAge = (DateTime)searchUrl.DatePageUpdated; } DateTime newAge = dateCreated; if (dateUpdated != null && (DateTime)dateUpdated > newAge) { newAge = (DateTime)dateUpdated; } if (dataAge.AddSeconds(1) < newAge) // allow for slight difference in timestamps { return(true); } // update the dateadded datetimes for all search terms on this page to reflect that we didn't search and just accept them again await MarkUpdatedAsync(searchUrl.SearchDataUrlId); } return(false); }
public async Task <bool> AddItemsAsync(List <SearchData> list, string pageUrl, PageDefinition.PageSecurityType pageSecurity, string pageDescription, string pageSummary, DateTime pageCreated, DateTime?pageUpdated, DateTime searchStarted, string customData) { if (!IsUsable) { return(false); } bool status = false; using (ILockObject lockObject = await YetaWF.Core.IO.Caching.LockProvider.LockResourceAsync($"{AreaRegistration.CurrentPackage.AreaName}_{nameof(SearchDataProvider)}")) { if (pageUpdated != null && (DateTime)pageUpdated < pageCreated) { pageCreated = (DateTime)pageUpdated; } using (SearchDataUrlDataProvider searchUrlDP = new SearchDataUrlDataProvider()) { SearchDataUrl searchUrl = await searchUrlDP.GetItemByUrlAsync(pageUrl); if (searchUrl == null) { searchUrl = new SearchDataUrl { DatePageCreated = pageCreated, DatePageUpdated = pageUpdated, PageTitle = pageDescription.Truncate(SearchDataUrl.MaxTitle), PageUrl = pageUrl, PageSecurity = pageSecurity, PageSummary = pageSummary.Truncate(SearchDataUrl.MaxSummary), CustomData = customData, }; if (!await searchUrlDP.AddItemAsync(searchUrl)) { throw new InternalError("Unexpected error adding SearchDataUrl for url {0}", pageUrl); } } else { searchUrl.PageTitle = pageDescription.Truncate(SearchDataUrl.MaxTitle); searchUrl.PageSummary = pageSummary.Truncate(SearchDataUrl.MaxSummary); searchUrl.DatePageCreated = pageCreated; searchUrl.DatePageUpdated = pageUpdated ?? pageCreated; searchUrl.PageSecurity = pageSecurity; searchUrl.CustomData = customData; UpdateStatusEnum updStatus = await searchUrlDP.UpdateItemAsync(searchUrl); if (updStatus != UpdateStatusEnum.OK) { throw new InternalError("Unexpected error updating SearchDataUrl for url {0} - {1}", pageUrl, updStatus); } } foreach (SearchData data in list) { data.SearchDataUrlId = searchUrl.SearchDataUrlId; data.DateAdded = searchStarted; await AddItemAsync(data); } } status = true; await lockObject.UnlockAsync(); } return(status); }
public async Task <DataProviderGetRecords <SearchData> > GetItemsWithUrlAsync(int skip, int take, List <DataProviderSortInfo> sort, List <DataProviderFilterInfo> filters) { if (!IsUsable) { return(new DataProviderGetRecords <SearchData>()); } using (SearchDataUrlDataProvider searchUrlDP = new SearchDataUrlDataProvider()) { List <JoinData> joins = new List <JoinData> { new JoinData { MainDP = this, JoinDP = searchUrlDP, MainColumn = nameof(SearchData.SearchDataUrlId), JoinColumn = nameof(SearchDataUrl.SearchDataUrlId) }, }; return(await DataProvider.GetRecordsAsync(skip, take, sort, filters, Joins : joins)); } }
internal async Task <SearchData> GetItemWithUrlAsync(int searchDataId) { if (!IsUsable) { return(null); } using (SearchDataUrlDataProvider searchUrlDP = new SearchDataUrlDataProvider()) { List <DataProviderFilterInfo> filters = null; filters = DataProviderFilterInfo.Join(filters, new DataProviderFilterInfo { Field = nameof(SearchData.SearchDataId), Operator = "==", Value = searchDataId }); List <JoinData> joins = new List <JoinData> { new JoinData { MainDP = this, JoinDP = searchUrlDP, MainColumn = nameof(SearchDataUrl.SearchDataUrlId), JoinColumn = nameof(SearchData.SearchDataUrlId) }, }; return(await DataProvider.GetOneRecordAsync(filters, Joins : joins)); } }
public async Task <int> RemoveItemsAsync(List <DataProviderFilterInfo> filters) { if (!IsUsable) { return(0); } int count = 0; count = await DataProvider.RemoveRecordsAsync(filters); if (filters == null) { using (SearchDataUrlDataProvider searchUrlDP = new SearchDataUrlDataProvider()) { await searchUrlDP.RemoveItemsAsync(null); } } else { await RemoveUnusedUrlsAsync(); } return(count); }