/// <summary>
/// Check whether the specified url contents have changed since last time we collected keywords
/// </summary>
public async Task <bool> PageUpdated(string pageUrl, DateTime dateCreated, DateTime?dateUpdated)
{
if (dateCreated == DateTime.MinValue && (dateUpdated == null || dateUpdated == DateTime.MinValue))// if no one supplied a date, we don't know
{
return(true);
}
using (SearchDataUrlDataProvider searchUrlDP = new SearchDataUrlDataProvider()) {
SearchDataUrl searchUrl = await searchUrlDP.GetItemByUrlAsync(pageUrl);
if (searchUrl == null)
{
return(true);
}
DateTime dataAge = searchUrl.DatePageCreated;
if (searchUrl.DatePageUpdated != null && (DateTime)searchUrl.DatePageUpdated < dataAge)
{
dataAge = (DateTime)searchUrl.DatePageUpdated;
}
DateTime newAge = dateCreated;
if (dateUpdated != null && (DateTime)dateUpdated > newAge)
{
newAge = (DateTime)dateUpdated;
}
if (dataAge.AddSeconds(1) < newAge) // allow for slight difference in timestamps
{
return(true);
}
// update the dateadded datetimes for all search terms on this page to reflect that we didn't search and just accept them again
await MarkUpdatedAsync(searchUrl.SearchDataUrlId);
}
return(false);
}
public async Task <bool> AddItemsAsync(List <SearchData> list, string pageUrl, PageDefinition.PageSecurityType pageSecurity, string pageDescription, string pageSummary, DateTime pageCreated, DateTime?pageUpdated, DateTime searchStarted, string customData)
{
if (!IsUsable)
{
return(false);
}
bool status = false;
using (ILockObject lockObject = await YetaWF.Core.IO.Caching.LockProvider.LockResourceAsync($"{AreaRegistration.CurrentPackage.AreaName}_{nameof(SearchDataProvider)}")) {
if (pageUpdated != null && (DateTime)pageUpdated < pageCreated)
{
pageCreated = (DateTime)pageUpdated;
}
using (SearchDataUrlDataProvider searchUrlDP = new SearchDataUrlDataProvider()) {
SearchDataUrl searchUrl = await searchUrlDP.GetItemByUrlAsync(pageUrl);
if (searchUrl == null)
{
searchUrl = new SearchDataUrl {
DatePageCreated = pageCreated,
DatePageUpdated = pageUpdated,
PageTitle = pageDescription.Truncate(SearchDataUrl.MaxTitle),
PageUrl = pageUrl,
PageSecurity = pageSecurity,
PageSummary = pageSummary.Truncate(SearchDataUrl.MaxSummary),
CustomData = customData,
};
if (!await searchUrlDP.AddItemAsync(searchUrl))
{
throw new InternalError("Unexpected error adding SearchDataUrl for url {0}", pageUrl);
}
}
else
{
searchUrl.PageTitle = pageDescription.Truncate(SearchDataUrl.MaxTitle);
searchUrl.PageSummary = pageSummary.Truncate(SearchDataUrl.MaxSummary);
searchUrl.DatePageCreated = pageCreated;
searchUrl.DatePageUpdated = pageUpdated ?? pageCreated;
searchUrl.PageSecurity = pageSecurity;
searchUrl.CustomData = customData;
UpdateStatusEnum updStatus = await searchUrlDP.UpdateItemAsync(searchUrl);
if (updStatus != UpdateStatusEnum.OK)
{
throw new InternalError("Unexpected error updating SearchDataUrl for url {0} - {1}", pageUrl, updStatus);
}
}
foreach (SearchData data in list)
{
data.SearchDataUrlId = searchUrl.SearchDataUrlId;
data.DateAdded = searchStarted;
await AddItemAsync(data);
}
}
status = true;
await lockObject.UnlockAsync();
}
return(status);
}
public async Task <DataProviderGetRecords <SearchData> > GetItemsWithUrlAsync(int skip, int take, List <DataProviderSortInfo> sort, List <DataProviderFilterInfo> filters)
{
if (!IsUsable)
{
return(new DataProviderGetRecords <SearchData>());
}
using (SearchDataUrlDataProvider searchUrlDP = new SearchDataUrlDataProvider()) {
List <JoinData> joins = new List <JoinData> {
new JoinData {
MainDP = this, JoinDP = searchUrlDP, MainColumn = nameof(SearchData.SearchDataUrlId), JoinColumn = nameof(SearchDataUrl.SearchDataUrlId)
},
};
return(await DataProvider.GetRecordsAsync(skip, take, sort, filters, Joins : joins));
}
}
internal async Task <SearchData> GetItemWithUrlAsync(int searchDataId)
{
if (!IsUsable)
{
return(null);
}
using (SearchDataUrlDataProvider searchUrlDP = new SearchDataUrlDataProvider()) {
List <DataProviderFilterInfo> filters = null;
filters = DataProviderFilterInfo.Join(filters, new DataProviderFilterInfo {
Field = nameof(SearchData.SearchDataId), Operator = "==", Value = searchDataId
});
List <JoinData> joins = new List <JoinData> {
new JoinData {
MainDP = this, JoinDP = searchUrlDP, MainColumn = nameof(SearchDataUrl.SearchDataUrlId), JoinColumn = nameof(SearchData.SearchDataUrlId)
},
};
return(await DataProvider.GetOneRecordAsync(filters, Joins : joins));
}
}
public async Task <int> RemoveItemsAsync(List <DataProviderFilterInfo> filters)
{
if (!IsUsable)
{
return(0);
}
int count = 0;
count = await DataProvider.RemoveRecordsAsync(filters);
if (filters == null)
{
using (SearchDataUrlDataProvider searchUrlDP = new SearchDataUrlDataProvider()) {
await searchUrlDP.RemoveItemsAsync(null);
}
}
else
{
await RemoveUnusedUrlsAsync();
}
return(count);
}
