/// <summary>
/// Check whether the specified url contents have changed since last time we collected keywords
/// </summary>
public async Task <bool> PageUpdated(string pageUrl, DateTime dateCreated, DateTime?dateUpdated)
{
if (dateCreated == DateTime.MinValue && (dateUpdated == null || dateUpdated == DateTime.MinValue))// if no one supplied a date, we don't know
{
return(true);
}
using (SearchDataUrlDataProvider searchUrlDP = new SearchDataUrlDataProvider()) {
SearchDataUrl searchUrl = await searchUrlDP.GetItemByUrlAsync(pageUrl);
if (searchUrl == null)
{
return(true);
}
DateTime dataAge = searchUrl.DatePageCreated;
if (searchUrl.DatePageUpdated != null && (DateTime)searchUrl.DatePageUpdated < dataAge)
{
dataAge = (DateTime)searchUrl.DatePageUpdated;
}
DateTime newAge = dateCreated;
if (dateUpdated != null && (DateTime)dateUpdated > newAge)
{
newAge = (DateTime)dateUpdated;
}
if (dataAge.AddSeconds(1) < newAge) // allow for slight difference in timestamps
{
return(true);
}
// update the dateadded datetimes for all search terms on this page to reflect that we didn't search and just accept them again
await MarkUpdatedAsync(searchUrl.SearchDataUrlId);
}
return(false);
}
public async Task <bool> AddItemsAsync(List <SearchData> list, string pageUrl, PageDefinition.PageSecurityType pageSecurity, string pageDescription, string pageSummary, DateTime pageCreated, DateTime?pageUpdated, DateTime searchStarted, string customData)
{
if (!IsUsable)
{
return(false);
}
bool status = false;
using (ILockObject lockObject = await YetaWF.Core.IO.Caching.LockProvider.LockResourceAsync($"{AreaRegistration.CurrentPackage.AreaName}_{nameof(SearchDataProvider)}")) {
if (pageUpdated != null && (DateTime)pageUpdated < pageCreated)
{
pageCreated = (DateTime)pageUpdated;
}
using (SearchDataUrlDataProvider searchUrlDP = new SearchDataUrlDataProvider()) {
SearchDataUrl searchUrl = await searchUrlDP.GetItemByUrlAsync(pageUrl);
if (searchUrl == null)
{
searchUrl = new SearchDataUrl {
DatePageCreated = pageCreated,
DatePageUpdated = pageUpdated,
PageTitle = pageDescription.Truncate(SearchDataUrl.MaxTitle),
PageUrl = pageUrl,
PageSecurity = pageSecurity,
PageSummary = pageSummary.Truncate(SearchDataUrl.MaxSummary),
CustomData = customData,
};
if (!await searchUrlDP.AddItemAsync(searchUrl))
{
throw new InternalError("Unexpected error adding SearchDataUrl for url {0}", pageUrl);
}
}
else
{
searchUrl.PageTitle = pageDescription.Truncate(SearchDataUrl.MaxTitle);
searchUrl.PageSummary = pageSummary.Truncate(SearchDataUrl.MaxSummary);
searchUrl.DatePageCreated = pageCreated;
searchUrl.DatePageUpdated = pageUpdated ?? pageCreated;
searchUrl.PageSecurity = pageSecurity;
searchUrl.CustomData = customData;
UpdateStatusEnum updStatus = await searchUrlDP.UpdateItemAsync(searchUrl);
if (updStatus != UpdateStatusEnum.OK)
{
throw new InternalError("Unexpected error updating SearchDataUrl for url {0} - {1}", pageUrl, updStatus);
}
}
foreach (SearchData data in list)
{
data.SearchDataUrlId = searchUrl.SearchDataUrlId;
data.DateAdded = searchStarted;
await AddItemAsync(data);
}
}
status = true;
await lockObject.UnlockAsync();
}
return(status);
}
public async Task <bool> AddItemAsync(SearchDataUrl data)
{
if (!SearchDataProvider.IsUsable)
{
return(false);
}
return(await DataProvider.AddAsync(data));
}
public async Task <UpdateStatusEnum> UpdateItemAsync(SearchDataUrl data)
{
if (!SearchDataProvider.IsUsable)
{
return(UpdateStatusEnum.RecordDeleted);
}
return(await DataProvider.UpdateByIdentityAsync(data.SearchDataUrlId, data));
}
internal async Task <SearchDataUrl> GetItemByUrlAsync(string pageUrl)
{
if (!SearchDataProvider.IsUsable)
{
return(null);
}
List <DataProviderFilterInfo> filters = null;
filters = DataProviderFilterInfo.Join(filters, new DataProviderFilterInfo {
Field = nameof(SearchDataUrl.PageUrl), Operator = "==", Value = pageUrl
});
SearchDataUrl searchUrl = await DataProvider.GetOneRecordAsync(filters);
return(searchUrl);
}