/// <summary>
/// Record a network issue has occurred trying to connect to this ip address
/// </summary>
/// <param name="ipAddress"></param>
/// <returns></returns>
public async Task RecordNetworkIssue(IPAddress ipAddress)
{
HostCacheManagerRecord record = await GetRecord(ipAddress.ToString());
if (record == null)
{
return;
}
record.NetworkIssues++;
}
public async Task UpdateHostSettings(HostInfo info)
{
HostCacheManagerRecord record = await GetRecord(info.Host);
if (record == null)
{
return;
}
record.SetFromHostInfo(info);
}
public async Task <HostInfo> GetHostInfo(string host)
{
HostCacheManagerRecord record = await GetRecord(host);
if (record == null)
{
return(HostInfo.None);
}
return(record.GetHostInfo());
}
/// <summary>
/// Delay until we can fetch
/// </summary>
/// <param name="hostName"></param>
/// <returns>True if we can fetch, false if timeout or cancelled</returns>
public async Task <bool> WaitToFetch(string hostName, int timeoutMilliseconds)
{
DateTime startTime = DateTime.UtcNow;
bool keepWaiting = true;
bool success = false;
HostCacheManagerRecord host_record = await GetRecord(hostName, true);
while (keepWaiting)
{
try
{
while (!await host_record.FetchWaitHandle.WaitAsync(360000))
{
Utility.LogInfo("Been waiting a long time to update a record: {0}", host_record.Host);
}
if (host_record.IsFetchable)
{
host_record.LastCall = DateTime.UtcNow;
host_record.TouchCount++;
keepWaiting = false;
success = true;
}
}
catch (Exception ex)
{
Utility.LogException(ex);
}
finally
{
host_record.FetchWaitHandle.Release();
}
if (keepWaiting)
{
if (timeoutMilliseconds != Timeout.Infinite && (DateTime.UtcNow - startTime).TotalMilliseconds > timeoutMilliseconds)
{
keepWaiting = false;
}
else
{
await Task.Delay(host_record.MaxFetchSpeedInMilliseconds);
}
}
}
return(success);
}
/// <summary>
/// Retrives a record from the cache or creates a new one
/// </summary>
/// <param name="fromHost"></param>
/// <returns></returns>
async Task <HostCacheManagerRecord> GetRecord(string fromHost, bool createIfNotExists = false)
{
HostCacheManagerRecord record = null;
try
{
while (!await hosts_lock.WaitAsync(360000))
{
Utility.LogInfo("GetRecord waiting {0}", fromHost);
}
bool exists = hosts.ContainsKey(fromHost);
// make it if its not there
if (createIfNotExists)
{
if (!exists)
{
hosts.Add(fromHost,
new HostCacheManagerRecord()
{
Host = fromHost,
LastCall = DateTime.MinValue,
RobotsChecked = false,
}
);
}
else
{
// bump the host
BumpHost(fromHost);
}
exists = true;
}
if (exists)
{
// if there's too many domains in memory we drop one to avoid filling up memory with cached robots objects
if (exists && recencyQueue.Count > FetchoConfiguration.Current.HostCacheManagerMaxInMemoryDomainRecords)
{
string domainToDrop = recencyQueue.Dequeue();
record = hosts[domainToDrop];
record.Dispose();
hosts.Remove(domainToDrop);
}
record = hosts[fromHost];
}
}
catch (Exception ex)
{
Utility.LogException(ex);
}
finally
{
hosts_lock.Release();
}
return(record);
}
/// <summary> /// Returns how much time we'd have to wait before fetching /// </summary> /// <param name="fromHost"></param> /// <returns></returns> int GetTimeRemainingToWait(HostCacheManagerRecord host_record) => host_record.MaxFetchSpeedInMilliseconds - (int)((DateTime.UtcNow - host_record.LastCall).TotalMilliseconds);
public async Task <bool> HasHostExceedNetworkIssuesThreshold(IPAddress ipAddress)
{
HostCacheManagerRecord record = await GetRecord(ipAddress.ToString());
return(record != null && record.NetworkIssues > FetchoConfiguration.Current.MaxNetworkIssuesThreshold);
}
