/// <summary>
/// This method gets called by the runtime. Use this method to add services to the container.
/// </summary>
/// <param name="services">The service collection to receive more services.</param>
public void ConfigureServices(IServiceCollection services)
{
services.AddMvc().SetCompatibilityVersion(CompatibilityVersion.Version_2_2);
var rssiSection = this.Configuration.GetSection(Program.RssiAquisitionServiceSectionKey);
if ((rssiSection == null) || rssiSection.GetValue <bool>("Enabled"))
{
services.AddHostedService <RssiAquisitionService>();
}
var bpgSection = this.Configuration.GetSection(Program.BgpAquisitionServiceSectionKey);
if ((bpgSection == null) || bpgSection.GetValue <bool>("Enabled"))
{
services.AddHostedService <BgpAquisitionService>();
}
var maintenanceSection = this.Configuration.GetSection(MaintenanceService.MaintenanceServiceSectionKey);
if ((maintenanceSection == null) || maintenanceSection.GetValue <bool>("Enabled"))
{
services.AddHostedService <MaintenanceService>();
}
services.AddTransient <VwRestRssiController>();
services.AddTransient <RestController>();
services.AddTransient <LinkTestController>();
services.AddTransient <StatusController>();
services.AddTransient <CacheInfoApiController>();
services.AddTransient <BgpController>();
services.AddTransient <ToolController>();
var hamnetDbAccess = HamnetDbProvider.Instance.GetHamnetDbFromConfiguration(this.Configuration.GetSection(HamnetDbProvider.HamnetDbSectionName));
services.AddSingleton(hamnetDbAccess);
IFailureRetryFilteringDataHandler retryFeasibleHandler = new FailureRetryFilteringDataHandler(this.Configuration);
services.AddSingleton <IFailureRetryFilteringDataHandler>(retryFeasibleHandler);
QueryResultDatabaseProvider.Instance.SetConfiguration(this.Configuration);
CacheMaintenance.SetDatabaseConfiguration(this.Configuration);
var databaseType = this.Configuration.GetSection(QueryResultDatabaseProvider.ResultDatabaseSectionName).GetValue <string>(QueryResultDatabaseProvider.DatabaseTypeKey)?.ToUpperInvariant();
switch (databaseType)
{
case "SQLITE":
services.AddDbContext <QueryResultDatabaseContext>(opt => opt.UseSqlite(this.Configuration.GetSection(QueryResultDatabaseProvider.ResultDatabaseSectionName).GetValue <string>(QueryResultDatabaseProvider.ConnectionStringKey)));
break;
case "MYSQL":
services.AddDbContext <QueryResultDatabaseContext>(opt => opt.UseMySql(this.Configuration.GetSection(QueryResultDatabaseProvider.ResultDatabaseSectionName).GetValue <string>(QueryResultDatabaseProvider.ConnectionStringKey)));
break;
default:
throw new ArgumentOutOfRangeException($"The configured database type '{databaseType}' is not supported for the query result database");
}
}
/// <summary>
/// Removes results (in result database) for which we didn't see an update for a configured amount of time.
/// </summary>
private void RemoveOutdatedResults(IConfigurationSection configuration)
{
TimeSpan resultsOutdatedAfter = configuration.GetValue <TimeSpan>("ResultsOutdatedAfter");
DateTime nowItIs = DateTime.UtcNow;
double currentUnixTimeStamp = (nowItIs - Program.UnixTimeStampBase).TotalSeconds;
using (var transaction = this.resultDatabaseContext.Database.BeginTransaction())
{
var outdatedRssis = this.resultDatabaseContext.RssiValues.Where(r => IsOutdatedUnixTimeStampColumn(currentUnixTimeStamp, r, resultsOutdatedAfter)).ToList();
foreach (var item in outdatedRssis)
{
this.logger.LogInformation($"Maintenance{(this.dryRunMode ? " DRY RUN: Would remove" : ": Removing")} RSSI entry for host {item.ForeignId} which hast last been updated at {item.TimeStampString} (i.e. {TimeSpan.FromSeconds(currentUnixTimeStamp - item.UnixTimeStamp)} ago)");
}
var outdatedRssiFailures = this.resultDatabaseContext.RssiFailingQueries.Where(r => IsOutdatedTimeStampColumn(nowItIs, r, resultsOutdatedAfter)).ToList();
foreach (var item in outdatedRssiFailures)
{
this.logger.LogInformation($"Maintenance{(this.dryRunMode ? " DRY RUN: Would remove" : ": Removing")} RSSI failing query entry for host {item.Subnet} which hast last been updated at {item.TimeStamp} (i.e. {item.TimeStamp - nowItIs} ago)");
}
var outdatedBgpPeers = this.resultDatabaseContext.BgpPeers.Where(r => IsOutdatedUnixTimeStampColumn(currentUnixTimeStamp, r, resultsOutdatedAfter)).ToList();
foreach (var item in outdatedBgpPeers)
{
this.logger.LogInformation($"Maintenance{(this.dryRunMode ? " DRY RUN: Would remove" : ": Removing")} BGP peer entry from host {item.LocalAddress} to {item.RemoteAddress} which hast last been updated at {item.TimeStampString} (i.e. {TimeSpan.FromSeconds(currentUnixTimeStamp - item.UnixTimeStamp)} ago)");
}
var outdatedBgpPeerFailures = this.resultDatabaseContext.BgpFailingQueries.Where(r => IsOutdatedTimeStampColumn(nowItIs, r, resultsOutdatedAfter)).ToList();
foreach (var item in outdatedBgpPeerFailures)
{
this.logger.LogInformation($"Maintenance{(this.dryRunMode ? " DRY RUN: Would remove" : ": Removing")} BGP failing peer entry from host {item.Host} which hast last been updated at {item.TimeStamp} (i.e. {item.TimeStamp - nowItIs} ago)");
}
var cacheMaintenance = new CacheMaintenance(this.dryRunMode);
cacheMaintenance.DeleteForAddress(outdatedRssis.Select(e => IPAddress.Parse(e.ForeignId)));
cacheMaintenance.DeleteForAddress(outdatedRssiFailures.SelectMany(e => e.AffectedHosts.Select(h => IPAddress.Parse(h))));
cacheMaintenance.DeleteForAddress(outdatedBgpPeers.Select(e => IPAddress.Parse(e.LocalAddress)));
cacheMaintenance.DeleteForAddress(outdatedBgpPeerFailures.Select(e => IPAddress.Parse(e.Host)));
if (!this.dryRunMode)
{
this.resultDatabaseContext.RemoveRange(outdatedRssis);
this.resultDatabaseContext.RemoveRange(outdatedRssiFailures);
this.resultDatabaseContext.RemoveRange(outdatedBgpPeers);
this.resultDatabaseContext.RemoveRange(outdatedBgpPeerFailures);
this.resultDatabaseContext.SaveChanges();
transaction.Commit();
}
else
{
transaction.Rollback();
}
}
}
/// <summary>
/// Fetches the cache data and converts to an array.
/// </summary>
/// <returns>The result list.</returns>
private ActionResult <IStatusReply> FetchCacheEntries(DeviceSupportedFeatures features)
{
try
{
var cacheMaintenance = new CacheMaintenance(true);
return(new HostsSupportingFeatureResult(cacheMaintenance.FetchEntryList().Where(e => ((e.SystemData?.SupportedFeatures ?? DeviceSupportedFeatures.None) & features) == features).Select(cacheEntry => new HostInfoReply(cacheEntry.Address, cacheEntry.SystemData, cacheEntry.ApiUsed, cacheEntry.LastModification))));
}
catch (Exception ex)
{
return(this.BadRequest($"Error: {ex.Message}"));
}
}
/// <summary>
/// Removes the cache entries for failures recorded in the result database
/// </summary>
/// <param name="cacheMaintenance">The cache maintenance object that supports deletion of entries.</param>
private void RemoveCacheEntriesForFailures(CacheMaintenance cacheMaintenance)
{
var failures = this.resultDatabaseContext.RssiFailingQueries;
var affectedHosts = failures.Select(q => q.AffectedHosts);
List <IPAddress> toDelete = new List <IPAddress>();
foreach (IReadOnlyCollection <string> item in affectedHosts)
{
toDelete.AddRange(item.Select(ah => IPAddress.Parse(ah)));
}
cacheMaintenance.DeleteForAddress(toDelete.Distinct());
}
/// <summary>
/// Runs all scheduled maintenance tasks.
/// </summary>
private void RunMaintenance()
{
var configurationSection = this.configuration.GetSection(MaintenanceServiceSectionKey);
this.NewDatabaseContext();
using (var transaction = this.resultDatabaseContext.Database.BeginTransaction())
{
var status = resultDatabaseContext.Status;
var nowItIs = DateTime.UtcNow;
var sinceLastScan = nowItIs - status.LastMaintenanceStart;
if ((sinceLastScan < this.maintenanceInterval - Hysteresis) && (status.LastMaintenanceStart <= status.LastMaintenanceEnd))
{
this.logger.LogInformation($"SKIPPING: Maintenance not yet due: Last aquisition started {status.LastMaintenanceStart} ({sinceLastScan} ago, hysteresis {Hysteresis}), configured interval {this.maintenanceInterval}");
return;
}
this.logger.LogInformation($"STARTING: Data maintenance - last run: Started {status.LastMaintenanceStart} ({sinceLastScan} ago)");
status.LastMaintenanceStart = DateTime.UtcNow;
resultDatabaseContext.SaveChanges();
transaction.Commit();
}
Program.RequestStatistics.MaintenanceRuns++;
this.RemoveOutdatedResults(configurationSection);
var cacheMaintenance = new CacheMaintenance(this.dryRunMode);
cacheMaintenance.RemoveFromCacheIfModificationOlderThan(configurationSection.GetValue <TimeSpan>("CacheInvalidAfter"));
this.RemoveCacheEntriesForFailures(cacheMaintenance);
using (var transaction = this.resultDatabaseContext.Database.BeginTransaction())
{
var status = resultDatabaseContext.Status;
status.LastMaintenanceEnd = DateTime.UtcNow;
this.logger.LogInformation($"COMPLETED: Database maintenance at {status.LastMaintenanceEnd}, duration {status.LastMaintenanceEnd - status.LastMaintenanceStart}");
resultDatabaseContext.SaveChanges();
transaction.Commit();
}
this.DisposeDatabaseContext();
}
/// <summary>
/// Fetches the cache data and converts to an array.
/// </summary>
/// <returns>The result list.</returns>
private ActionResult <IEnumerable <ICacheData> > FetchCacheEntries()
{
var cacheMaintenance = new CacheMaintenance(true);
return(cacheMaintenance.FetchEntryList().ToArray());
}
/// <summary>
/// Collects and returns the version information.
/// </summary>
/// <returns>The collected version information.</returns>
private ActionResult <IServerStatusReply> GetVersionInformation()
{
var myProcess = Process.GetCurrentProcess();
var reply = new ServerStatusReply
{
MaximumSupportedApiVersion = 1, // change this when creating new API version
ServerVersion = Program.LibraryInformationalVersion,
ProcessUptime = DateTime.UtcNow - myProcess.StartTime,
ProcessCpuTime = myProcess.TotalProcessorTime,
ProcessWorkingSet = myProcess.WorkingSet64,
ProcessPrivateSet = myProcess.PrivateMemorySize64,
ProcessThreads = myProcess.Threads.Count,
ProcessStartTime = myProcess.StartTime,
};
reply.Add("WebRequests", new Statistic(StatsProperties.Select(sp => new KeyValuePair <string, string>(sp.Name, sp.GetValue(Program.RequestStatistics)?.ToString()))));
this.AddConfiguration(reply, Program.RssiAquisitionServiceSectionKey);
this.AddConfiguration(reply, MaintenanceService.MaintenanceServiceSectionKey);
this.AddConfiguration(reply, Program.InfluxSectionKey);
this.AddConfiguration(reply, QueryResultDatabaseProvider.ResultDatabaseSectionName);
this.AddConfiguration(reply, HamnetDbProvider.HamnetDbSectionName);
this.AddConfiguration(reply, "CacheDatabase");
this.AddConfiguration(reply, "DeviceDatabase");
this.AddConfiguration(reply, Program.MonitoringAccountsSectionKey, Program.BgpAccountSectionKey);
this.AddConfiguration(reply, Program.PenaltySystemSectionKey);
var statusTableRow = this.dbContext.MonitoringStatus.First();
var rssiQueryResultStats = new Statistic()
{
{ "UniqueValues", this.dbContext.RssiValues.Count().ToString() },
{ "TotalFailures", this.dbContext.RssiFailingQueries.Count().ToString() },
{ "TimeoutFailures", this.dbContext.RssiFailingQueries.Where(q => q.ErrorInfo.Contains("Timeout") || q.ErrorInfo.Contains("Request has reached maximum retries")).Count().ToString() },
{ "NonTimeoutFailures", this.dbContext.RssiFailingQueries.Where(q => !q.ErrorInfo.Contains("Timeout") && !q.ErrorInfo.Contains("Request has reached maximum retries")).Count().ToString() },
{ "LastAquisitionStart", statusTableRow.LastRssiQueryStart.ToString("yyyy-MM-ddTHH\\:mm\\:sszzz") },
{ "LastAquisitionEnd", statusTableRow.LastRssiQueryEnd.ToString("yyyy-MM-ddTHH\\:mm\\:sszzz") },
};
reply.Add("RssiResultDatabase", rssiQueryResultStats);
var bgpQueryResultStats = new Statistic()
{
{ "UniqueValues", this.dbContext.BgpPeers.Count().ToString() },
{ "TotalFailures", this.dbContext.BgpFailingQueries.Count().ToString() },
{ "TimeoutFailures", this.dbContext.BgpFailingQueries.Where(q => q.ErrorInfo.Contains("Timeout") || q.ErrorInfo.Contains("timed out")).Count().ToString() },
{ "NonTimeoutFailures", this.dbContext.BgpFailingQueries.Where(q => !q.ErrorInfo.Contains("Timeout") && !q.ErrorInfo.Contains("timed out")).Count().ToString() },
{ "LastAquisitionStart", statusTableRow.LastBgpQueryStart.ToString("yyyy-MM-ddTHH\\:mm\\:sszzz") },
{ "LastAquisitionEnd", statusTableRow.LastBgpQueryEnd.ToString("yyyy-MM-ddTHH\\:mm\\:sszzz") },
};
reply.Add("BgpResultDatabase", bgpQueryResultStats);
var maintenanceResultStats = new Statistic()
{
{ "LastMaintenanceStart", statusTableRow.LastMaintenanceStart.ToString("yyyy-MM-ddTHH\\:mm\\:sszzz") },
{ "LastMaintenanceEnd", statusTableRow.LastMaintenanceEnd.ToString("yyyy-MM-ddTHH\\:mm\\:sszzz") },
};
reply.Add("MaintenanceDatabase", maintenanceResultStats);
var cacheMaintenance = new CacheMaintenance(true /* we don't want to modify anything - so set dry-run to be sure */);
reply.Add("CacheDatabase", new Statistic(cacheMaintenance.CacheStatistics()));
var devDbMaintenance = new DeviceDatabaseMaintenance(true /* we don't want to modify anything - so set dry-run to be sure */);
reply.Add("DeviceDatabase", new Statistic(devDbMaintenance.CacheStatistics()));
return(reply);
}
