public IDatabaseCollector Create(EDatabaseType type) { IDatabaseCollector collector = null; try { switch (type) { case EDatabaseType.Oracle: collector = new OracleCollector(); break; case EDatabaseType.SqlServer: collector = new SqlServerCollector(); break; case EDatabaseType.Postgres: collector = new PostgresCollector(); break; case EDatabaseType.Unknown: default: throw new Exception($"Unknown database type {type}"); } } catch (Exception ex) { ApplicationEventLog log = new ApplicationEventLog(); log.LogError("Error in DatabaseCollectorFactory connection"); log.Log(ex); } return(collector); }
public void Interpret(Data d, SQLiteConnection conn) { if (d.Type != ECollectorType.Disk) { return; } if (d is GenericDictionaryData <DiskUsage> ) { Dictionary <string, DiskUsage> value = ((GenericDictionaryData <DiskUsage>)d).Data; // Don't alert on drives that aren't being monitored MonitoredDrivesRequest request = new MonitoredDrivesRequest(d.Name); RequestBus.Instance.MakeRequest(request); DiskSpaceLowAlert low_level = new DiskSpaceLowAlert(); DiskSpaceCriticallyLowAlert critically_low_level = new DiskSpaceCriticallyLowAlert(); double low = low_level.GetValueAsDouble(conn) ?? 80.0; double critically_low = critically_low_level.GetValueAsDouble(conn) ?? 90.0; EStatusType status = EStatusType.AdequateDiskSpace; List <string> messages = new List <string>(); DiskSpaceStatus ds_status = new DiskSpaceStatus(low, critically_low); foreach (string drive in value.Keys) { if (request.IsHandled && request.DriveManager.IsDriveMonitored(drive) == false) { continue; } DiskUsage data = value[drive]; Tuple <EStatusType, double> drive_status = ds_status.GetStatus(data.UsedNum, data.CapacityNum); status = status.DiskSpaceCompare(drive_status.Item1); if (drive_status.Item1 == EStatusType.CriticallyLowOnDiskSpace || drive_status.Item1 == EStatusType.LowOnDiskSpace) { messages.Add($"{drive} -- {drive_status.Item2:0.0} %"); } } string message = messages.JoinStrings(", "); long device_id = GetDeviceID(d, conn); if (device_id >= 0) { SetDeviceStatus(device_id, status, DiskSpaceStatus.Types, message, conn); } else { ApplicationEventLog log = new ApplicationEventLog(); log.LogError($"DiskSpaceInterpreter: unable to get device id for {d.Context.Name}"); } } else { throw new Exception("DiskSpaceInterpreter: data type is wrong"); } }
public void Interpret(Data d, SQLiteConnection conn) { if (d.Type != ECollectorType.Ping) { return; } if (d is ListData <PingResult> ) { ListData <PingResult> data = d as ListData <PingResult>; Dictionary <string, long> ip_to_device_id_map = new Dictionary <string, long>(); string sql = "SELECT IPAddress, DeviceID FROM Devices WHERE DateDisabled IS NULL;"; using (SQLiteCommand command = new SQLiteCommand(sql, conn)) using (SQLiteDataReader reader = command.ExecuteReader()) { while (reader.Read()) { if (reader.IsDBNull(0) == false) { ip_to_device_id_map[reader.GetString(0)] = reader.GetInt64(1); } } } foreach (PingResult result in data.Data) { string ip = result.Address.ToString(); if (ip_to_device_id_map.TryGetValue(ip, out long device_id)) { Tuple <EStatusType, EStatusType?> ping_status = PingStatus.GetStatus(result.IsPingable, result.AvgTime); SetDeviceStatus(device_id, ping_status.Item1, PingStatus.OnlineOrOffline, string.Empty, conn); // If the device isn't pingable, make sure we clear the good-or-slow statuses as well if (result.IsPingable == false) { ClearDeviceStatus(device_id, PingStatus.GoodOrSlow, conn); } if (ping_status.Item2.HasValue) { SetDeviceStatus(device_id, ping_status.Item2.Value, PingStatus.GoodOrSlow, string.Empty, conn); } } else { ApplicationEventLog log = new ApplicationEventLog(); log.LogError($"OfflineInterpreter: unable to find device_id from {ip}"); } } } else { string json = JsonConvert.SerializeObject(d); throw new Exception($"OfflineInterpreter: data type is wrong {json}"); } }
public void UpdateNextCollectionTime(long collector_id) { //logging.EventLog log = new ApplicationEventLog(); try { string sql = $"SELECT FrequencyInMinutes, NextCollectionTime FROM Collectors WHERE CollectorID = {collector_id};"; DateTimeOffset next = DateTimeOffset.Now; TimeSpan? span = null; using (SQLiteCommand command = new SQLiteCommand(sql, Conn)) using (SQLiteDataReader reader = command.ExecuteReader()) { if (reader.Read()) { int frequency = reader.GetInt32(0); if (frequency > 0) { span = TimeSpan.FromMinutes(frequency); } if (reader.IsDBNull(1) == false) { next = DateTimeOffset.Parse(reader.GetString(1)); } } } if (span.HasValue) { // It's possible that the next collection time is way in the past, such as would be the case // when we start up after having been down for a long time. We don't just want to increment // the next collection time and leave it still in the past, so loop here until the next // time is in the future. while (next <= DateTimeOffset.Now) { next += span.Value; } Updater updater = new Updater("Collectors", $"CollectorID = {collector_id}", Conn); updater.Set("NextCollectionTime", next); //log.LogInformation("UpdateNextCollectionTime: " + updater.Statement); updater.Execute(); } } catch (Exception e) { logging.EventLog log = new ApplicationEventLog(); log.LogError($"Error in UpdateNextCollectionTime({collector_id})"); log.Log(e); } }
public void UpdateCollectionAttemptTime(long collector_id, DateTimeOffset time) { try { Updater updater = new Updater("Collectors", $"CollectorID = {collector_id}", Conn); updater.Set("LastCollectionAttempt", time); updater.Execute(); } catch (Exception e) { logging.EventLog log = new ApplicationEventLog(); log.LogError($"Error in UpdateCollectionAttemptTime({collector_id}, {time})"); log.Log(e); } }
public long Insert(DataRecord dr, SQLiteConnection conn) { long data_id = dr.InsertData(conn); if (dr.D != null) { m_interpreters.ForEach(i => { try { i.Interpret(dr.D, conn); } catch (Exception e) { ApplicationEventLog log = new ApplicationEventLog(); log.LogError($"DataStorage.Insert: interpretation error ({dr.D.Name})"); log.Log(e); } }); } return(data_id); }
public bool Insert(IEnumerable <DataRecord> records, Database db) { foreach (DataRecord dr in records) { try { using (SQLiteConnection conn = db.Connection) { conn.Open(); Insert(dr, conn); } } catch (Exception e) { ApplicationEventLog log = new ApplicationEventLog(); log.LogError("DataStorage.Insert error"); log.Log(e); } } return(true); }
public void DoWrite(SQLiteConnection conn) { if (GlobalIsRunning.IsRunning == false) { return; } if (IsTimeToWrite == false) { return; } m_timer.Reset(); logging.EventLog log = new ApplicationEventLog(); Stopwatch watch = Stopwatch.StartNew(); List <DailyReport> reports = new Retriever().GetIncompleteDailyReports(true, conn); string directory = Filename.Directory; log.LogInformation($"Writing {reports.Count} daily file(s)"); JsonSerializer serializer = new JsonSerializer(); foreach (DailyReport report in reports) { Stopwatch watch2 = Stopwatch.StartNew(); Filename f = new Filename(report.countryCode, report.siteName, report.day); string filename_base = directory + "\\" + f.Name; string filename_json = filename_base + ".json"; using (StreamWriter sw = new StreamWriter(filename_json)) using (JsonWriter writer = new JsonTextWriter(sw)) { writer.Formatting = Formatting.Indented; serializer.Serialize(writer, report); } CompressDailyFile compress = new CompressDailyFile(); bool?do_compress = compress.GetValueAsBoolean(conn); if (do_compress.HasValue && do_compress.Value) { try { using (ZipFile zip = new ZipFile()) { ZipEntry entry = zip.AddFile(filename_json); entry.FileName = f.Name + ".json"; // Note that we save the filename as zip.json, and not the more // likely json.zip. // // This is because the daily-file-upload-tool just uploads files from // COMMON or whoever else might be generating daily files, and the back-end // processing of those uploaded files uses the extension to do additional processing. // We don't want special processing for compressed JSON files and uncompressed JSON files, // so we use an unconventional naming convention. zip.Save(filename_base + ".zip.json"); } // Optionally delete the original JSON file once it's compressed DeleteDailyFileAfterCompression del = new DeleteDailyFileAfterCompression(); bool?do_delete = del.GetValueAsBoolean(conn); if (do_delete.HasValue && do_delete.Value) { File.Delete(filename_json); } } catch (Exception e) { log.LogError(e.Message); } } log.LogInformation($"Writing {filename_json} took {watch2.ElapsedMilliseconds} ms"); } watch.Stop(); log.LogInformation($"Done writing {reports.Count} daily file(s). It took {watch.ElapsedMilliseconds} ms"); }
protected void ThreadFunc() { try { int max_collections_per_pass = 10; logging.EventLog elog = new ApplicationEventLog(); Database db = new Database(); DataStorage storage = new DataStorage(); m_interpreters.ForEach(i => storage.AddInterpreter(i)); while (GlobalIsRunning.IsRunning) { int collector_count = 0; using (SQLiteConnection conn = db.Connection) { conn.Open(); CheckForConfigurationChanges(storage, conn); // Used to hold which collector was doing its thing if/when an exception occurs. // It's used in the exception handler. string collector_name = string.Empty; try { DBCollectionTimeRetriever retriever = new DBCollectionTimeRetriever(conn); // Gets the list of things that need to be collected right now. They'll // be in the order they should be collected. List <DataCollector> collectors = m_system_device.GetCollectors(retriever); collector_count = collectors.Count; // Limit this to the top 10 or so while (collectors.Count > max_collections_per_pass) { collectors.RemoveAt(collectors.Count - 1); } foreach (DataCollector collector in collectors) { collector_name = collector.Context.Name; //elog.LogInformation($"Collecting {collector_name}"); Stopwatch watch = Stopwatch.StartNew(); // Records that the collector is being collected, updates the next collection // time, and records when the collection attempt was started. When it's destroyed // when exiting the using, it records that it is no longer being collected. // // This was done so even if an exception occurs within Acquire(), the flag // that the collector is being collected will be cleared. using (BeingCollected bc = new BeingCollected(collector.Context.ID.ID, conn)) { collector.Acquire(); } //long elapsed_ms = watch.ElapsedMilliseconds; //if(elapsed_ms > 500) // elog.LogInformation($"Collecting {collector_name} took {elapsed_ms} ms"); if (GlobalIsRunning.IsRunning == false) { break; // out of the foreach loop } } } catch (Exception e) { elog.LogError($"Exception from within {collector_name}"); elog.Log(e); } // Will write the daily file when it's the right time to do so; otherwise, it does nothing. if (GlobalIsRunning.IsRunning) { m_daily_file_writer.DoWrite(conn); } if (GlobalIsRunning.IsRunning) { m_db_cleaner.CleanOldData(m_days_to_keep, conn); } } // Deletes any old daily files if (GlobalIsRunning.IsRunning) { m_daily_file_cleaner.DoClean(); } // Delete any old log files too, if it's time to do so. if (GlobalIsRunning.IsRunning) { LogManager.CleanOldData(); } // And make sure we update our logging if it changed if (GlobalIsRunning.IsRunning) { LogManager.CheckConfiguration(); } // m_shutdown will be reset when the thread starts, and set when it's time to // stop the thread. So this will wait if this event hasn't been // set, but will return immediately if it has been set. // // If there's still more data to collect let's make another run right away if (GlobalIsRunning.IsRunning && collector_count < max_collections_per_pass) { m_shutdown.WaitOne(TimeSpan.FromSeconds(10)); } } m_interpreters.ForEach(i => storage.RemoveInterpreter(i)); } catch (Exception e) { logging.EventLog elog = new ApplicationEventLog(); elog.Log(e); m_shutdown.Reset(); m_thread.Abort(); m_thread = null; } }
public void Interpret(Data d, SQLiteConnection conn) { if (d.Type != ECollectorType.Ping) { return; } if (d is ListData <PingResult> ) { ListData <PingResult> data = d as ListData <PingResult>; Dictionary <string, string> ip_to_name_map = new Dictionary <string, string>(); string sql = "SELECT IPAddress, Name FROM Devices WHERE DateDisabled IS NULL;"; using (SQLiteCommand command = new SQLiteCommand(sql, conn)) using (SQLiteDataReader reader = command.ExecuteReader()) { while (reader.Read()) { if (reader.IsDBNull(0) == false) { ip_to_name_map[reader.GetString(0)] = reader.GetString(1); } } } foreach (PingResult result in data.Data) { try { string ip = result.Address.ToString(); string name = ip; if (ip_to_name_map.ContainsKey(ip)) { name = ip_to_name_map[ip]; } sql = $"SELECT IPAddress FROM NetworkStatus WHERE IPAddress = '{ip}';"; Changer changer = null; bool existing = false; using (SQLiteCommand command = new SQLiteCommand(sql, conn)) using (SQLiteDataReader reader = command.ExecuteReader()) { if (reader.Read()) { // It already exists--update the status and name in case it's been changed existing = true; changer = new Updater("NetworkStatus", $"IPAddress = '{ip}'", conn); } else { changer = new Inserter("NetworkStatus", conn); changer.Set("IPAddress", ip, false); } if (changer != null) { changer.Set("Name", name, false); changer.Set("SuccessfulPing", result.IsPingable ? 1 : 0); changer.Set("DatePingAttempted", data.CollectedAt); if (result.IsPingable) { changer.Set("DateSuccessfulPingOccurred", data.CollectedAt); } else if (!existing) { // It's new, and wasn't pingable, so we need to report that. We'll do that by // having an empty date/time changer.Set("DateSuccessfulPingOccurred", "", false, false); } // else it exists, but isn't pingable, so leave the DateSuccessfulPingOccurred alone } } if (changer != null) { changer.Execute(); } } catch (Exception e) { ApplicationEventLog log = new ApplicationEventLog(); log.LogError($"PingInterpreter -- {result.Address.ToString()}"); log.Log(e); } } } else { throw new Exception("PingInterpreter: data type is wrong"); } }
/// <summary> /// Make the return list a priority list. The IDs in the list will be returned /// sorted by the time they should be collected. /// </summary> /// <returns></returns> public List <long> GetCollectorIDs() { Stopwatch watch = Stopwatch.StartNew(); List <long> devices_to_collect = new List <long>(); List <Tuple <long, DateTimeOffset> > to_collect = new List <Tuple <long, DateTimeOffset> >(); string sql = "SELECT C.CollectorID, C.NextCollectionTime, C.CollectorType FROM Collectors C INNER JOIN Devices D ON C.DeviceID = D.DeviceID WHERE D.DateDisabled IS NULL AND C.IsEnabled = 1;"; //ILog log = LogManager.GetLogger(typeof(Database)); //logging.EventLog log = new ApplicationEventLog(); try { using (SQLiteCommand command = new SQLiteCommand(sql, Conn)) using (SQLiteDataReader reader = command.ExecuteReader()) { DateTimeOffset now = DateTimeOffset.Now; while (reader.Read()) { try { long id = reader.GetInt64(0); int type = reader.GetInt32(2); if (type == (int)ECollectorType.Configuration) { continue; } // If it's never been collected, collect. No need to // record the collection time--just add it straight to devices_to_collect if (reader.IsDBNull(1)) { devices_to_collect.Add(id); } else { DateTimeOffset next_collection_time = DateTimeOffset.Parse(reader.GetString(1)); if (next_collection_time < now) { to_collect.Add(Tuple.Create(id, next_collection_time)); } } } catch (Exception ex) { logging.EventLog log = new ApplicationEventLog(); log.LogError("GetDevicesToCollect"); log.Log(ex); } } } } catch (Exception ex) { logging.EventLog log = new ApplicationEventLog(); log.LogError("GetDevicesToCollect: " + sql); log.Log(ex); } to_collect.Sort((a, b) => a.Item2.CompareTo(b.Item2)); to_collect.ForEach(c => devices_to_collect.Add(c.Item1)); //log.LogInformation($"GetDevicesToCollect took {watch.ElapsedMilliseconds} ms for {devices_to_collect.Count} devices"); return(devices_to_collect); }