#pragma warning restore IDE0044 // Add readonly modifier #endregion #region Init and process Methods /// <summary> /// Init all class wide settings. /// </summary> /// <param name="dbContext">DawaReplicationDBContext instance.</param> /// <param name="tableList">A list of tables to process.</param> /// <param name="dawaApiUri">Typical 'https://dawa.aws.dk/' .</param> /// <param name="readTimeoutInSeconds">DAWA api timeout in seconds. Normally 300 seconds.</param> /// <param name="udtraekRowsMax">Max number of rows to read. When 0 all rows are read.</param> public static void InitSettings(DawaReplicationDBContext dbContext, List <string> tableList, string dawaApiUri, int readTimeoutInSeconds, int udtraekRowsMax, int udtraekBulkSize, int dkStedDataStartPos, int dkStedBulkSize, int dbCommandTimeoutInSeconds, string tempDataFolderPath, bool activeFixes, string activeFixesListFileLocation, bool useMSApplicationInsights, bool jsonSerializerIgnoreNullValue) { if (tableList == null) { throw new ArgumentNullException(nameof(tableList)); } if (tableList == null && tableList.Count == 0) { throw new ArgumentNullException(nameof(tableList)); } if (string.IsNullOrEmpty(dawaApiUri)) { throw new ArgumentNullException(nameof(dawaApiUri)); } if (readTimeoutInSeconds == 0) { throw new ArgumentOutOfRangeException(nameof(readTimeoutInSeconds)); } if (udtraekBulkSize == 0) { throw new ArgumentOutOfRangeException(nameof(udtraekBulkSize)); } if (tempDataFolderPath == null) { throw new ArgumentNullException(nameof(tempDataFolderPath)); } DBContext = dbContext ?? throw new ArgumentNullException(nameof(dbContext)); TableList = tableList; DawaApiUri = dawaApiUri.EndsWith("/") ? dawaApiUri : dawaApiUri + "/"; ReadTimeoutInSeconds = readTimeoutInSeconds; UdtraekRowsMax = udtraekRowsMax; UdtraekBulkSize = udtraekBulkSize; DKStedDataStartPos = dkStedDataStartPos; DKStedBulkSize = dkStedBulkSize; dbContext.Database.SetCommandTimeout(dbCommandTimeoutInSeconds); TempDataFolderPath = tempDataFolderPath.EndsWith(@"\") ? tempDataFolderPath : tempDataFolderPath + @"\"; ActiveFixes = activeFixes; ActiveFixesListFileLocation = activeFixesListFileLocation; UseMSApplicationInsights = useMSApplicationInsights; UserJsonSerializerSettings = new JsonSerializerSettings { }; JsonSerializerIgnoreNullValue = jsonSerializerIgnoreNullValue; if (jsonSerializerIgnoreNullValue) { UserJsonSerializerSettings = new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }; } if (ActiveFixes) { fixInfoList = FixInfo.FixInfoList(ActiveFixesListFileLocation); } }
public static void UpdateDatabase() { var logger = log4net.LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType); using (var dBContext = new DawaReplicationDBContext()) { if (dBContext.Database.CanConnect()) { try { dBContext.Database.Migrate(); Console.WriteLine(DbUpdateComplete); } catch (Exception e) { logger.Error("ERROR!: Could not apply the latest migration.", e); Console.WriteLine("ERROR! Could not apply the latest migration. See error message below."); Console.WriteLine(e); } } else { logger.Error("ERROR!: No migration applied, failed to connect to database."); Console.WriteLine("Error: Failed to connect to the database. Check your connection string."); } } }
/// <summary>Check if the program can establish a connection to the database.</summary> public static void HaltIfNoDBConnection() { try { using (var context = new DawaReplicationDBContext()) { // context.Entitystate.Max(t => t.Txid).GetValueOrDefault(0); if (!context.Database.CanConnect()) { var conn = ConfigurationManager.ConnectionStrings["DawaDatabase"].ConnectionString; var msg = $"ERROR! Failed to connect to database. Check ConnectionString '{conn}' in config file and database user role."; Console.WriteLine(msg); _logger.Error($"Main(): {msg}"); Environment.Exit((int)ReturnCode.DBConnectionError); } } } catch (Exception ex) { var conn = ConfigurationManager.ConnectionStrings["DawaDatabase"].ConnectionString; var msg = $"ERROR! Failed to connect to database. Check ConnectionString '{conn}' in config file and database user role. Problem: {ex.Message}"; Console.WriteLine(msg); _logger.Error($"Main(): {msg}", ex); Environment.Exit((int)ReturnCode.DBConnectionError); } }
/// <summary> /// Create row in EntityStateHistory table for current transaction. /// </summary> /// <param name="dbContext">Database context.</param> /// <param name="mode">EntityProcessMode mode. Udtraek/Update/DAGI.</param> /// <param name="entity">Entity name.</param> /// <param name="txidTil">Transaction ID for updating the table up to.</param> /// <returns>DateTime to synchronize between EntityState tables.</returns> public static DateTime SetEntityStateHistoryStart(DawaReplicationDBContext dbContext, EntityProcessMode mode, string entity, long?txidTil) { if (dbContext == null) { throw new ArgumentNullException(nameof(dbContext)); } if (entity == null) { throw new ArgumentNullException(nameof(entity)); } entity = entity.ToLower(); EntitystateHistory row = new EntitystateHistory { Entity = entity, }; row.TxidFra = (mode == EntityProcessMode.Udtraek || mode == EntityProcessMode.Dagi) ? -1 : dbContext.EntitystateHistory.Where(c => c.Entity == entity && c.Success == true).OrderByDescending(c => c.Starttime).First().TxidTil; row.TxidTil = mode == EntityProcessMode.Dagi ? null : txidTil; row.Success = null; row.Starttime = DateTime.Now; row.Finishtime = null; row.InsertUpdateCount = null; row.DeleteCount = null; row.Message = null; dbContext.Add(row); dbContext.SaveChanges(); return(row.Starttime); }
/// <summary> /// Set EntityState row for entity to complete. /// </summary> /// <param name="dbContext">Database context.</param> /// <param name="entity">Entity name.</param> /// <param name="success">Bool success.</param> /// <param name="txid">Transaction ID.</param> /// <param name="count">Items processed.</param> /// <param name="message">Error message if success = 0.</param> /// <returns>DateTime finishtime to synchronize EntityState rows with EntityStateHistory´.</returns> public static DateTime SetEntityStateDone(DawaReplicationDBContext dbContext, string entity, bool success, long?txid, int?count, string message = null) { if (dbContext == null) { throw new ArgumentNullException(nameof(dbContext)); } if (entity == null) { throw new ArgumentNullException(nameof(entity)); } var finishtime = DateTime.Now; var row = dbContext.Entitystate.Single(c => c.Entity == entity.ToLower()); row.Txid = success ? txid : row.Txid; row.Success = success; row.Successtime = success ? finishtime : row.Successtime; row.SuccesstimeChange = success && count.GetValueOrDefault() > 0 ? finishtime : row.SuccesstimeChange; row.Finishtime = finishtime; row.Count = count; row.Message = message; row.MonitoringIgnoreError = row.MonitoringIgnoreError == false || row.MonitoringIgnoreError == null ? false : true; dbContext.Update(row); dbContext.SaveChanges(); return(finishtime); }
/// <summary> /// Runs a cleanup job in the EntityStateHistory table, deleting old rows. /// </summary> /// <param name="days">Number of days after which rows should be deleted.</param> /// <returns>int number of deleted records.</returns> public static int CleanupOldEntityStateRecords(int days) { var retval = 0; using (var context = new DawaReplicationDBContext()) { var oldRecords = context.EntitystateHistory.Where(c => (DateTime.Now - c.Starttime).TotalDays > days); retval = oldRecords.Count(); context.EntitystateHistory.RemoveRange(oldRecords); context.SaveChanges(); } return(retval); }
/// <summary> /// GetTxid for last for Entity to create a new search query. /// </summary> /// <param name="dbContext">The database context.</param> /// <param name="entity">Name of the entity worked on.</param> /// <returns>int transaction ID.</returns> public static long?GetTxid(DawaReplicationDBContext dbContext, string entity) { if (dbContext == null) { throw new ArgumentNullException(nameof(dbContext)); } if (entity == null) { throw new ArgumentNullException(nameof(entity)); } return(dbContext.Entitystate.Single(c => c.Entity == entity.ToLower()).Txid); }
/// <summary>Check if the program can establish a connection to the database.</summary> public static void HaltIfDBSchemeNotInitialized() { try { using (var context = new DawaReplicationDBContext()) { context.Entitystate.Max(t => t.Txid).GetValueOrDefault(0); } } catch (Exception ex) { var conn = ConfigurationManager.ConnectionStrings["DawaDatabase"].ConnectionString; var msg = $"ERROR! The database scheme has not been initialized. Check ConnectionString '{conn}' in config file and make sure the database is initialized. Problem: {ex.Message}"; Console.WriteLine(msg); _logger.Error($"Main(): {msg}", ex); Environment.Exit((int)ReturnCode.DBConnectionError); } }
/// <summary> /// Updates an EntityStateHistory row after transaction is completed. /// </summary> /// <param name="dbContext">Database context.</param> /// <param name="entity">Entity name.</param> /// <param name="success">Bool success.</param> /// <param name="starttime">Start time.</param> /// <param name="finishtime">Time of completion.</param> /// <param name="txidFra">Updated entity from transaction Id txidFra.</param> /// <param name="insertUpdateCount">Inserted or updated row count.</param> /// <param name="deleteCount">Deleted row count.</param> /// <param name="message">Error message if success = 0.</param> public static void SetEntityStateHistoryDone(DawaReplicationDBContext dbContext, string entity, bool success, DateTime starttime, DateTime finishtime, long?txidFra, int?insertUpdateCount, int?deleteCount, string message = null) { if (dbContext == null) { throw new ArgumentNullException(nameof(dbContext)); } var row = dbContext.EntitystateHistory.Single(c => c.Entity == entity.ToLower() && c.Starttime == starttime); // var row = dbContext.EntitystateHistory.OrderByDescending(x => x.Starttime).Where(c => c.Entity == entity.ToLower()).First(); row.TxidFra = success == true ? txidFra : row.TxidFra; row.Success = success; row.Finishtime = finishtime; row.InsertUpdateCount = success == true ? insertUpdateCount : null; row.DeleteCount = success == true ? deleteCount : null; row.Message = message; dbContext.Update(row); dbContext.SaveChanges(); }
/// <summary> /// Create a new row in Entitystate for given entity. /// </summary> /// <param name="dbContext">Database context.</param> /// <param name="mode">EntityProcessMode. Udtraek/Update/Dagi.</param> /// <param name="entity">Entity name.</param> /// <param name="txid">Transaction ID.</param> public static void SetEntityStateStart(DawaReplicationDBContext dbContext, EntityProcessMode mode, string entity, long txid) { if (dbContext == null) { throw new ArgumentNullException(nameof(dbContext)); } if (entity == null) { throw new ArgumentNullException(nameof(entity)); } entity = entity.ToLower(); // Create entity if missing: if (!dbContext.Entitystate.Any(c => c.Entity == entity)) { var entitystate = new Entitystate { Entity = entity, }; dbContext.Add(entitystate); dbContext.SaveChanges(); } var row = dbContext.Entitystate.Single(c => c.Entity == entity); row.Txid = mode == EntityProcessMode.Update ? row.Txid : txid; row.Success = null; row.Successtime = null; row.SuccesstimeChange = row.SuccesstimeChange; row.Starttime = DateTime.Now; row.Finishtime = null; row.Count = null; row.Message = null; dbContext.Update(row); dbContext.SaveChanges(); }
public static void DeleteOldRows(DawaReplicationDBContext dbContext, string name, DateTime date) { if (dbContext == null) { throw new ArgumentNullException(nameof(dbContext)); } switch (name) { case "steder": dbContext.Dagi_Steder.RemoveRange(dbContext.Dagi_Steder.Where(c => c.EntityUpdated < date)); dbContext.SaveChanges(); break; case "stednavne": dbContext.Dagi_Stednavne.RemoveRange(dbContext.Dagi_Stednavne.Where(c => c.EntityUpdated < date)); dbContext.SaveChanges(); break; default: break; } }
/// <summary> /// Main program. /// </summary> /// <param name="args">String arguments for program initialization.</param> public static void Main(string[] args) { Thread.CurrentThread.CurrentCulture = System.Globalization.CultureInfo.InvariantCulture; var settings = Properties.Settings.Default; if (settings.UseMSApplicationInsights) { try { ApplicationInsightInitializer.Initialize(); } catch (Exception ex) { var msg = $"ERROR! Failed to initialize MS Application Insights. Problem: {ex.Message}"; Console.WriteLine(msg); _logger.Error(msg, ex); Environment.Exit((int)ReturnCode.ApplicationInsightInitializeError); } } Helpers.InitializeHelpers.HaltIfNoDBConnection(); Helpers.InitializeHelpers.HaltIfAllreadyRunning(); try { #pragma warning disable CS0436 // Type conflicts with imported type SqlServerTypes.Utilities.LoadNativeAssemblies(AppDomain.CurrentDomain.BaseDirectory); #pragma warning restore CS0436 // Type conflicts with imported type string programAssemblyVersion = Assembly.GetExecutingAssembly().GetName().Version.ToString(); string dataAccessAssemblyVersion = Assembly.GetAssembly(typeof(Adgangsadresse)).GetName().Version.ToString(); var methodName = LoggingUtils.GetMethodName(); Console.WriteLine($"Starting DawaReplication assembly ver. {programAssemblyVersion}; DataAccess ver. {dataAccessAssemblyVersion} ...."); _logger.Info($"{methodName}: Starting DawaReplication assembly ver. {programAssemblyVersion}; DataAccess ver. {dataAccessAssemblyVersion}"); _logger.Info($"{methodName}: Settings = {SettingsHelper.SettingsAsString()}"); _logger.Info($"{methodName}: Time of initialization: {DateTime.Now.ToShortTimeString()}"); var processMode = InitialArgumentHelper.InitializeProcess(args); Helpers.InitializeHelpers.HaltIfDBSchemeNotInitialized(); Console.WriteLine($"Time: {DateTime.Now.ToShortTimeString()}. ProcessMode: {processMode}\n"); var list = TableInfo.GetTableInfoList(AppDomain.CurrentDomain.BaseDirectory + settings.TableInfoFile, false); #pragma warning disable CA2000 // Dispose objects before losing scope var dbContext = new DawaReplicationDBContext(); #pragma warning restore CA2000 // Dispose objects before losing scope if (processMode == EntityProcessMode.Udtraek) { FixDBProblems.FixDbProblems(dbContext); } EntityManager.InitSettings(dbContext, list.Select(x => x.Name).ToList(), settings.DawaApiUri, settings.DawaApiReadTimeoutInSeconds, settings.UdtraekRowsMax, settings.UdtraekBulkSize, settings.DKStedDataStartPos, settings.DKStedBulkSize, settings.DBCommandTimeoutInSeconds, settings.TempDataFolderPath, settings.ActiveFixes, settings.ActiveFixesListFileLocation, settings.UseMSApplicationInsights, settings.JsonSerializerIgnoreNullValue); LocalDataHelper.MakeFolder(settings.TempDataFolderPath); var msg = LoggingUtils.MakeMessage(methodName, " TableInfoFile", list); _logger.Info(msg); var watch = Stopwatch.StartNew(); var dawaProcessInfo = Helpers.InitializeHelpers.MakeDawaProcessInfo(processMode, settings.DawaApiUri, settings.DawaApiReadTimeoutInSeconds, settings.TxidOverride); _logger.Info($"{methodName}: Got latest transaction ID = {dawaProcessInfo.Txid}"); EntityManager.ProcessTables(processMode, dawaProcessInfo); if (dawaProcessInfo.FailedTables.Any() && (processMode == EntityProcessMode.Dagi || processMode == EntityProcessMode.Udtraek)) { for (int i = 1; i <= settings.RetryCount; i++) { msg = $"{methodName}: Failed to process all tables. No of failed tables: {dawaProcessInfo.FailedTables.Count}. Retrying again in {settings.RetryTimerInMinutes * i} minutes."; Console.WriteLine($"\nWARNING. Time: {DateTime.Now.ToShortTimeString()}. {msg}"); _logger.Warn(msg); EntityManager.TableList = new List <string>(dawaProcessInfo.FailedTables); dawaProcessInfo.FailedTables.Clear(); Thread.Sleep(settings.RetryTimerInMinutes * 60 * 1000 * i); EntityManager.ProcessTables(processMode, dawaProcessInfo); if (dawaProcessInfo.FailedTables.Count == 0) { break; } } } if (processMode == EntityProcessMode.Udtraek) { if (settings.RebuildIndicesAfterReplication) { FixDBProblems.RebuildIndices(dbContext); } } if (processMode == EntityProcessMode.Dagi) { LocalDataHelper.CleanUpHelper(settings.EntitystateHistoryDeleteOldNumOfDays, settings.ArchiveLogsAfterDays, settings.DeleteOldArchivesAfterDays, settings.DeleteLogsAfterDays); } dbContext.Dispose(); _logger.Info($"{methodName}: Done. Execution time: {watch.Elapsed}"); _logger.Info($"{methodName}: Time of completion: {DateTime.Now.ToShortTimeString()}"); if (settings.UseMSApplicationInsights) { TelemetryHelper.AddTime(methodName, watch.Elapsed.TotalMinutes, "Minutes"); } if (settings.WaitForUserInput) { Console.WriteLine($"Done: {DateTime.Now.ToShortTimeString()}. Execution time: {watch.Elapsed}" + "\n"); Console.WriteLine("Press any key to exit..."); Console.ReadKey(); } } catch (Exception ex) { var msg = $"ERROR! Unexpected exception. Problem: {ex.Message}"; Console.WriteLine(msg); Console.WriteLine($"Time at which error occured {DateTime.Now.ToShortTimeString()}"); _logger.Error(msg, ex); if (Properties.Settings.Default.WaitForUserInput) { Console.WriteLine("Press any key to exit..."); Console.ReadKey(); } Environment.Exit((int)ReturnCode.UnknownError); } Environment.Exit((int)ReturnCode.Success); }