#pragma warning restore IDE0044 // Add readonly modifier
#endregion
#region Init and process Methods
/// <summary>
/// Init all class wide settings.
/// </summary>
/// <param name="dbContext">DawaReplicationDBContext instance.</param>
/// <param name="tableList">A list of tables to process.</param>
/// <param name="dawaApiUri">Typical 'https://dawa.aws.dk/' .</param>
/// <param name="readTimeoutInSeconds">DAWA api timeout in seconds. Normally 300 seconds.</param>
/// <param name="udtraekRowsMax">Max number of rows to read. When 0 all rows are read.</param>
public static void InitSettings(DawaReplicationDBContext dbContext, List <string> tableList, string dawaApiUri, int readTimeoutInSeconds, int udtraekRowsMax, int udtraekBulkSize, int dkStedDataStartPos, int dkStedBulkSize, int dbCommandTimeoutInSeconds, string tempDataFolderPath, bool activeFixes, string activeFixesListFileLocation, bool useMSApplicationInsights, bool jsonSerializerIgnoreNullValue)
{
if (tableList == null)
{
throw new ArgumentNullException(nameof(tableList));
}
if (tableList == null && tableList.Count == 0)
{
throw new ArgumentNullException(nameof(tableList));
}
if (string.IsNullOrEmpty(dawaApiUri))
{
throw new ArgumentNullException(nameof(dawaApiUri));
}
if (readTimeoutInSeconds == 0)
{
throw new ArgumentOutOfRangeException(nameof(readTimeoutInSeconds));
}
if (udtraekBulkSize == 0)
{
throw new ArgumentOutOfRangeException(nameof(udtraekBulkSize));
}
if (tempDataFolderPath == null)
{
throw new ArgumentNullException(nameof(tempDataFolderPath));
}
DBContext = dbContext ?? throw new ArgumentNullException(nameof(dbContext));
TableList = tableList;
DawaApiUri = dawaApiUri.EndsWith("/") ? dawaApiUri : dawaApiUri + "/";
ReadTimeoutInSeconds = readTimeoutInSeconds;
UdtraekRowsMax = udtraekRowsMax;
UdtraekBulkSize = udtraekBulkSize;
DKStedDataStartPos = dkStedDataStartPos;
DKStedBulkSize = dkStedBulkSize;
dbContext.Database.SetCommandTimeout(dbCommandTimeoutInSeconds);
TempDataFolderPath = tempDataFolderPath.EndsWith(@"\") ? tempDataFolderPath : tempDataFolderPath + @"\";
ActiveFixes = activeFixes;
ActiveFixesListFileLocation = activeFixesListFileLocation;
UseMSApplicationInsights = useMSApplicationInsights;
UserJsonSerializerSettings = new JsonSerializerSettings {
};
JsonSerializerIgnoreNullValue = jsonSerializerIgnoreNullValue;
if (jsonSerializerIgnoreNullValue)
{
UserJsonSerializerSettings = new JsonSerializerSettings {
NullValueHandling = NullValueHandling.Ignore
};
}
if (ActiveFixes)
{
fixInfoList = FixInfo.FixInfoList(ActiveFixesListFileLocation);
}
}
public static void UpdateDatabase()
{
var logger = log4net.LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
using (var dBContext = new DawaReplicationDBContext())
{
if (dBContext.Database.CanConnect())
{
try
{
dBContext.Database.Migrate();
Console.WriteLine(DbUpdateComplete);
}
catch (Exception e)
{
logger.Error("ERROR!: Could not apply the latest migration.", e);
Console.WriteLine("ERROR! Could not apply the latest migration. See error message below.");
Console.WriteLine(e);
}
}
else
{
logger.Error("ERROR!: No migration applied, failed to connect to database.");
Console.WriteLine("Error: Failed to connect to the database. Check your connection string.");
}
}
}
/// <summary>Check if the program can establish a connection to the database.</summary>
public static void HaltIfNoDBConnection()
{
try
{
using (var context = new DawaReplicationDBContext())
{
// context.Entitystate.Max(t => t.Txid).GetValueOrDefault(0);
if (!context.Database.CanConnect())
{
var conn = ConfigurationManager.ConnectionStrings["DawaDatabase"].ConnectionString;
var msg = $"ERROR! Failed to connect to database. Check ConnectionString '{conn}' in config file and database user role.";
Console.WriteLine(msg);
_logger.Error($"Main(): {msg}");
Environment.Exit((int)ReturnCode.DBConnectionError);
}
}
}
catch (Exception ex)
{
var conn = ConfigurationManager.ConnectionStrings["DawaDatabase"].ConnectionString;
var msg = $"ERROR! Failed to connect to database. Check ConnectionString '{conn}' in config file and database user role. Problem: {ex.Message}";
Console.WriteLine(msg);
_logger.Error($"Main(): {msg}", ex);
Environment.Exit((int)ReturnCode.DBConnectionError);
}
}
/// <summary>
/// Create row in EntityStateHistory table for current transaction.
/// </summary>
/// <param name="dbContext">Database context.</param>
/// <param name="mode">EntityProcessMode mode. Udtraek/Update/DAGI.</param>
/// <param name="entity">Entity name.</param>
/// <param name="txidTil">Transaction ID for updating the table up to.</param>
/// <returns>DateTime to synchronize between EntityState tables.</returns>
public static DateTime SetEntityStateHistoryStart(DawaReplicationDBContext dbContext, EntityProcessMode mode, string entity, long?txidTil)
{
if (dbContext == null)
{
throw new ArgumentNullException(nameof(dbContext));
}
if (entity == null)
{
throw new ArgumentNullException(nameof(entity));
}
entity = entity.ToLower();
EntitystateHistory row = new EntitystateHistory
{
Entity = entity,
};
row.TxidFra = (mode == EntityProcessMode.Udtraek || mode == EntityProcessMode.Dagi) ? -1 : dbContext.EntitystateHistory.Where(c => c.Entity == entity && c.Success == true).OrderByDescending(c => c.Starttime).First().TxidTil;
row.TxidTil = mode == EntityProcessMode.Dagi ? null : txidTil;
row.Success = null;
row.Starttime = DateTime.Now;
row.Finishtime = null;
row.InsertUpdateCount = null;
row.DeleteCount = null;
row.Message = null;
dbContext.Add(row);
dbContext.SaveChanges();
return(row.Starttime);
}
/// <summary>
/// Set EntityState row for entity to complete.
/// </summary>
/// <param name="dbContext">Database context.</param>
/// <param name="entity">Entity name.</param>
/// <param name="success">Bool success.</param>
/// <param name="txid">Transaction ID.</param>
/// <param name="count">Items processed.</param>
/// <param name="message">Error message if success = 0.</param>
/// <returns>DateTime finishtime to synchronize EntityState rows with EntityStateHistory´.</returns>
public static DateTime SetEntityStateDone(DawaReplicationDBContext dbContext, string entity, bool success, long?txid, int?count, string message = null)
{
if (dbContext == null)
{
throw new ArgumentNullException(nameof(dbContext));
}
if (entity == null)
{
throw new ArgumentNullException(nameof(entity));
}
var finishtime = DateTime.Now;
var row = dbContext.Entitystate.Single(c => c.Entity == entity.ToLower());
row.Txid = success ? txid : row.Txid;
row.Success = success;
row.Successtime = success ? finishtime : row.Successtime;
row.SuccesstimeChange = success && count.GetValueOrDefault() > 0 ? finishtime : row.SuccesstimeChange;
row.Finishtime = finishtime;
row.Count = count;
row.Message = message;
row.MonitoringIgnoreError = row.MonitoringIgnoreError == false || row.MonitoringIgnoreError == null ? false : true;
dbContext.Update(row);
dbContext.SaveChanges();
return(finishtime);
}
/// <summary>
/// Runs a cleanup job in the EntityStateHistory table, deleting old rows.
/// </summary>
/// <param name="days">Number of days after which rows should be deleted.</param>
/// <returns>int number of deleted records.</returns>
public static int CleanupOldEntityStateRecords(int days)
{
var retval = 0;
using (var context = new DawaReplicationDBContext())
{
var oldRecords = context.EntitystateHistory.Where(c => (DateTime.Now - c.Starttime).TotalDays > days);
retval = oldRecords.Count();
context.EntitystateHistory.RemoveRange(oldRecords);
context.SaveChanges();
}
return(retval);
}
/// <summary>
/// GetTxid for last for Entity to create a new search query.
/// </summary>
/// <param name="dbContext">The database context.</param>
/// <param name="entity">Name of the entity worked on.</param>
/// <returns>int transaction ID.</returns>
public static long?GetTxid(DawaReplicationDBContext dbContext, string entity)
{
if (dbContext == null)
{
throw new ArgumentNullException(nameof(dbContext));
}
if (entity == null)
{
throw new ArgumentNullException(nameof(entity));
}
return(dbContext.Entitystate.Single(c => c.Entity == entity.ToLower()).Txid);
}
/// <summary>Check if the program can establish a connection to the database.</summary>
public static void HaltIfDBSchemeNotInitialized()
{
try
{
using (var context = new DawaReplicationDBContext())
{
context.Entitystate.Max(t => t.Txid).GetValueOrDefault(0);
}
}
catch (Exception ex)
{
var conn = ConfigurationManager.ConnectionStrings["DawaDatabase"].ConnectionString;
var msg = $"ERROR! The database scheme has not been initialized. Check ConnectionString '{conn}' in config file and make sure the database is initialized. Problem: {ex.Message}";
Console.WriteLine(msg);
_logger.Error($"Main(): {msg}", ex);
Environment.Exit((int)ReturnCode.DBConnectionError);
}
}
/// <summary>
/// Updates an EntityStateHistory row after transaction is completed.
/// </summary>
/// <param name="dbContext">Database context.</param>
/// <param name="entity">Entity name.</param>
/// <param name="success">Bool success.</param>
/// <param name="starttime">Start time.</param>
/// <param name="finishtime">Time of completion.</param>
/// <param name="txidFra">Updated entity from transaction Id txidFra.</param>
/// <param name="insertUpdateCount">Inserted or updated row count.</param>
/// <param name="deleteCount">Deleted row count.</param>
/// <param name="message">Error message if success = 0.</param>
public static void SetEntityStateHistoryDone(DawaReplicationDBContext dbContext, string entity, bool success, DateTime starttime, DateTime finishtime, long?txidFra, int?insertUpdateCount, int?deleteCount, string message = null)
{
if (dbContext == null)
{
throw new ArgumentNullException(nameof(dbContext));
}
var row = dbContext.EntitystateHistory.Single(c => c.Entity == entity.ToLower() && c.Starttime == starttime);
// var row = dbContext.EntitystateHistory.OrderByDescending(x => x.Starttime).Where(c => c.Entity == entity.ToLower()).First();
row.TxidFra = success == true ? txidFra : row.TxidFra;
row.Success = success;
row.Finishtime = finishtime;
row.InsertUpdateCount = success == true ? insertUpdateCount : null;
row.DeleteCount = success == true ? deleteCount : null;
row.Message = message;
dbContext.Update(row);
dbContext.SaveChanges();
}
/// <summary>
/// Create a new row in Entitystate for given entity.
/// </summary>
/// <param name="dbContext">Database context.</param>
/// <param name="mode">EntityProcessMode. Udtraek/Update/Dagi.</param>
/// <param name="entity">Entity name.</param>
/// <param name="txid">Transaction ID.</param>
public static void SetEntityStateStart(DawaReplicationDBContext dbContext, EntityProcessMode mode, string entity, long txid)
{
if (dbContext == null)
{
throw new ArgumentNullException(nameof(dbContext));
}
if (entity == null)
{
throw new ArgumentNullException(nameof(entity));
}
entity = entity.ToLower();
// Create entity if missing:
if (!dbContext.Entitystate.Any(c => c.Entity == entity))
{
var entitystate = new Entitystate
{
Entity = entity,
};
dbContext.Add(entitystate);
dbContext.SaveChanges();
}
var row = dbContext.Entitystate.Single(c => c.Entity == entity);
row.Txid = mode == EntityProcessMode.Update ? row.Txid : txid;
row.Success = null;
row.Successtime = null;
row.SuccesstimeChange = row.SuccesstimeChange;
row.Starttime = DateTime.Now;
row.Finishtime = null;
row.Count = null;
row.Message = null;
dbContext.Update(row);
dbContext.SaveChanges();
}
public static void DeleteOldRows(DawaReplicationDBContext dbContext, string name, DateTime date)
{
if (dbContext == null)
{
throw new ArgumentNullException(nameof(dbContext));
}
switch (name)
{
case "steder":
dbContext.Dagi_Steder.RemoveRange(dbContext.Dagi_Steder.Where(c => c.EntityUpdated < date));
dbContext.SaveChanges();
break;
case "stednavne":
dbContext.Dagi_Stednavne.RemoveRange(dbContext.Dagi_Stednavne.Where(c => c.EntityUpdated < date));
dbContext.SaveChanges();
break;
default:
break;
}
}
/// <summary>
/// Main program.
/// </summary>
/// <param name="args">String arguments for program initialization.</param>
public static void Main(string[] args)
{
Thread.CurrentThread.CurrentCulture = System.Globalization.CultureInfo.InvariantCulture;
var settings = Properties.Settings.Default;
if (settings.UseMSApplicationInsights)
{
try
{
ApplicationInsightInitializer.Initialize();
}
catch (Exception ex)
{
var msg = $"ERROR! Failed to initialize MS Application Insights. Problem: {ex.Message}";
Console.WriteLine(msg);
_logger.Error(msg, ex);
Environment.Exit((int)ReturnCode.ApplicationInsightInitializeError);
}
}
Helpers.InitializeHelpers.HaltIfNoDBConnection();
Helpers.InitializeHelpers.HaltIfAllreadyRunning();
try
{
#pragma warning disable CS0436 // Type conflicts with imported type
SqlServerTypes.Utilities.LoadNativeAssemblies(AppDomain.CurrentDomain.BaseDirectory);
#pragma warning restore CS0436 // Type conflicts with imported type
string programAssemblyVersion = Assembly.GetExecutingAssembly().GetName().Version.ToString();
string dataAccessAssemblyVersion = Assembly.GetAssembly(typeof(Adgangsadresse)).GetName().Version.ToString();
var methodName = LoggingUtils.GetMethodName();
Console.WriteLine($"Starting DawaReplication assembly ver. {programAssemblyVersion}; DataAccess ver. {dataAccessAssemblyVersion} ....");
_logger.Info($"{methodName}: Starting DawaReplication assembly ver. {programAssemblyVersion}; DataAccess ver. {dataAccessAssemblyVersion}");
_logger.Info($"{methodName}: Settings = {SettingsHelper.SettingsAsString()}");
_logger.Info($"{methodName}: Time of initialization: {DateTime.Now.ToShortTimeString()}");
var processMode = InitialArgumentHelper.InitializeProcess(args);
Helpers.InitializeHelpers.HaltIfDBSchemeNotInitialized();
Console.WriteLine($"Time: {DateTime.Now.ToShortTimeString()}. ProcessMode: {processMode}\n");
var list = TableInfo.GetTableInfoList(AppDomain.CurrentDomain.BaseDirectory + settings.TableInfoFile, false);
#pragma warning disable CA2000 // Dispose objects before losing scope
var dbContext = new DawaReplicationDBContext();
#pragma warning restore CA2000 // Dispose objects before losing scope
if (processMode == EntityProcessMode.Udtraek)
{
FixDBProblems.FixDbProblems(dbContext);
}
EntityManager.InitSettings(dbContext, list.Select(x => x.Name).ToList(), settings.DawaApiUri, settings.DawaApiReadTimeoutInSeconds, settings.UdtraekRowsMax, settings.UdtraekBulkSize, settings.DKStedDataStartPos, settings.DKStedBulkSize, settings.DBCommandTimeoutInSeconds, settings.TempDataFolderPath, settings.ActiveFixes, settings.ActiveFixesListFileLocation, settings.UseMSApplicationInsights, settings.JsonSerializerIgnoreNullValue);
LocalDataHelper.MakeFolder(settings.TempDataFolderPath);
var msg = LoggingUtils.MakeMessage(methodName, " TableInfoFile", list);
_logger.Info(msg);
var watch = Stopwatch.StartNew();
var dawaProcessInfo = Helpers.InitializeHelpers.MakeDawaProcessInfo(processMode, settings.DawaApiUri, settings.DawaApiReadTimeoutInSeconds, settings.TxidOverride);
_logger.Info($"{methodName}: Got latest transaction ID = {dawaProcessInfo.Txid}");
EntityManager.ProcessTables(processMode, dawaProcessInfo);
if (dawaProcessInfo.FailedTables.Any() && (processMode == EntityProcessMode.Dagi || processMode == EntityProcessMode.Udtraek))
{
for (int i = 1; i <= settings.RetryCount; i++)
{
msg = $"{methodName}: Failed to process all tables. No of failed tables: {dawaProcessInfo.FailedTables.Count}. Retrying again in {settings.RetryTimerInMinutes * i} minutes.";
Console.WriteLine($"\nWARNING. Time: {DateTime.Now.ToShortTimeString()}. {msg}");
_logger.Warn(msg);
EntityManager.TableList = new List <string>(dawaProcessInfo.FailedTables);
dawaProcessInfo.FailedTables.Clear();
Thread.Sleep(settings.RetryTimerInMinutes * 60 * 1000 * i);
EntityManager.ProcessTables(processMode, dawaProcessInfo);
if (dawaProcessInfo.FailedTables.Count == 0)
{
break;
}
}
}
if (processMode == EntityProcessMode.Udtraek)
{
if (settings.RebuildIndicesAfterReplication)
{
FixDBProblems.RebuildIndices(dbContext);
}
}
if (processMode == EntityProcessMode.Dagi)
{
LocalDataHelper.CleanUpHelper(settings.EntitystateHistoryDeleteOldNumOfDays, settings.ArchiveLogsAfterDays, settings.DeleteOldArchivesAfterDays, settings.DeleteLogsAfterDays);
}
dbContext.Dispose();
_logger.Info($"{methodName}: Done. Execution time: {watch.Elapsed}");
_logger.Info($"{methodName}: Time of completion: {DateTime.Now.ToShortTimeString()}");
if (settings.UseMSApplicationInsights)
{
TelemetryHelper.AddTime(methodName, watch.Elapsed.TotalMinutes, "Minutes");
}
if (settings.WaitForUserInput)
{
Console.WriteLine($"Done: {DateTime.Now.ToShortTimeString()}. Execution time: {watch.Elapsed}" + "\n");
Console.WriteLine("Press any key to exit...");
Console.ReadKey();
}
}
catch (Exception ex)
{
var msg = $"ERROR! Unexpected exception. Problem: {ex.Message}";
Console.WriteLine(msg);
Console.WriteLine($"Time at which error occured {DateTime.Now.ToShortTimeString()}");
_logger.Error(msg, ex);
if (Properties.Settings.Default.WaitForUserInput)
{
Console.WriteLine("Press any key to exit...");
Console.ReadKey();
}
Environment.Exit((int)ReturnCode.UnknownError);
}
Environment.Exit((int)ReturnCode.Success);
}
