#pragma warning disable CA1801 // Remove unused parameter
#pragma warning disable IDE0060 // Remove unused parameter
/// <summary>
/// Get data from DAWA DAGI WebApi and bootload database.
/// </summary>
/// <param name="transactionInfo">Dummy - not used.</param>
/// <returns>Count of all rows.</returns>
/// <typeparam name="T">Any DAGI entity class in the JOInformatik.DawaReplication.DataAccess namespace.</typeparam>
public static int Dagi <T>(DawaProcessInfo transactionInfo, DateTime starttime)
#pragma warning restore IDE0060 // Remove unused parameter
#pragma warning restore CA1801 // Remove unused parameter
where T : DAGIBase
{
var methodName = LoggingUtils.GetMethodName();
string entityName = typeof(T).Name;
Logger?.Info($"{methodName}: Processing entity {entityName}");
var entityNameApi = entityName.Substring(6).ToLowerInvariant(); // Get rid of starting "DAGI__"
int counter = 0;
var itemList = new List <T>();
var syncDeletesTime = DateTime.Now;
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
using (var httpClient = new HttpClient())
{
// Always use "&noformat" for increased performance:
var stream = httpClient.GetStreamAsync($"{DawaApiUri}{entityNameApi}?format=geojson&srid={(int)KoordinatsystemSrid.ETRS89}&noformat").Result;
stream.ReadTimeout = ReadTimeoutInSeconds * 1000;
var fileStream = LocalDataHelper.CreateTempFile(entityName);
try
{
// Remove geojson header and save to file. Read from same file and stop when reaching end of array(list of objects).
for (int x = 0; x < DKStedDataStartPos; x++)
{
stream.ReadByte();
}
stream.CopyTo(fileStream);
stream.Close();
fileStream.Close();
}
catch
{
// If something goes wrong, make sure to close filestream, so we can try to write to the file again.
stream.Close();
fileStream.Close();
throw;
}
}
var fileLocation = LocalDataHelper.RenameTempFile(entityName);
Console.Write("Done saving to file.");
using (var reader = new JsonTextReader(new StreamReader(fileLocation)))
{
DBContext.Database.BeginTransaction();
try
{
while (reader.Read())
{
if (reader.TokenType == JsonToken.EndArray)
{
break;
}
if (reader.TokenType == JsonToken.StartObject)
{
var item = JObject.Load(reader);
T rootItemAsObject = JsonConvert.DeserializeObject <T>(item.ToString(Formatting.None), UserJsonSerializerSettings);
T itemAsObject = JsonConvert.DeserializeObject <T>(item["properties"].ToString(Formatting.None), UserJsonSerializerSettings);
itemAsObject.SetEntityFields(item);
itemAsObject.Geometry = rootItemAsObject.Geometry;
itemList.Add(itemAsObject);
counter += 1;
if (itemList.Count % DKStedBulkSize == 0)
{
DBContext.BulkInsertOrUpdate(itemList);
itemList.Clear();
}
}
}
Logger?.Info($"{methodName}: Total fetched: {counter}");
DBContext.BulkInsertOrUpdate(itemList);
DagiStedHelper.DeleteOldRows(DBContext, entityName, syncDeletesTime);
var finishtime = EntityStateHelper.SetEntityStateDone(DBContext, entityName, true, -1, counter);
EntityStateHelper.SetEntityStateHistoryDone(DBContext, entityName, true, starttime, finishtime, -1, itemList.Count, 0);
if (UseMSApplicationInsights)
{
TelemetryHelper.AddTelemetryForEntity(EntityProcessMode.Dagi, entityName, stopwatch);
}
DBContext.Database.CommitTransaction();
}
catch (Exception ex)
{
if (DBContext.Database.CurrentTransaction != null)
{
DBContext.Database.RollbackTransaction();
}
var exception = ex.InnerException ?? ex;
var finishtime = EntityStateHelper.SetEntityStateDone(DBContext, entityName, false, -1, counter, exception.Message);
EntityStateHelper.SetEntityStateHistoryDone(DBContext, entityName, false, starttime, finishtime, -1, null, null, exception.Message);
if (reader != null)
{
reader.Close();
}
throw;
}
}
// Delete temporary file.
LocalDataHelper.RemoveTempFile(fileLocation);
return(counter);
}
/// <summary>
/// Get data from DAWA WebApi and bootload database.
/// </summary>
/// <param name="dawaProcessInfo">Latest id from Dawa.</param>
/// <returns>Count of all inserted rows.</returns>
/// <typeparam name="T">Any entity class in the JOInformatik.DawaReplication.DataAccess namespace.</typeparam>
public static int Udtraek <T>(DawaProcessInfo dawaProcessInfo, DateTime starttime)
where T : ReplicationBase
{
if (dawaProcessInfo == null)
{
throw new ArgumentNullException(nameof(dawaProcessInfo));
}
if (dawaProcessInfo.Txid < 1)
{
#pragma warning disable CA2208 // Instantiate argument exceptions correctly
throw new ArgumentOutOfRangeException(nameof(dawaProcessInfo.Txid));
#pragma warning restore CA2208 // Instantiate argument exceptions correctly
}
DBContext.ChangeTracker.AutoDetectChangesEnabled = false;
var methodName = LoggingUtils.GetMethodName();
string entityName = typeof(T).Name;
Logger?.Info($"{methodName}: Processing entity {entityName}");
var sql = $"TRUNCATE TABLE [{entityName}]";
#pragma warning disable EF1000 // Possible SQL injection vulnerability.
DBContext.Database.ExecuteSqlCommand(sql);
#pragma warning restore EF1000 // Possible SQL injection vulnerability.
var itemList = new List <T>();
int counter = 0;
// Always use "&noformat" for increased performance:
using (var httpClient = new HttpClient())
{
var stream = httpClient.GetStreamAsync($"{DawaApiUri}replikering/udtraek?entitet={entityName.ToLowerInvariant()}&txid={dawaProcessInfo.Txid}&noformat").Result;
stream.ReadTimeout = ReadTimeoutInSeconds * 1000;
using (var reader = new JsonTextReader(new StreamReader(stream)))
{
try
{
while (reader.Read())
{
if (reader.TokenType == JsonToken.StartObject)
{
T itemAsObject = JsonConvert.DeserializeObject <T>(JObject.Load(reader).ToString(Formatting.None), UserJsonSerializerSettings);
itemAsObject.EntityTxid = dawaProcessInfo.Txid;
itemList.Add(itemAsObject);
counter += 1;
// Prevent memory overflow, insert some items to database and clear the list to make space for new items.
// TODO Ver 1.5 Variable Bulkinsert size depending on table.
if (itemList.Count % UdtraekBulkSize == 0)
{
DBContext.BulkInsert(itemList);
Logger?.Debug($"{methodName}: Bulk- Fetched: {counter}");
itemList.Clear();
}
if (UdtraekRowsMax != 0 && counter >= UdtraekRowsMax)
{
break;
}
}
}
Logger?.Info($"{methodName}: Total fetched: {counter}");
Logger?.Info($"{methodName}: Entity {entityName} time of completion: {DateTime.Now.ToShortTimeString()}");
DBContext.BulkInsert(itemList);
var finishtime = EntityStateHelper.SetEntityStateDone(DBContext, entityName, true, dawaProcessInfo.Txid, counter);
EntityStateHelper.SetEntityStateHistoryDone(DBContext, entityName, true, starttime, finishtime, 0, counter, 0);
}
catch (Exception ex)
{
var exception = ex.InnerException ?? ex;
var finishtime = EntityStateHelper.SetEntityStateDone(DBContext, entityName, false, dawaProcessInfo.Txid, counter, exception.Message);
EntityStateHelper.SetEntityStateHistoryDone(DBContext, entityName, false, starttime, finishtime, 0, null, null, exception.Message);
throw;
}
}
}
return(counter);
}
/// <summary>
/// Get data from DAWA WebApi and update database.
/// </summary>
/// <param name="dawaProcessInfo">Latest id from Dawa.</param>
/// <returns>Count of all deleted, inserted and updated rows.</returns>
/// <typeparam name="T">Any entity class in the JOInformatik.DawaReplication.DataAccess namespace.</typeparam>
public static int Update <T>(DawaProcessInfo dawaProcessInfo, DateTime starttime)
where T : ReplicationBase
{
if (dawaProcessInfo == null)
{
throw new ArgumentNullException(nameof(dawaProcessInfo));
}
if (dawaProcessInfo.Txid < 1)
{
#pragma warning disable CA2208 // Instantiate argument exceptions correctly
throw new ArgumentOutOfRangeException(nameof(dawaProcessInfo.Txid));
#pragma warning restore CA2208 // Instantiate argument exceptions correctly
}
var methodName = LoggingUtils.GetMethodName();
string entityName = typeof(T).Name;
Logger?.Info($"{methodName}: Processing entity {entityName}");
var fixInfo = fixInfoList.Find(x => x.TableName == entityName);
var listInsertOrUpdate = new List <T>();
var listDelete = new List <T>();
long txidfra = EntityStateHelper.GetTxid(DBContext, entityName).Value + 1;
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
// Always use "&noformat" for increased performance:
using (var httpClient = new HttpClient())
{
var stream = httpClient.GetStreamAsync($"{DawaApiUri}replikering/haendelser?entitet={entityName.ToLowerInvariant()}&txidfra={txidfra}&txidtil={dawaProcessInfo.Txid}&noformat").Result;
stream.ReadTimeout = ReadTimeoutInSeconds * 1000;
using (var reader = new JsonTextReader(new StreamReader(stream)))
{
while (reader.Read())
{
if (reader.TokenType == JsonToken.StartObject)
{
var item = JObject.Load(reader);
var operation = item.Property("operation").Value.ToString();
if (operation != null && (operation == "update" || operation == "insert" || operation == "delete"))
{
var itemData = item.Property("data").First.ToString(Formatting.None);
if (fixInfo != null)
{
itemData = itemData.Replace(fixInfo.DataValueBad, fixInfo.DataValueValid);
}
T itemAsObject = JsonConvert.DeserializeObject <T>(itemData, UserJsonSerializerSettings);
itemAsObject.SetEntityFields(item);
switch (operation)
{
case "insert":
case "update":
listInsertOrUpdate.Add(itemAsObject);
break;
case "delete":
listDelete.Add(itemAsObject);
break;
}
}
else
{
Logger?.Warn($"{methodName}: Unrecognized operation '{operation}' for {entityName}. Txid = {dawaProcessInfo.Txid}");
}
}
}
}
}
UpdateEntityHelper.ProcessOperationLists(listDelete, listInsertOrUpdate);
int deleteCount = listDelete.Count;
int insertUpdateCount = listInsertOrUpdate.Count;
if (insertUpdateCount == 0 && deleteCount == 0)
{
Logger?.Info($"{methodName}: {entityName}: No changes");
var finishtime = EntityStateHelper.SetEntityStateDone(DBContext, entityName, true, dawaProcessInfo.Txid, 0);
EntityStateHelper.SetEntityStateHistoryDone(DBContext, entityName, true, starttime, finishtime, txidfra, insertUpdateCount, deleteCount);
}
else
{
Logger?.Info($"{methodName}: {entityName}: Inserted and Updated = {insertUpdateCount}, Deleted={deleteCount}");
int totalCount = insertUpdateCount + deleteCount;
DBContext.Database.BeginTransaction();
try
{
DBContext.BulkInsertOrUpdate(listInsertOrUpdate);
DBContext.BulkDelete(listDelete);
var finishtime = EntityStateHelper.SetEntityStateDone(DBContext, entityName, true, dawaProcessInfo.Txid, totalCount);
EntityStateHelper.SetEntityStateHistoryDone(DBContext, entityName, true, starttime, finishtime, txidfra, insertUpdateCount, deleteCount);
DBContext.Database.CommitTransaction();
if (UseMSApplicationInsights)
{
TelemetryHelper.AddTelemetryForEntity(EntityProcessMode.Update, entityName, stopwatch);
}
}
catch (Exception ex)
{
if (DBContext.Database.CurrentTransaction != null)
{
DBContext.Database.RollbackTransaction();
}
var exception = ex.InnerException ?? ex;
var finishtime = EntityStateHelper.SetEntityStateDone(DBContext, entityName, false, txidfra, totalCount, exception.Message);
EntityStateHelper.SetEntityStateHistoryDone(DBContext, entityName, false, starttime, finishtime, txidfra, null, null, exception.Message);
throw;
}
}
return(listInsertOrUpdate.Count + listDelete.Count);
}
/// <summary>
/// Main program.
/// </summary>
/// <param name="args">String arguments for program initialization.</param>
public static void Main(string[] args)
{
Thread.CurrentThread.CurrentCulture = System.Globalization.CultureInfo.InvariantCulture;
var settings = Properties.Settings.Default;
if (settings.UseMSApplicationInsights)
{
try
{
ApplicationInsightInitializer.Initialize();
}
catch (Exception ex)
{
var msg = $"ERROR! Failed to initialize MS Application Insights. Problem: {ex.Message}";
Console.WriteLine(msg);
_logger.Error(msg, ex);
Environment.Exit((int)ReturnCode.ApplicationInsightInitializeError);
}
}
Helpers.InitializeHelpers.HaltIfNoDBConnection();
Helpers.InitializeHelpers.HaltIfAllreadyRunning();
try
{
#pragma warning disable CS0436 // Type conflicts with imported type
SqlServerTypes.Utilities.LoadNativeAssemblies(AppDomain.CurrentDomain.BaseDirectory);
#pragma warning restore CS0436 // Type conflicts with imported type
string programAssemblyVersion = Assembly.GetExecutingAssembly().GetName().Version.ToString();
string dataAccessAssemblyVersion = Assembly.GetAssembly(typeof(Adgangsadresse)).GetName().Version.ToString();
var methodName = LoggingUtils.GetMethodName();
Console.WriteLine($"Starting DawaReplication assembly ver. {programAssemblyVersion}; DataAccess ver. {dataAccessAssemblyVersion} ....");
_logger.Info($"{methodName}: Starting DawaReplication assembly ver. {programAssemblyVersion}; DataAccess ver. {dataAccessAssemblyVersion}");
_logger.Info($"{methodName}: Settings = {SettingsHelper.SettingsAsString()}");
_logger.Info($"{methodName}: Time of initialization: {DateTime.Now.ToShortTimeString()}");
var processMode = InitialArgumentHelper.InitializeProcess(args);
Helpers.InitializeHelpers.HaltIfDBSchemeNotInitialized();
Console.WriteLine($"Time: {DateTime.Now.ToShortTimeString()}. ProcessMode: {processMode}\n");
var list = TableInfo.GetTableInfoList(AppDomain.CurrentDomain.BaseDirectory + settings.TableInfoFile, false);
#pragma warning disable CA2000 // Dispose objects before losing scope
var dbContext = new DawaReplicationDBContext();
#pragma warning restore CA2000 // Dispose objects before losing scope
if (processMode == EntityProcessMode.Udtraek)
{
FixDBProblems.FixDbProblems(dbContext);
}
EntityManager.InitSettings(dbContext, list.Select(x => x.Name).ToList(), settings.DawaApiUri, settings.DawaApiReadTimeoutInSeconds, settings.UdtraekRowsMax, settings.UdtraekBulkSize, settings.DKStedDataStartPos, settings.DKStedBulkSize, settings.DBCommandTimeoutInSeconds, settings.TempDataFolderPath, settings.ActiveFixes, settings.ActiveFixesListFileLocation, settings.UseMSApplicationInsights, settings.JsonSerializerIgnoreNullValue);
LocalDataHelper.MakeFolder(settings.TempDataFolderPath);
var msg = LoggingUtils.MakeMessage(methodName, " TableInfoFile", list);
_logger.Info(msg);
var watch = Stopwatch.StartNew();
var dawaProcessInfo = Helpers.InitializeHelpers.MakeDawaProcessInfo(processMode, settings.DawaApiUri, settings.DawaApiReadTimeoutInSeconds, settings.TxidOverride);
_logger.Info($"{methodName}: Got latest transaction ID = {dawaProcessInfo.Txid}");
EntityManager.ProcessTables(processMode, dawaProcessInfo);
if (dawaProcessInfo.FailedTables.Any() && (processMode == EntityProcessMode.Dagi || processMode == EntityProcessMode.Udtraek))
{
for (int i = 1; i <= settings.RetryCount; i++)
{
msg = $"{methodName}: Failed to process all tables. No of failed tables: {dawaProcessInfo.FailedTables.Count}. Retrying again in {settings.RetryTimerInMinutes * i} minutes.";
Console.WriteLine($"\nWARNING. Time: {DateTime.Now.ToShortTimeString()}. {msg}");
_logger.Warn(msg);
EntityManager.TableList = new List <string>(dawaProcessInfo.FailedTables);
dawaProcessInfo.FailedTables.Clear();
Thread.Sleep(settings.RetryTimerInMinutes * 60 * 1000 * i);
EntityManager.ProcessTables(processMode, dawaProcessInfo);
if (dawaProcessInfo.FailedTables.Count == 0)
{
break;
}
}
}
if (processMode == EntityProcessMode.Udtraek)
{
if (settings.RebuildIndicesAfterReplication)
{
FixDBProblems.RebuildIndices(dbContext);
}
}
if (processMode == EntityProcessMode.Dagi)
{
LocalDataHelper.CleanUpHelper(settings.EntitystateHistoryDeleteOldNumOfDays, settings.ArchiveLogsAfterDays, settings.DeleteOldArchivesAfterDays, settings.DeleteLogsAfterDays);
}
dbContext.Dispose();
_logger.Info($"{methodName}: Done. Execution time: {watch.Elapsed}");
_logger.Info($"{methodName}: Time of completion: {DateTime.Now.ToShortTimeString()}");
if (settings.UseMSApplicationInsights)
{
TelemetryHelper.AddTime(methodName, watch.Elapsed.TotalMinutes, "Minutes");
}
if (settings.WaitForUserInput)
{
Console.WriteLine($"Done: {DateTime.Now.ToShortTimeString()}. Execution time: {watch.Elapsed}" + "\n");
Console.WriteLine("Press any key to exit...");
Console.ReadKey();
}
}
catch (Exception ex)
{
var msg = $"ERROR! Unexpected exception. Problem: {ex.Message}";
Console.WriteLine(msg);
Console.WriteLine($"Time at which error occured {DateTime.Now.ToShortTimeString()}");
_logger.Error(msg, ex);
if (Properties.Settings.Default.WaitForUserInput)
{
Console.WriteLine("Press any key to exit...");
Console.ReadKey();
}
Environment.Exit((int)ReturnCode.UnknownError);
}
Environment.Exit((int)ReturnCode.Success);
}