#pragma warning disable CA1801 // Remove unused parameter #pragma warning disable IDE0060 // Remove unused parameter /// <summary> /// Get data from DAWA DAGI WebApi and bootload database. /// </summary> /// <param name="transactionInfo">Dummy - not used.</param> /// <returns>Count of all rows.</returns> /// <typeparam name="T">Any DAGI entity class in the JOInformatik.DawaReplication.DataAccess namespace.</typeparam> public static int Dagi <T>(DawaProcessInfo transactionInfo, DateTime starttime) #pragma warning restore IDE0060 // Remove unused parameter #pragma warning restore CA1801 // Remove unused parameter where T : DAGIBase { var methodName = LoggingUtils.GetMethodName(); string entityName = typeof(T).Name; Logger?.Info($"{methodName}: Processing entity {entityName}"); var entityNameApi = entityName.Substring(6).ToLowerInvariant(); // Get rid of starting "DAGI__" int counter = 0; var itemList = new List <T>(); var syncDeletesTime = DateTime.Now; var stopwatch = System.Diagnostics.Stopwatch.StartNew(); using (var httpClient = new HttpClient()) { // Always use "&noformat" for increased performance: var stream = httpClient.GetStreamAsync($"{DawaApiUri}{entityNameApi}?format=geojson&srid={(int)KoordinatsystemSrid.ETRS89}&noformat").Result; stream.ReadTimeout = ReadTimeoutInSeconds * 1000; var fileStream = LocalDataHelper.CreateTempFile(entityName); try { // Remove geojson header and save to file. Read from same file and stop when reaching end of array(list of objects). for (int x = 0; x < DKStedDataStartPos; x++) { stream.ReadByte(); } stream.CopyTo(fileStream); stream.Close(); fileStream.Close(); } catch { // If something goes wrong, make sure to close filestream, so we can try to write to the file again. stream.Close(); fileStream.Close(); throw; } } var fileLocation = LocalDataHelper.RenameTempFile(entityName); Console.Write("Done saving to file."); using (var reader = new JsonTextReader(new StreamReader(fileLocation))) { DBContext.Database.BeginTransaction(); try { while (reader.Read()) { if (reader.TokenType == JsonToken.EndArray) { break; } if (reader.TokenType == JsonToken.StartObject) { var item = JObject.Load(reader); T rootItemAsObject = JsonConvert.DeserializeObject <T>(item.ToString(Formatting.None), UserJsonSerializerSettings); T itemAsObject = JsonConvert.DeserializeObject <T>(item["properties"].ToString(Formatting.None), UserJsonSerializerSettings); itemAsObject.SetEntityFields(item); itemAsObject.Geometry = rootItemAsObject.Geometry; itemList.Add(itemAsObject); counter += 1; if (itemList.Count % DKStedBulkSize == 0) { DBContext.BulkInsertOrUpdate(itemList); itemList.Clear(); } } } Logger?.Info($"{methodName}: Total fetched: {counter}"); DBContext.BulkInsertOrUpdate(itemList); DagiStedHelper.DeleteOldRows(DBContext, entityName, syncDeletesTime); var finishtime = EntityStateHelper.SetEntityStateDone(DBContext, entityName, true, -1, counter); EntityStateHelper.SetEntityStateHistoryDone(DBContext, entityName, true, starttime, finishtime, -1, itemList.Count, 0); if (UseMSApplicationInsights) { TelemetryHelper.AddTelemetryForEntity(EntityProcessMode.Dagi, entityName, stopwatch); } DBContext.Database.CommitTransaction(); } catch (Exception ex) { if (DBContext.Database.CurrentTransaction != null) { DBContext.Database.RollbackTransaction(); } var exception = ex.InnerException ?? ex; var finishtime = EntityStateHelper.SetEntityStateDone(DBContext, entityName, false, -1, counter, exception.Message); EntityStateHelper.SetEntityStateHistoryDone(DBContext, entityName, false, starttime, finishtime, -1, null, null, exception.Message); if (reader != null) { reader.Close(); } throw; } } // Delete temporary file. LocalDataHelper.RemoveTempFile(fileLocation); return(counter); }
/// <summary> /// Get data from DAWA WebApi and bootload database. /// </summary> /// <param name="dawaProcessInfo">Latest id from Dawa.</param> /// <returns>Count of all inserted rows.</returns> /// <typeparam name="T">Any entity class in the JOInformatik.DawaReplication.DataAccess namespace.</typeparam> public static int Udtraek <T>(DawaProcessInfo dawaProcessInfo, DateTime starttime) where T : ReplicationBase { if (dawaProcessInfo == null) { throw new ArgumentNullException(nameof(dawaProcessInfo)); } if (dawaProcessInfo.Txid < 1) { #pragma warning disable CA2208 // Instantiate argument exceptions correctly throw new ArgumentOutOfRangeException(nameof(dawaProcessInfo.Txid)); #pragma warning restore CA2208 // Instantiate argument exceptions correctly } DBContext.ChangeTracker.AutoDetectChangesEnabled = false; var methodName = LoggingUtils.GetMethodName(); string entityName = typeof(T).Name; Logger?.Info($"{methodName}: Processing entity {entityName}"); var sql = $"TRUNCATE TABLE [{entityName}]"; #pragma warning disable EF1000 // Possible SQL injection vulnerability. DBContext.Database.ExecuteSqlCommand(sql); #pragma warning restore EF1000 // Possible SQL injection vulnerability. var itemList = new List <T>(); int counter = 0; // Always use "&noformat" for increased performance: using (var httpClient = new HttpClient()) { var stream = httpClient.GetStreamAsync($"{DawaApiUri}replikering/udtraek?entitet={entityName.ToLowerInvariant()}&txid={dawaProcessInfo.Txid}&noformat").Result; stream.ReadTimeout = ReadTimeoutInSeconds * 1000; using (var reader = new JsonTextReader(new StreamReader(stream))) { try { while (reader.Read()) { if (reader.TokenType == JsonToken.StartObject) { T itemAsObject = JsonConvert.DeserializeObject <T>(JObject.Load(reader).ToString(Formatting.None), UserJsonSerializerSettings); itemAsObject.EntityTxid = dawaProcessInfo.Txid; itemList.Add(itemAsObject); counter += 1; // Prevent memory overflow, insert some items to database and clear the list to make space for new items. // TODO Ver 1.5 Variable Bulkinsert size depending on table. if (itemList.Count % UdtraekBulkSize == 0) { DBContext.BulkInsert(itemList); Logger?.Debug($"{methodName}: Bulk- Fetched: {counter}"); itemList.Clear(); } if (UdtraekRowsMax != 0 && counter >= UdtraekRowsMax) { break; } } } Logger?.Info($"{methodName}: Total fetched: {counter}"); Logger?.Info($"{methodName}: Entity {entityName} time of completion: {DateTime.Now.ToShortTimeString()}"); DBContext.BulkInsert(itemList); var finishtime = EntityStateHelper.SetEntityStateDone(DBContext, entityName, true, dawaProcessInfo.Txid, counter); EntityStateHelper.SetEntityStateHistoryDone(DBContext, entityName, true, starttime, finishtime, 0, counter, 0); } catch (Exception ex) { var exception = ex.InnerException ?? ex; var finishtime = EntityStateHelper.SetEntityStateDone(DBContext, entityName, false, dawaProcessInfo.Txid, counter, exception.Message); EntityStateHelper.SetEntityStateHistoryDone(DBContext, entityName, false, starttime, finishtime, 0, null, null, exception.Message); throw; } } } return(counter); }
/// <summary> /// Get data from DAWA WebApi and update database. /// </summary> /// <param name="dawaProcessInfo">Latest id from Dawa.</param> /// <returns>Count of all deleted, inserted and updated rows.</returns> /// <typeparam name="T">Any entity class in the JOInformatik.DawaReplication.DataAccess namespace.</typeparam> public static int Update <T>(DawaProcessInfo dawaProcessInfo, DateTime starttime) where T : ReplicationBase { if (dawaProcessInfo == null) { throw new ArgumentNullException(nameof(dawaProcessInfo)); } if (dawaProcessInfo.Txid < 1) { #pragma warning disable CA2208 // Instantiate argument exceptions correctly throw new ArgumentOutOfRangeException(nameof(dawaProcessInfo.Txid)); #pragma warning restore CA2208 // Instantiate argument exceptions correctly } var methodName = LoggingUtils.GetMethodName(); string entityName = typeof(T).Name; Logger?.Info($"{methodName}: Processing entity {entityName}"); var fixInfo = fixInfoList.Find(x => x.TableName == entityName); var listInsertOrUpdate = new List <T>(); var listDelete = new List <T>(); long txidfra = EntityStateHelper.GetTxid(DBContext, entityName).Value + 1; var stopwatch = System.Diagnostics.Stopwatch.StartNew(); // Always use "&noformat" for increased performance: using (var httpClient = new HttpClient()) { var stream = httpClient.GetStreamAsync($"{DawaApiUri}replikering/haendelser?entitet={entityName.ToLowerInvariant()}&txidfra={txidfra}&txidtil={dawaProcessInfo.Txid}&noformat").Result; stream.ReadTimeout = ReadTimeoutInSeconds * 1000; using (var reader = new JsonTextReader(new StreamReader(stream))) { while (reader.Read()) { if (reader.TokenType == JsonToken.StartObject) { var item = JObject.Load(reader); var operation = item.Property("operation").Value.ToString(); if (operation != null && (operation == "update" || operation == "insert" || operation == "delete")) { var itemData = item.Property("data").First.ToString(Formatting.None); if (fixInfo != null) { itemData = itemData.Replace(fixInfo.DataValueBad, fixInfo.DataValueValid); } T itemAsObject = JsonConvert.DeserializeObject <T>(itemData, UserJsonSerializerSettings); itemAsObject.SetEntityFields(item); switch (operation) { case "insert": case "update": listInsertOrUpdate.Add(itemAsObject); break; case "delete": listDelete.Add(itemAsObject); break; } } else { Logger?.Warn($"{methodName}: Unrecognized operation '{operation}' for {entityName}. Txid = {dawaProcessInfo.Txid}"); } } } } } UpdateEntityHelper.ProcessOperationLists(listDelete, listInsertOrUpdate); int deleteCount = listDelete.Count; int insertUpdateCount = listInsertOrUpdate.Count; if (insertUpdateCount == 0 && deleteCount == 0) { Logger?.Info($"{methodName}: {entityName}: No changes"); var finishtime = EntityStateHelper.SetEntityStateDone(DBContext, entityName, true, dawaProcessInfo.Txid, 0); EntityStateHelper.SetEntityStateHistoryDone(DBContext, entityName, true, starttime, finishtime, txidfra, insertUpdateCount, deleteCount); } else { Logger?.Info($"{methodName}: {entityName}: Inserted and Updated = {insertUpdateCount}, Deleted={deleteCount}"); int totalCount = insertUpdateCount + deleteCount; DBContext.Database.BeginTransaction(); try { DBContext.BulkInsertOrUpdate(listInsertOrUpdate); DBContext.BulkDelete(listDelete); var finishtime = EntityStateHelper.SetEntityStateDone(DBContext, entityName, true, dawaProcessInfo.Txid, totalCount); EntityStateHelper.SetEntityStateHistoryDone(DBContext, entityName, true, starttime, finishtime, txidfra, insertUpdateCount, deleteCount); DBContext.Database.CommitTransaction(); if (UseMSApplicationInsights) { TelemetryHelper.AddTelemetryForEntity(EntityProcessMode.Update, entityName, stopwatch); } } catch (Exception ex) { if (DBContext.Database.CurrentTransaction != null) { DBContext.Database.RollbackTransaction(); } var exception = ex.InnerException ?? ex; var finishtime = EntityStateHelper.SetEntityStateDone(DBContext, entityName, false, txidfra, totalCount, exception.Message); EntityStateHelper.SetEntityStateHistoryDone(DBContext, entityName, false, starttime, finishtime, txidfra, null, null, exception.Message); throw; } } return(listInsertOrUpdate.Count + listDelete.Count); }
/// <summary> /// Main program. /// </summary> /// <param name="args">String arguments for program initialization.</param> public static void Main(string[] args) { Thread.CurrentThread.CurrentCulture = System.Globalization.CultureInfo.InvariantCulture; var settings = Properties.Settings.Default; if (settings.UseMSApplicationInsights) { try { ApplicationInsightInitializer.Initialize(); } catch (Exception ex) { var msg = $"ERROR! Failed to initialize MS Application Insights. Problem: {ex.Message}"; Console.WriteLine(msg); _logger.Error(msg, ex); Environment.Exit((int)ReturnCode.ApplicationInsightInitializeError); } } Helpers.InitializeHelpers.HaltIfNoDBConnection(); Helpers.InitializeHelpers.HaltIfAllreadyRunning(); try { #pragma warning disable CS0436 // Type conflicts with imported type SqlServerTypes.Utilities.LoadNativeAssemblies(AppDomain.CurrentDomain.BaseDirectory); #pragma warning restore CS0436 // Type conflicts with imported type string programAssemblyVersion = Assembly.GetExecutingAssembly().GetName().Version.ToString(); string dataAccessAssemblyVersion = Assembly.GetAssembly(typeof(Adgangsadresse)).GetName().Version.ToString(); var methodName = LoggingUtils.GetMethodName(); Console.WriteLine($"Starting DawaReplication assembly ver. {programAssemblyVersion}; DataAccess ver. {dataAccessAssemblyVersion} ...."); _logger.Info($"{methodName}: Starting DawaReplication assembly ver. {programAssemblyVersion}; DataAccess ver. {dataAccessAssemblyVersion}"); _logger.Info($"{methodName}: Settings = {SettingsHelper.SettingsAsString()}"); _logger.Info($"{methodName}: Time of initialization: {DateTime.Now.ToShortTimeString()}"); var processMode = InitialArgumentHelper.InitializeProcess(args); Helpers.InitializeHelpers.HaltIfDBSchemeNotInitialized(); Console.WriteLine($"Time: {DateTime.Now.ToShortTimeString()}. ProcessMode: {processMode}\n"); var list = TableInfo.GetTableInfoList(AppDomain.CurrentDomain.BaseDirectory + settings.TableInfoFile, false); #pragma warning disable CA2000 // Dispose objects before losing scope var dbContext = new DawaReplicationDBContext(); #pragma warning restore CA2000 // Dispose objects before losing scope if (processMode == EntityProcessMode.Udtraek) { FixDBProblems.FixDbProblems(dbContext); } EntityManager.InitSettings(dbContext, list.Select(x => x.Name).ToList(), settings.DawaApiUri, settings.DawaApiReadTimeoutInSeconds, settings.UdtraekRowsMax, settings.UdtraekBulkSize, settings.DKStedDataStartPos, settings.DKStedBulkSize, settings.DBCommandTimeoutInSeconds, settings.TempDataFolderPath, settings.ActiveFixes, settings.ActiveFixesListFileLocation, settings.UseMSApplicationInsights, settings.JsonSerializerIgnoreNullValue); LocalDataHelper.MakeFolder(settings.TempDataFolderPath); var msg = LoggingUtils.MakeMessage(methodName, " TableInfoFile", list); _logger.Info(msg); var watch = Stopwatch.StartNew(); var dawaProcessInfo = Helpers.InitializeHelpers.MakeDawaProcessInfo(processMode, settings.DawaApiUri, settings.DawaApiReadTimeoutInSeconds, settings.TxidOverride); _logger.Info($"{methodName}: Got latest transaction ID = {dawaProcessInfo.Txid}"); EntityManager.ProcessTables(processMode, dawaProcessInfo); if (dawaProcessInfo.FailedTables.Any() && (processMode == EntityProcessMode.Dagi || processMode == EntityProcessMode.Udtraek)) { for (int i = 1; i <= settings.RetryCount; i++) { msg = $"{methodName}: Failed to process all tables. No of failed tables: {dawaProcessInfo.FailedTables.Count}. Retrying again in {settings.RetryTimerInMinutes * i} minutes."; Console.WriteLine($"\nWARNING. Time: {DateTime.Now.ToShortTimeString()}. {msg}"); _logger.Warn(msg); EntityManager.TableList = new List <string>(dawaProcessInfo.FailedTables); dawaProcessInfo.FailedTables.Clear(); Thread.Sleep(settings.RetryTimerInMinutes * 60 * 1000 * i); EntityManager.ProcessTables(processMode, dawaProcessInfo); if (dawaProcessInfo.FailedTables.Count == 0) { break; } } } if (processMode == EntityProcessMode.Udtraek) { if (settings.RebuildIndicesAfterReplication) { FixDBProblems.RebuildIndices(dbContext); } } if (processMode == EntityProcessMode.Dagi) { LocalDataHelper.CleanUpHelper(settings.EntitystateHistoryDeleteOldNumOfDays, settings.ArchiveLogsAfterDays, settings.DeleteOldArchivesAfterDays, settings.DeleteLogsAfterDays); } dbContext.Dispose(); _logger.Info($"{methodName}: Done. Execution time: {watch.Elapsed}"); _logger.Info($"{methodName}: Time of completion: {DateTime.Now.ToShortTimeString()}"); if (settings.UseMSApplicationInsights) { TelemetryHelper.AddTime(methodName, watch.Elapsed.TotalMinutes, "Minutes"); } if (settings.WaitForUserInput) { Console.WriteLine($"Done: {DateTime.Now.ToShortTimeString()}. Execution time: {watch.Elapsed}" + "\n"); Console.WriteLine("Press any key to exit..."); Console.ReadKey(); } } catch (Exception ex) { var msg = $"ERROR! Unexpected exception. Problem: {ex.Message}"; Console.WriteLine(msg); Console.WriteLine($"Time at which error occured {DateTime.Now.ToShortTimeString()}"); _logger.Error(msg, ex); if (Properties.Settings.Default.WaitForUserInput) { Console.WriteLine("Press any key to exit..."); Console.ReadKey(); } Environment.Exit((int)ReturnCode.UnknownError); } Environment.Exit((int)ReturnCode.Success); }