public ConnectorWorker(ConnectorConfig config, CancellationTokenSource cancellation)
{
PerfomanceAnalyser analyser = null;
if (config.UsePerfomanceAnalyzer)
{
logger.Info("Use perfomance analyser.");
analyser = new PerfomanceAnalyser(new AnalyserOptions()
{
AnalyserFolder = Path.GetDirectoryName(SystemGeneral.GetLogFileName("file"))
});
}
var restClient = new ReportingRestApiClient(new Uri(config?.RestServiceUrl?.TrimEnd('/')), config.RestTimeout);
RuntimeOptions = new RuntimeOptions()
{
Config = config,
SessionHelper = new SessionHelper(),
RestClient = restClient,
Analyser = analyser,
Cancellation = cancellation,
TaskPool = new ManagedTaskPool()
};
RuntimeOptions.TaskPool.Run(RuntimeOptions);
var rendererClient = new ReportingRendererApiClient(new Uri(config?.RendererServiceUrl?.TrimEnd('/')), config.RestTimeout);
var rendererVersion = rendererClient.GetVersion();
logger.Info($"Renderer Version: {rendererVersion}");
config.PackageVersions.Add($"AnalyticsGate Renderer: {rendererVersion}");
}
/// <summary>
/// Method: GetPGRowsAsync
/// Description: It is used to get postgres sync data by page wise.
/// </summary>
/// <param name="id"></param>
/// <param name="page"></param>
/// <param name="isRowsRead"></param>
/// <param name="connectorConfig"></param>
/// <returns></returns>
private async Task <IList <IDictionary <string, object> > > GetChildPGRowsAsync(int id, int?page, bool isRowsRead, string ctid)
{
Task <IList <IDictionary <string, object> > > pgRows = null;
try
{
var ccid = HttpContext.GetClaimValue(ClaimTypes.NameIdentifier);
ConnectorConfig connectorConfig = _connectorRepository.Get <ConnectorConfig>(ccid, id, IsSetConfig: true);
if (!string.IsNullOrEmpty(ccid) && id > 0)
{
ViewBag.connectorConfig = connectorConfig;
//Get the current page
int currentPage = page.HasValue ? (int)page : 1;
var result = _syncRepository.GetChildRecordsByParentForReviewAndDelete(ccid, id, ctid);
pgRows = result;
}
}
catch (Exception ex)
{
if (Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT") == Environments.Development)
{
_logger.LogError(ex.Message, ex);
}
else
{
Console.WriteLine("ERROR: {0}", ex.Message);
}
}
return(await pgRows);
}
public Config Check()
{
ConnectorConfig.Check();
PushConfig.Check();
DatabaseConfig.Check();
return(this);
}
public static IServiceCollection AddConnector(this IServiceCollection services, IConfiguration configuration)
{
var conf = ConnectorConfig.Get(configuration);
services.AddSingleton <ConnectorConfig>(conf);
services.AddSingleton <IConnector, Connector>();
return(services);
}
/// <summary>
/// Use this mode to configure the given target.
/// </summary>
protected override void Configure(ConnectorConfig target, AddressList addresses, int subMode)
{
for (int i = 0; i < 4; i++)
{
var pin = target.Pins[i];
pin.Mode = GetPin(i);
pin.Address = addresses[i];
}
}
/// <summary>
/// Use this mode to configure the given target.
/// </summary>
protected override void Configure(ConnectorConfig target, AddressList addresses, int subMode)
{
for (int i = 0; i < 2; i++)
{
var pin = target.Pins[i];
pin.Mode = GetPin(i);
pin.Address = addresses[i];
}
}
/// <summary>
/// Delete ct-index table query execution
/// </summary>
/// <param name="connectorConfig"></param>
/// <returns></returns>
private bool deleteCtIndextable(ConnectorConfig connectorConfig)
{
StringBuilder sb = new StringBuilder();
if (connectorConfig != null)
{
sb.Append($"DROP TABLE IF EXISTS \"{connectorConfig.destDBSchema}\".\"{connectorConfig.destObjectName}_ctindex\";");
using (ConnectionFactory connectionFactory = new ConnectionFactory(connectorConfig.destDBConfig.syncDefaultDatabaseUrl))
{
Console.WriteLine("Delete ctindex by query{0} ", sb.ToString());
connectionFactory.DbConnection.QueryAsync <dynamic>(sb.ToString());
sb.Clear();
sb = null;
}
}
return(true);
}
private static Uri QlikConnectionCheck(string configJson, string serverUrl)
{
try
{
logger.Debug($"Use Url: {serverUrl}");
if (String.IsNullOrEmpty(serverUrl))
{
logger.Info("No special url set up in the configuration file.");
return(null);
}
dynamic configObject = JObject.Parse(configJson);
var serverUri = new Uri($"{serverUrl}");
if (!serverUrl.EndsWith("/ser"))
{
var uriBuilder = new UriBuilder(serverUrl)
{
Path = "ser"
};
serverUri = uriBuilder.Uri;
}
configObject.connection.serverUri = serverUri;
ConnectorConfig connectorConfig = JsonConvert.DeserializeObject <ConnectorConfig>(configObject.ToString());
ServerCertificateValidation.Connection = connectorConfig.Connection;
var qlikUser = new DomainUser("INTERNAL\\ser_scheduler");
var taskManager = new SessionHelper();
var session = taskManager.GetSession(connectorConfig.Connection, new QlikRequest()
{
QlikUser = qlikUser
});
if (session?.Cookie != null)
{
logger.Info("The connection to Qlik Sense was successful.");
return(serverUri);
}
logger.Warn($"Connection check to qlik with url '{serverUrl}' was not successfully...");
return(null);
}
catch (Exception ex)
{
logger.Error(ex, "The connection check to qlik has an error");
return(null);
}
}
public ConnectorConfig ReadConnectorConfiguration()
{
var config = new ConnectorConfig();
try
{
XmlDocument xmlDoc = new XmlDocument();
xmlDoc.Load("connectorconfig.xml");
XmlNodeList nodeList = xmlDoc.DocumentElement.SelectNodes("/Config");
foreach (XmlNode node in nodeList)
{
config.EndPointIp = node.FirstChild.InnerText;
config.Port = node.LastChild.InnerText;
}
}
catch (Exception ex)
{
Logger.WriteLog($"Exception in : ReadConnectorConfiguration() .InnerException :{ex.InnerException} ", true);
}
return(config);
}
public static string GetParentRecordsForReview(ConnectorConfig connectorConfig, int offSet, int pageSize)
{
tableName = Utilities.RemoveSpecialChars(connectorConfig.destObjectName);
schemaName = connectorConfig.destDBSchema;
destTableName = schemaName + "." + tableName;
destCtIndex = tableName + "_ctindex";;
destCTIndexTableName = schemaName + "." + destCtIndex;
sourceObjectFields = new List <string>();
sourceObjectFields = (from c in connectorConfig.sourceObjectFields
select $"\"{c}\"").ToList();
dataFields = string.Join(",", sourceObjectFields.Select(c => $"\"{tableName}\"" + "." + c).ToArray());
queryBuilder = new StringBuilder();
queryBuilder.Append($" SELECT " + dataFields + "," + destCtIndex + ".myctid," + destCtIndex + ".marked_for_delete FROM \"" + destTableName + "\"");
queryBuilder.Append($" join \"{destCTIndexTableName}\" on " + destTableName + ".ctid=" + destCtIndex + ".myctid ");
queryBuilder.Append($" where " + destCtIndex + ".parentctid is null LIMIT " + pageSize + " OFFSET " + offSet + ";");
return(queryBuilder.ToString());
}
public async Task<IActionResult> Index()
{
List<ConnectorConfig> ConnectorConfigs = null;
ConnectorConfig connectorConfig = null;
try
{
if (ViewBag.CurrentPlan.IsInitialized)
{
if (ViewBag.CurrentPlan.isLicenseAccepted)
{
//Get all connectors from connectors table by ccid
ConnectorConfigs = _connectorsRepository.Get<List<ConnectorConfig>>(HttpContext.GetClaimValue(ClaimTypes.NameIdentifier), null, null);
}
}
}
catch (Exception ex)
{
Console.WriteLine("ERROR: {0}", ex.Message);
}
return View(await Task.FromResult(ConnectorConfigs));
}
/// <summary>
/// This method is used to delete the CTindex table from destination database
/// </summary>
/// <param name="ccid"></param>
/// <param name="connectorId"></param>
/// <param name="IsSingle"></param>
/// <returns></returns>
public bool DeletectindexConfigById(string ccid, int connectorId, bool IsSingle)
{
if (IsSingle)
{
if (connectorId > 0 && !string.IsNullOrEmpty(ccid))
{
Console.WriteLine("Delete ctindex by connectorId-{0} and Resource Id-{1} ", connectorId, ccid);
ConnectorConfig connectorConfig = Get <ConnectorConfig>(ccid, connectorId, null, true);
if (connectorConfig != null)
{
return(deleteCtIndextable(connectorConfig));
}
}
}
else
{
if (connectorId == 0 && !string.IsNullOrEmpty(ccid))
{
Console.WriteLine("Delete ctindex while user deprovision using Resource Id-{0} ", ccid);
List <ConnectorConfig> connectorConfigs = Get <List <ConnectorConfig> >(ccid, 0, null, true);
if (connectorConfigs != null && connectorConfigs.Count > 0)
{
for (int i = 0; i < connectorConfigs.Count; i++)
{
Console.WriteLine("Delete ctindex by connectorId-{0} and Resource Id-{1} ", connectorConfigs[i].connectorId, ccid);
deleteCtIndextable(connectorConfigs[i]);
if (i == connectorConfigs.Count - 1)
{
return(true);
}
}
}
}
}
return(false);
}
/// <summary>
/// Use this mode to configure the given target.
/// </summary>
protected override void Configure(ConnectorConfig target, AddressList addresses, int subMode)
{
}
/// <summary>
/// Method: AddEditExtensionsConfig
/// Description: It is used to add/update connector in connectors table
/// </summary>
/// <param name="connectorConfig"></param>
public void AddEditExtensionsConfig(ConnectorConfig connectorConfig)
{
using (var transaction = _context.Database.BeginTransaction())
{
Connectors entity;
DeDupSettings dedupSettings = null;
bool isNew = false;
int isSyncTableExist = 0;
int isSyncGoldenTableExist = 0;
if (connectorConfig.connectorId.HasValue)
{
entity = Find(connectorConfig.ccid, connectorConfig.connectorId);
if (entity != null)
{
//delete sync process if it is in progress while updating connector
if (!string.IsNullOrEmpty(entity.job_id) ||
entity.schedule_type != ScheduleType.MANUAL_SYNC)
{
//delete old jobs
JobScheduler.Instance.DeleteJob(ccid: entity.ccid, connectorId: entity.connector_id, jobId: entity.job_id, scheduleType: entity.schedule_type);
//reset job id once deleted
entity.job_id = string.Empty;
if (entity.sync_status == 1)
{
//set status interrupted
entity.sync_status = 10;
}
else if (entity.sync_status.HasValue)
{
//reset sync status
entity.sync_status = 0;
}
}
else if (entity.sync_status.HasValue)
{
//reset sync status
entity.sync_status = 0;
}
if (entity.sync_status != 2 && entity.sync_status != 10)
{
entity.sync_count = null;
entity.unique_records_count = null;
entity.sync_updated_count = null;
entity.total_records_count = null;
entity.sync_log_json = null;
entity.sync_started_at = null;
entity.sync_ended_at = null;
entity.last_sync_at = null;
entity.last_sync_status = null;
}
dedupSettings = entity.DeDupSetting;
//check table exist or not. if not then create table and sync
isSyncTableExist = SyncRepository.SyncTableIsExist(connectorConfig);
isSyncGoldenTableExist = SyncRepository.SyncGoldenTableIsExist(connectorConfig);
}
}
else
{
isNew = true;
entity = new Connectors()
{
ccid = connectorConfig.ccid
};
//Set next connector id
entity.connector_id = GetMaxID(entity.ccid, ref dedupSettings) + 1;
}
entity.connector_name = connectorConfig.connectorName;
if (isNew)
{
entity.sync_src = connectorConfig.dataSource;
entity.src_object_name = connectorConfig.sourceObjectName;
//commented by Kathir on 12-Aug-2020
// entity.src_object_fields_json = JsonConvert.SerializeObject(connectorConfig.sourceObjectFields);
if (entity.sync_src == DataSource.Heroku_Postgres ||
entity.sync_src == DataSource.Azure_Postgres ||
entity.sync_src == DataSource.AWS_Postgres ||
entity.sync_src == DataSource.Azure_SQL)
{
entity.src_config_json = JsonConvert.SerializeObject(connectorConfig.dbConfig);
}
if (connectorConfig.dedup_type != DedupType.Full_Dedup)
{
entity.connector_type = (ConnectorType)connectorConfig.syncDestination;
entity.dest_object_name = connectorConfig.destObjectName;
}
else
{
if (connectorConfig.dataSource == DataSource.Heroku_Postgres)
{
entity.connector_type = ConnectorType.Heroku_Postgres;
}
if (connectorConfig.dataSource == DataSource.AWS_Postgres)
{
entity.connector_type = ConnectorType.AWS_Postgres;
}
if (connectorConfig.dataSource == DataSource.Azure_Postgres)
{
entity.connector_type = ConnectorType.Azure_Postgres;
}
entity.dest_object_name = string.Empty;
}
entity.src_schema = connectorConfig.dbSchema;
entity.dedup_type = connectorConfig.dedup_type;
entity.dedup_source_type = connectorConfig.dedupSourceType;
if ((connectorConfig.dedupSourceType == SourceType.Copy_Source_data_to_Destination_and_Remove_Duplicates_from_Destination || connectorConfig.dedupSourceType == SourceType.Merge_Table_A_Data_to_Table_B_and_Remove_Duplicates_from_Table_B))
{
entity.compare_object_fields = JsonConvert.SerializeObject(connectorConfig.compareObjectFieldsMapping);
entity.compare_config_json = JsonConvert.SerializeObject(connectorConfig.dbConfig_compare);
}
//if (IsMultipleConfigSupported)
{
if (entity.connector_type == ConnectorType.Heroku_Postgres ||
entity.connector_type == ConnectorType.Azure_Postgres ||
entity.connector_type == ConnectorType.AWS_Postgres ||
entity.connector_type == ConnectorType.Azure_SQL)
{
entity.dest_config_json = JsonConvert.SerializeObject(connectorConfig.destDBConfig);
}
}
//set global db setting
if (dedupSettings != null)
{
//save sync database url
if (connectorConfig.dbConfig != null &&
!string.IsNullOrEmpty(connectorConfig.dbConfig.syncDefaultDatabaseUrl))
{
if (string.IsNullOrEmpty(dedupSettings.database_config_json))
{
dedupSettings.database_config_json = JsonConvert.SerializeObject((new List <DatabaseConfig>()
{
connectorConfig.dbConfig
}));
_context.Entry(dedupSettings).State = EntityState.Modified;
}
else
{
var dbConfigs = dedupSettings.ToModel <List <DatabaseConfig> >();
if (dbConfigs != null && dbConfigs.FirstOrDefault(p => p.databaseType == connectorConfig.dbConfig.databaseType) == null)
{
dbConfigs.Add(connectorConfig.dbConfig);
dedupSettings.database_config_json = JsonConvert.SerializeObject(dbConfigs);
_context.Entry(dedupSettings).State = EntityState.Modified;
}
}
}
}
else
{
dedupSettings = new DeDupSettings()
{
ccid = connectorConfig.ccid
};
//save sync database url
if (connectorConfig.dbConfig != null && !string.IsNullOrEmpty(connectorConfig.dbConfig.syncDefaultDatabaseUrl))
{
dedupSettings.database_config_json = JsonConvert.SerializeObject((new List <DatabaseConfig>()
{
connectorConfig.dbConfig
}));
}
//add dedupsetting
_context.Entry(dedupSettings).State = EntityState.Added;
}
entity.unique_records_count = null;
entity.sync_updated_count = null;
entity.sync_count = null;
entity.total_records_count = null;
}
entity.schedule_type = connectorConfig.scheduleType;
entity.src_new_record_filter = connectorConfig.srcNewRecordFilter;
entity.src_update_record_filter = connectorConfig.srcUpdateRecordFilter;
entity.two_way_sync_priority = connectorConfig.twoWaySyncPriority;
entity.dest_schema = connectorConfig.destDBSchema;
entity.custom_schedule_in_minutes = connectorConfig.customScheduleInMinutes;
//Added by Kathir on 20-8-2020
entity.dedup_method = connectorConfig.dedup_method;
//if (connectorConfig.dedup_method == SimilarityType.Fuzzy_Compare)
//{
// entity.fuzzy_ratio = connectorConfig.fuzzy_ratio;
//}
entity.fuzzy_ratio = connectorConfig.fuzzy_ratio;
entity.review_before_delete = connectorConfig.review_before_delete;
entity.backup_before_delete = connectorConfig.backup_before_delete;
if (connectorConfig.dedup_type == DedupType.Full_Dedup)
{
entity.simulation_count = -1;
}
else
{
entity.simulation_count = connectorConfig.simulation_count;
}
if (string.IsNullOrEmpty(entity.src_object_name))
{
entity.src_object_name = connectorConfig.sourceObjectName;
}
if ((connectorConfig.dedupSourceType == SourceType.Copy_Source_data_to_Destination_and_Remove_Duplicates_from_Destination || connectorConfig.dedupSourceType == SourceType.Merge_Table_A_Data_to_Table_B_and_Remove_Duplicates_from_Table_B))
{
//entity.compare_object_fields = JsonConvert.SerializeObject(connectorConfig.compareObjectFieldsMapping);
entity.compare_config_json = JsonConvert.SerializeObject(connectorConfig.dbConfig_compare);
}
//commented by Kathir on 12-Aug-2020
//if (string.IsNullOrEmpty(entity.src_object_fields_json))
//{
// entity.src_object_fields_json = JsonConvert.SerializeObject(connectorConfig.sourceObjectFields);
//}
if (string.IsNullOrEmpty(entity.dest_object_name))
{
if (connectorConfig.dedup_type == DedupType.Full_Dedup && connectorConfig.dedupSourceType == SourceType.Remove_Duplicates_from_a_Single_Table)
{
entity.dest_object_name = connectorConfig.sourceObjectName;
}
else if (connectorConfig.dedup_type == DedupType.Full_Dedup && (connectorConfig.dedupSourceType == SourceType.Copy_Source_data_to_Destination_and_Remove_Duplicates_from_Destination || connectorConfig.dedupSourceType == SourceType.Merge_Table_A_Data_to_Table_B_and_Remove_Duplicates_from_Table_B))
{
entity.dest_object_name = (connectorConfig.dbConfig_compare.table_type == SelectedTableType.Create_New_Table ? connectorConfig.dbConfig_compare.new_table_name : connectorConfig.dbConfig_compare.object_name);
}
else
{
entity.dest_object_name = connectorConfig.destObjectName;
}
}
//Assigned source object fields to compare object field json
entity.compare_object_fields = JsonConvert.SerializeObject(connectorConfig.sourceObjectFields);
//if (connectorConfig.dedupSourceType == SourceType.Remove_Duplicates_from_a_Single_Table)
//{
// if (connectorConfig.sourceObjectFields != null && connectorConfig.sourceObjectFields.Count > 0)
// {
// foreach (string sel_fields in connectorConfig.sourceObjectFields)
// {
// connectorConfig.compareObjectFieldsMapping.Add(new KeyValuePair<string, string>(sel_fields, sel_fields));
// }
// }
// entity.compare_object_fields = JsonConvert.SerializeObject(connectorConfig.sourceObjectFields);
//}
if (connectorConfig.dedup_type == DedupType.Full_Dedup)
{
entity.dest_object_name = (connectorConfig.dedupSourceType == SourceType.Remove_Duplicates_from_a_Single_Table ? connectorConfig.sourceObjectName : (connectorConfig.dbConfig_compare.table_type == SelectedTableType.Create_New_Table ? connectorConfig.dbConfig_compare.new_table_name : connectorConfig.dbConfig_compare.object_name));
connectorConfig.destDBConfig.syncDefaultDatabaseUrl = (connectorConfig.dedupSourceType == SourceType.Remove_Duplicates_from_a_Single_Table ? connectorConfig.dbConfig.syncDefaultDatabaseUrl : connectorConfig.dbConfig_compare.syncDefaultDatabaseUrl);
connectorConfig.destDBConfig.databaseType = (connectorConfig.dedupSourceType == SourceType.Remove_Duplicates_from_a_Single_Table ? connectorConfig.dbConfig.databaseType : connectorConfig.dbConfig_compare.databaseType);
connectorConfig.destDBConfig.dataSource = (connectorConfig.dedupSourceType == SourceType.Remove_Duplicates_from_a_Single_Table ? connectorConfig.dbConfig.dataSource : connectorConfig.dbConfig_compare.dataSource);
entity.dest_config_json = JsonConvert.SerializeObject(connectorConfig.destDBConfig);
entity.dest_schema = (connectorConfig.dedupSourceType == SourceType.Remove_Duplicates_from_a_Single_Table ? connectorConfig.dbSchema : connectorConfig.dbConfig_compare.db_schema);
}
//save extension setting
_context.Entry(entity).State = isNew ? EntityState.Added : EntityState.Modified;
_context.SaveChanges();
//if (isSyncTableExist == 0 && connectorConfig.dedup_type != DedupType.Full_Dedup)
//{
// var errorMessage = string.Empty;
// if (connectorConfig.syncDestination == ConnectorType.Heroku_Postgres
// || connectorConfig.syncDestination == ConnectorType.Azure_Postgres
// || connectorConfig.syncDestination == ConnectorType.AWS_Postgres)
// {
// var dbStatus = SyncRepository.CreatePostgresTable(connectorConfig, out errorMessage);
// if (dbStatus == -1)
// {
// transaction.Rollback();
// throw new Exception(message: (string.IsNullOrEmpty(errorMessage) ? "Not able to create postgres table." : errorMessage)) { };
// }
// else if (dbStatus == 2)
// {
// transaction.Rollback();
// throw new Exception(message: "The " + connectorConfig.destObjectName + " postgres table already exists.") { };
// }
// }
//}
//if (isSyncGoldenTableExist == 0 && connectorConfig.dbConfig_compare.table_type == SelectedTableType.Create_New_Table)
//{
// var errorMessage = string.Empty;
// if (connectorConfig.dbConfig_compare.dataSource == DataSource.Heroku_Postgres
// || connectorConfig.dbConfig_compare.dataSource == DataSource.Azure_Postgres
// || connectorConfig.dbConfig_compare.dataSource == DataSource.AWS_Postgres)
// {
// var dbStatus = SyncRepository.CreatePostgresGoldenTable(connectorConfig, out errorMessage);
// if (dbStatus == -1)
// {
// transaction.Rollback();
// throw new Exception(message: (string.IsNullOrEmpty(errorMessage) ? "Not able to create postgres table." : errorMessage)) { };
// }
// else if (dbStatus == 2)
// {
// transaction.Rollback();
// throw new Exception(message: "The " + connectorConfig.dbConfig_compare.new_table_name + " postgres table already exists.") { };
// }
// }
//}
transaction.Commit();
//Schedule job if scheduletype is not manual
if (entity.schedule_type != ScheduleType.MANUAL_SYNC)
{
if (entity.schedule_type == ScheduleType.CUSTOM)
{
JobScheduler.Instance.ScheduleJob(entity.ccid, entity.connector_id, entity.connector_type, entity.schedule_type, (entity.custom_schedule_in_minutes.HasValue ? entity.custom_schedule_in_minutes.Value : 1200));
}
else
{
JobScheduler.Instance.ScheduleJob(entity.ccid, entity.connector_id, entity.connector_type, entity.schedule_type);
}
}
//});
}
}
public Connector(ConnectorConfig config)
{
conf = config;
}
public static T ToModel <T>(this List <Connectors> connectors, bool isSetConfig = true) where T : class
{
if (typeof(T) == typeof(List <ConnectorConfig>))
{
List <ConnectorConfig> connectorConfigs = null;
if (connectors != null)
{
DeDupSettings dedupSettings = isSetConfig ? connectors.FirstOrDefault().DeDupSetting : null;
connectorConfigs = connectors.Select(c =>
{
ConnectorConfig conConfig;
conConfig = new ConnectorConfig()
{
ccid = c.ccid,
connectorId = c.connector_id,
connectorName = c.connector_name,
sourceObjectName = c.src_object_name,
destObjectName = c.dest_object_name,
scheduleType = c.schedule_type,
srcNewRecordFilter = c.src_new_record_filter,
srcUpdateRecordFilter = c.src_update_record_filter,
twoWaySyncPriority = c.two_way_sync_priority,
syncDestination = c.connector_type,
syncCount = c.sync_count,
syncStatus = c.sync_status,
dedup_type = c.dedup_type,
jobId = c.job_id,
syncStartedAt = c.sync_started_at,
syncEndedAt = c.sync_ended_at,
lastSyncAt = c.last_sync_at,
lastSyncStatus = c.last_sync_status,
dbSchema = c.src_schema,
dataSource = c.sync_src,
destDBSchema = c.dest_schema,
customScheduleInMinutes = c.custom_schedule_in_minutes,
dedupSourceType = c.dedup_source_type,
dedup_method = c.dedup_method,
review_before_delete = c.review_before_delete,
backup_before_delete = c.backup_before_delete,
simulation_count = c.simulation_count,
total_records_count = c.total_records_count,
deduped_count = c.deduped_count,
fuzzy_ratio = c.fuzzy_ratio * 100,
unique_records_count = c.unique_records_count
};
//if (!string.IsNullOrEmpty(c.src_object_fields_json))
//{
// conConfig.sourceObjectFields = JsonConvert.DeserializeObject<List<string>>(c.compare_object_fields);
//}
if (!string.IsNullOrEmpty(c.compare_config_json) && (c.dedup_source_type == SourceType.Copy_Source_data_to_Destination_and_Remove_Duplicates_from_Destination || c.dedup_source_type == SourceType.Merge_Table_A_Data_to_Table_B_and_Remove_Duplicates_from_Table_B))
{
conConfig.dbConfig_compare = JsonConvert.DeserializeObject <DatabaseConfig>(c.compare_config_json);
}
if (!string.IsNullOrEmpty(c.compare_object_fields))
{
conConfig.compareObjectFieldsMapping = JsonConvert.DeserializeObject <List <string> >(c.compare_object_fields);
if (c.dedup_source_type == SourceType.Remove_Duplicates_from_a_Single_Table || conConfig.compareObjectFieldsMapping != null)
{
conConfig.sourceObjectFields = conConfig.compareObjectFieldsMapping;
conConfig.dbConfig_compare.compareObjectFields = conConfig.compareObjectFieldsMapping;
}
}
//if (!string.IsNullOrEmpty(c.sync_log_json))
//{
// conConfig.connectorLogs = new ConnectorLogs()
// {
// sync_started_at = conConfig.syncStartedAt,
// sync_ended_at = conConfig.syncEndedAt,
// sync_count = conConfig.syncCount,
// sync_logs= JsonConvert.DeserializeObject<List<string>>(c.sync_log_json)
// };
//}
if (conConfig != null)
{
if (conConfig.syncDestination == ConnectorType.Heroku_Postgres ||
conConfig.syncDestination == ConnectorType.Azure_Postgres ||
conConfig.syncDestination == ConnectorType.AWS_Postgres ||
conConfig.syncDestination == ConnectorType.Azure_SQL)
{
DatabaseType databaseType;
if (conConfig.syncDestination == ConnectorType.Azure_SQL)
{
databaseType = DatabaseType.Azure_SQL;
}
else if (conConfig.syncDestination == ConnectorType.Azure_Postgres)
{
databaseType = DatabaseType.Azure_Postgres;
}
else if (conConfig.syncDestination == ConnectorType.AWS_Postgres)
{
databaseType = DatabaseType.AWS_Postgres;
}
else
{
databaseType = DatabaseType.Heroku_Postgres;
}
if (!string.IsNullOrEmpty(c.dest_config_json))
{
conConfig.destDBConfig = JsonConvert.DeserializeObject <DatabaseConfig>(c.dest_config_json);
if (conConfig.destDBConfig.databaseType == DatabaseType.None)
{
conConfig.destDBConfig.databaseType = databaseType;
}
}
else if (c.DeDupSetting != null && !string.IsNullOrEmpty(c.DeDupSetting.database_config_json))
{
conConfig.destDBConfig = c.DeDupSetting.ToModel <DatabaseConfig>(databaseType: databaseType);
}
}
if (string.IsNullOrEmpty(conConfig.dbSchema))
{
if ((conConfig.dataSource == DataSource.Heroku_Postgres ||
conConfig.dataSource == DataSource.Azure_Postgres ||
conConfig.dataSource == DataSource.AWS_Postgres))
{
conConfig.dbSchema = Constants.POSTGRES_DEFAULT_SCHEMA;
}
else if (conConfig.dataSource == DataSource.Azure_SQL)
{
conConfig.dbSchema = Constants.MSSQL_DEFAULT_SCHEMA;
}
}
if (string.IsNullOrEmpty(conConfig.destDBSchema))
{
if ((conConfig.syncDestination == ConnectorType.Heroku_Postgres ||
conConfig.syncDestination == ConnectorType.Azure_Postgres ||
conConfig.syncDestination == ConnectorType.AWS_Postgres))
{
conConfig.destDBSchema = Constants.POSTGRES_DEFAULT_SCHEMA;
}
else if (conConfig.syncDestination == ConnectorType.Azure_SQL)
{
conConfig.destDBSchema = Constants.MSSQL_DEFAULT_SCHEMA;
}
}
if (isSetConfig)
{
if (conConfig.dataSource == DataSource.Heroku_Postgres ||
conConfig.dataSource == DataSource.Azure_Postgres ||
conConfig.dataSource == DataSource.AWS_Postgres ||
conConfig.dataSource == DataSource.Azure_SQL)
{
DatabaseType databaseType;
if (conConfig.syncDestination == ConnectorType.Azure_SQL)
{
databaseType = DatabaseType.Azure_SQL;
}
else if (conConfig.syncDestination == ConnectorType.Azure_Postgres)
{
databaseType = DatabaseType.Azure_Postgres;
}
else if (conConfig.syncDestination == ConnectorType.AWS_Postgres)
{
databaseType = DatabaseType.AWS_Postgres;
}
else
{
databaseType = DatabaseType.Heroku_Postgres;
}
if (!string.IsNullOrEmpty(c.compare_config_json) && (c.dedup_source_type == SourceType.Copy_Source_data_to_Destination_and_Remove_Duplicates_from_Destination || c.dedup_source_type == SourceType.Merge_Table_A_Data_to_Table_B_and_Remove_Duplicates_from_Table_B))
{
if (c.compare_config_json.StartsWith("["))
{
conConfig.multipleDBConfigs.AddRange(JsonConvert.DeserializeObject <List <DatabaseConfig> >(c.compare_config_json));
//when multiple source assigned then we need to use th below line
//conConfig.dbConfig_compare = JsonConvert.DeserializeObject<DatabaseConfig>(connector.compare_config_json);
}
else
{
conConfig.dbConfig_compare = JsonConvert.DeserializeObject <DatabaseConfig>(c.compare_config_json);
conConfig.multipleDBConfigs.Add(conConfig.dbConfig_compare);
}
}
if (!string.IsNullOrEmpty(c.src_config_json))
{
conConfig.dbConfig = JsonConvert.DeserializeObject <DatabaseConfig>(c.src_config_json);
if (conConfig.dbConfig.databaseType == DatabaseType.None)
{
conConfig.dbConfig.databaseType = databaseType;
}
conConfig.dbConfig.db_schema = conConfig.dbSchema;
conConfig.dbConfig.object_name = conConfig.sourceObjectName;
conConfig.multipleDBConfigs.Add(conConfig.dbConfig);
}
else if (dedupSettings != null && !string.IsNullOrEmpty(dedupSettings.database_config_json))
{
conConfig.dbConfig = dedupSettings.ToModel <DatabaseConfig>(databaseType: databaseType);
conConfig.dbConfig.db_schema = conConfig.dbSchema;
conConfig.dbConfig.object_name = conConfig.sourceObjectName;
conConfig.multipleDBConfigs.Add(conConfig.dbConfig);
}
}
if (conConfig.syncDestination == ConnectorType.Heroku_Postgres ||
conConfig.syncDestination == ConnectorType.Azure_Postgres ||
conConfig.syncDestination == ConnectorType.AWS_Postgres ||
conConfig.syncDestination == ConnectorType.Azure_SQL)
{
DatabaseType databaseType;
if (conConfig.syncDestination == ConnectorType.Azure_SQL)
{
databaseType = DatabaseType.Azure_SQL;
}
else if (conConfig.syncDestination == ConnectorType.Azure_Postgres)
{
databaseType = DatabaseType.Azure_Postgres;
}
else if (conConfig.syncDestination == ConnectorType.AWS_Postgres)
{
databaseType = DatabaseType.AWS_Postgres;
}
else
{
databaseType = DatabaseType.Heroku_Postgres;
}
if (!string.IsNullOrEmpty(c.dest_config_json))
{
conConfig.destDBConfig = JsonConvert.DeserializeObject <DatabaseConfig>(c.dest_config_json);
if (conConfig.destDBConfig.databaseType == DatabaseType.None)
{
conConfig.destDBConfig.databaseType = databaseType;
}
}
else if (dedupSettings != null && !string.IsNullOrEmpty(dedupSettings.database_config_json))
{
conConfig.destDBConfig = dedupSettings.ToModel <DatabaseConfig>(databaseType: databaseType);
}
}
}
conConfig.child_record_count = SyncRepository.GetChildRecordsCount(conConfig);
}
return(conConfig);
}).ToList();
}
return(connectorConfigs as T);
}
else if (typeof(T) == typeof(List <ConnectorLogs>))
{
List <ConnectorLogs> connectorLogs = null;
if (connectors != null)
{
connectorLogs = connectors.Select(c =>
{
return(new ConnectorLogs()
{
sync_connector_name = c.connector_name,
sync_started_at = c.sync_started_at,
sync_ended_at = c.sync_ended_at,
sync_count = c.sync_count,
last_sync_at = c.last_sync_at,
last_sync_status = c.last_sync_status,
sync_status = c.sync_status,
sync_logs = Utilities.GetJsonPropertyValueByKeyPath <List <string> >(c.sync_log_json, "")
});
}).ToList();
}
return(connectorLogs as T);
}
return(null);
}
/// <summary>
/// Method: InvokeAsync
/// Description: It is used to get postgres sync data by page wise.
/// </summary>
/// <param name="id"></param>
/// <param name="page"></param>
/// <param name="isRowsRead"></param>
/// <param name="connectorConfig"></param>
/// <returns></returns>
public async Task <IViewComponentResult> InvokeAsync(int id, int?page, bool isRowsRead = true, ConnectorConfig connectorConfig = null, string ctid = null)
{
//Get postgres sync data
return(View(await GetPGRowsAsync(id, page, isRowsRead, connectorConfig, ctid)));
}
/// <summary>
/// Method: GetPGRowsAsync
/// Description: It is used to get postgres sync data by page wise.
/// </summary>
/// <param name="id"></param>
/// <param name="page"></param>
/// <param name="isRowsRead"></param>
/// <param name="connectorConfig"></param>
/// <returns></returns>
private async Task <IList <IDictionary <string, object> > > GetPGRowsAsync(int id, int?page, bool isRowsRead, ConnectorConfig connectorConfig, string ctid)
{
IEnumerable <dynamic> etDataRows = null;
IList <IDictionary <string, object> > pgRows = new List <IDictionary <string, object> >();
try
{
var ccid = HttpContext.GetClaimValue(ClaimTypes.NameIdentifier);
if (!string.IsNullOrEmpty(ccid) && id > 0)
{
//Get ConnectorConfig
if (connectorConfig == null)
{
connectorConfig = _connectorRepository.Get <ConnectorConfig>(ccid, id, IsSetConfig: true);
}
//Get the current page
int currentPage = page.HasValue ? (int)page : 1;
int totalRecords = 0;
int marked_for_delete_count = 0;
if (connectorConfig != null)
{
ViewBag.connectorConfig = connectorConfig;
//Get total record
//totalRecords = SyncRepository.GetPGRecordCountByName(connectorConfig);
totalRecords = _syncRepository.GetCTIndexTableCount(ccid, id);
//marked_for_delete_count = _syncRepository.GetMarkedForDeleteCount(ccid, id);
ViewBag.marked_for_delete_count = marked_for_delete_count;
ViewData["count_" + id.ToString()] = totalRecords;
//Read sync data if isRowsRead flag is true
if (String.IsNullOrEmpty(ctid))
{
if (isRowsRead && totalRecords > 0)
{
//Get page settings
ViewData[id.ToString()] = _paginatedMetaService.GetMetaData(totalRecords, currentPage, PAGESIZE);
//Get sync data by pageNo, ccid, connectorId and limit
//etDataRows = SyncRepository.FindPGRowsByPageNo(connectorConfig, currentPage, PAGESIZE);
pgRows = await _syncRepository.GetParentRecordsPageByPageForReviewAndDelete(connectorConfig.ccid, id, currentPage, PAGESIZE);
//pgRows = result.Cast<IDictionary<string, object>>().ToList();
//pgRows = SyncRepository.GetParentRecordsPageByPageForReviewAndDelete(connectorConfig, currentPage, PAGESIZE);
}
}
else
{
pgRows = await _syncRepository.GetChildRecordsByParentForReviewAndDelete(connectorConfig.ccid, id, ctid);
}
}
}
}
catch (Exception ex)
{
if (Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT") == Environments.Development)
{
_logger.LogError(ex.Message, ex);
}
else
{
Console.WriteLine("ERROR: {0}", ex.Message);
}
}
return(pgRows);
}
/// <summary>
/// Connect this control to the given configuration
/// </summary>
public void Connect(ConnectorConfig config)
{
}
/// <summary>
/// Connect this control to the given configuration
/// </summary>
public void Connect(ConnectorConfig config)
{
}
/// <summary>
/// Use this mode to create a configuration using the
/// given addresses for the given connector.
/// </summary>
public ConnectorConfig CreateConfig(Connector connector, AddressList addresses, int subMode)
{
var pinConfigs = new PinConfig[PinCount];
var pinOffset = GetPinOffset(subMode);
for (int i = 0; i < PinCount; i++)
{
var pin = i + 1 + pinOffset;
if (connector == Connector.Second)
pin += 8;
pinConfigs[i] = new PinConfig(pin);
}
var result = new ConnectorConfig(pinConfigs);
Configure(result, addresses, subMode);
return result;
}
/// <summary>
/// Method: GetPGRowsAsync
/// Description: It is used to get postgres sync data by page wise.
/// </summary>
/// <param name="id"></param>
/// <param name="page"></param>
/// <param name="isRowsRead"></param>
/// <param name="connectorConfig"></param>
/// <returns></returns>
private Task <IEnumerable <dynamic> > GetSQLRowsAsync(int id, int?page, bool isRowsRead, ConnectorConfig connectorConfig)
{
IEnumerable <dynamic> etDataRows = null;
try
{
var resourceId = HttpContext.GetClaimValue(ClaimTypes.NameIdentifier);
if (!string.IsNullOrEmpty(resourceId) && id > 0)
{
//Get ConnectorConfig
if (connectorConfig == null)
{
connectorConfig = _connectorRepository.Get <ConnectorConfig>(resourceId, id, IsSetConfig: true);
}
//Get the current page
int currentPage = page.HasValue ? (int)page : 1;
int totalRecords = 0;
if (connectorConfig != null)
{
ViewBag.connectorConfig = connectorConfig;
//Get total record
totalRecords = 0;// SyncRepository.GetSqlRecordCountByName(connectorConfig);
if (totalRecords > 0)
{
ViewData["count_" + id.ToString()] = totalRecords;
//Read sync data if isRowsRead flag is true
if (isRowsRead)
{
//Get page settings
// ViewData[id.ToString()] = _paginatedMetaService.GetMetaData(totalRecords, currentPage, PAGESIZE);
//Get sync data by pageNo, ccid, connectorId and limit
// etDataRows = SyncRepository.FindSqlRowsByPageNo(connectorConfig, currentPage, PAGESIZE);
}
}
}
}
}
catch (Exception ex)
{
if (Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT") == Environments.Development)
{
_logger.LogError(ex.Message, ex);
}
else
{
Console.WriteLine("ERROR: {0}", ex.Message);
}
}
return(Task.FromResult(etDataRows));
}
/// <summary>
/// Use this mode to configure the given target.
/// </summary>
protected override void Configure(ConnectorConfig target, AddressList addresses, int subMode)
{
}
private void StartupConnector(Uri serverUri)
{
// Find the right server uri
var serverUriObj = ConfigObject?.connection?.serverUri;
if (serverUriObj != null && serverUri.OriginalString != serverUriObj.Value)
{
logger.Warn($"Write the correct server uri '{serverUri?.AbsoluteUri?.Trim('/')}' in the config file.");
logger.Warn(">>> Run in alternative mode. <<<");
}
ConfigObject.connection.serverUri = serverUri;
ConnectorConfig config = JsonConvert.DeserializeObject <ConnectorConfig>(ConfigObject.ToString());
if (config.StopTimeout < 5)
{
config.StopTimeout = 5;
}
//Start Rest Service
var rootContentFolder = HelperUtilities.GetFullPathFromApp(config.WorkingDir);
if (!config.UseExternalRestService)
{
var arguments = new List <string>()
{
$"--Mode=NoService", $"--Urls={config.RestServiceUrl}", $"--contentRoot={rootContentFolder}"
};
StartRestServer(arguments.ToArray());
}
var mainVersion = ConnectorVersion.GetMainVersion();
config.PackageVersions.Add($"AnalyticsGate Version: {mainVersion}");
config.PackageVersion = mainVersion;
logger.Info($"Connector Version: {mainVersion}");
config.ExternalPackageJson = ConnectorVersion.GetExternalPackageJson();
var packages = JArray.Parse(config.ExternalPackageJson);
foreach (var package in packages)
{
logger.Info($"Package: {JsonConvert.SerializeObject(package)}");
}
logger.Debug($"Plattfom: {config.OS}");
logger.Debug($"Architecture: {config.Architecture}");
logger.Debug($"Framework: {config.Framework}");
logger.Debug("Service running...");
logger.Debug($"Start Service on Port \"{config.BindingPort}\" with Host \"{config.BindingHost}");
logger.Debug($"Server start...");
// Wait for slow IO perfomance
if (config.StartRestTimeout < 0)
{
config.StartRestTimeout = 0;
}
if (config.StartRestTimeout > 120)
{
config.StartRestTimeout = 120;
}
logger.Debug($"Connector start timeout is '{config.StartRestTimeout}' seconds...");
Thread.Sleep(config.StartRestTimeout * 1000);
worker = new ConnectorWorker(config, cts);
worker.CleanupOldFiles();
worker.RestServiceHealthCheck();
logger.Info($"Start GRPC listening on port '{config.BindingPort}' on Host '{config.BindingHost}'...");
server = new Server()
{
Services = { Connector.BindService(worker) },
Ports = { new ServerPort(config.BindingHost, config.BindingPort, ServerCredentials.Insecure) },
};
server.Start();
logger.Info($"The GRPC server is ready...");
}
/// <summary>
/// Description:This method is used to change the dedup type from Simulate to Full-Dedup.
/// User can check 'N' number of time to simulate to verify the data before DeDup.
/// Once they satisfied then it will change it to Full Dedup.
/// Once its change to Full Dedup then we can not revert back to Simulate mode
/// </summary>
/// <param name="connectorConfig"></param>
/// <param name="dbConfig"></param>
/// <param name="ccid"></param>
public string FinalizedForDedup_Repository(ConnectorConfig connectorConfig)
{
try
{
if (connectorConfig == null && (connectorConfig != null && (string.IsNullOrEmpty(connectorConfig.ccid) || !connectorConfig.connectorId.HasValue)))
{
return("");
}
if ((connectorConfig.syncDestination == ConnectorType.Heroku_Postgres ||
connectorConfig.syncDestination == ConnectorType.Azure_Postgres ||
connectorConfig.syncDestination == ConnectorType.AWS_Postgres ||
connectorConfig.syncDestination == ConnectorType.Azure_SQL) &&
(connectorConfig.dbConfig == null ||
(connectorConfig.dbConfig != null && string.IsNullOrEmpty(connectorConfig.dbConfig.syncDefaultDatabaseUrl))))
{
return("");
}
var entity = Find(connectorConfig.ccid, connectorConfig.connectorId);
//Drop the ctindex table & backup table before changing to Real Dedup mode
StringBuilder sb = new StringBuilder();
sb.Append($"DROP TABLE IF EXISTS \"{entity.dest_schema}\".\"{entity.dest_object_name}_ctindex\";");
sb.Append($"DROP TABLE IF EXISTS \"{entity.dest_schema}\".\"{entity.dest_object_name}_deleted\";");
using (ConnectionFactory connectionFactory = new ConnectionFactory(connectorConfig.destDBConfig.syncDefaultDatabaseUrl))
{
try
{
connectionFactory.DbConnection.ExecuteScalarAsync <int>(sb.ToString()).ConfigureAwait(false);
}
catch (Exception ex)
{
Console.WriteLine("Error:{0}", ex.Message);
return(ex.Message);
}
}
//Assign value to destination object
entity.dest_object_name = (connectorConfig.dedupSourceType == SourceType.Remove_Duplicates_from_a_Single_Table ? connectorConfig.sourceObjectName
: (connectorConfig.dbConfig_compare.table_type == SelectedTableType.Create_New_Table ? connectorConfig.dbConfig_compare.new_table_name : connectorConfig.dbConfig_compare.object_name));
connectorConfig.destDBConfig.syncDefaultDatabaseUrl = (connectorConfig.dedupSourceType == SourceType.Remove_Duplicates_from_a_Single_Table ? connectorConfig.dbConfig.syncDefaultDatabaseUrl
: connectorConfig.dbConfig_compare.syncDefaultDatabaseUrl);
connectorConfig.destDBConfig.dataSource = (connectorConfig.dedupSourceType == SourceType.Remove_Duplicates_from_a_Single_Table ? connectorConfig.dbConfig.dataSource
: connectorConfig.dbConfig_compare.dataSource);
connectorConfig.destDBConfig.databaseType = (connectorConfig.dedupSourceType == SourceType.Remove_Duplicates_from_a_Single_Table ? connectorConfig.dbConfig.databaseType
: connectorConfig.dbConfig_compare.databaseType);
entity.dest_config_json = JsonConvert.SerializeObject(connectorConfig.destDBConfig);
entity.dest_schema = (connectorConfig.dedupSourceType == SourceType.Remove_Duplicates_from_a_Single_Table ? connectorConfig.dbSchema
: connectorConfig.dbConfig_compare.db_schema);
entity.sync_started_at = null;
entity.sync_ended_at = null;
entity.job_id = "";
entity.simulation_count = -1;
entity.sync_count = null;
entity.dedup_type = DedupType.Full_Dedup;
entity.sync_status = null;
entity.unique_records_count = 0;
entity.sync_updated_count = 0;
entity.sync_ended_at = null;
entity.sync_started_at = null;
entity.last_sync_at = null;
entity.last_sync_status = null;
//Delete destination table
SyncRepository.RemovePGSyncTable(connectorConfig);
_context.Entry(entity).State = EntityState.Modified;
_context.SaveChanges();
}
catch (Exception ex)
{
Console.WriteLine("Error:{0}", ex.Message);
return(ex.Message);
}
return("success");
}
/// <summary> /// Use this mode to configure the given target. /// </summary> protected abstract void Configure(ConnectorConfig target, AddressList addresses, int subMode);
