/// <summary>
/// Action: OnStateElection
/// Description: It is the error filter to capture any error occur while performing the data sync. If any error occured then it will update sync status to 3.
/// Sync status 3 means failed.
/// </summary>
/// <param name="context"></param>
void IElectStateFilter.OnStateElection(ElectStateContext context)
{
Console.WriteLine(string.Format("Job `{0}` has been changed to state `{1}`", context.BackgroundJob?.Id, context.CandidateState.Name));
//Get current state
var failedState = context.CandidateState as FailedState;
if (failedState != null)
{
Console.WriteLine(string.Format("Job `{0}` has been failed due to an exception `{1}`", context.BackgroundJob.Id, failedState.Exception));
var ccid = context.BackgroundJob.Job.Args.ElementAt(2) as string;
int connectorId = (int)context.BackgroundJob.Job.Args.ElementAt(1);
if (!string.IsNullOrEmpty(ccid) && connectorId > 0)
{
if (GC.GetTotalMemory(false) >= 67108864)
{
Console.WriteLine($"GC.Generation: 2, max allocated memory: {GC.GetTotalMemory(false)}");
GC.Collect(2);
GC.WaitForPendingFinalizers();
GC.Collect(2);
Console.WriteLine($"Max allocated memory after GC.Collect: {GC.GetTotalMemory(false)}");
}
if (GC.GetTotalMemory(false) >= 33554432)
{
Console.WriteLine($"GC.Generation: 1, max allocated memory: {GC.GetTotalMemory(false)}");
GC.Collect(1);
GC.WaitForPendingFinalizers();
GC.Collect(1);
Console.WriteLine($"Max allocated memory after GC.Collect: {GC.GetTotalMemory(false)}");
}
if (GC.GetTotalMemory(false) >= 20971520)
{
Console.WriteLine($"GC.Generation: 0, max allocated memory: {GC.GetTotalMemory(false)}");
GC.Collect();
GC.WaitForPendingFinalizers();
GC.Collect();
Console.WriteLine($"Max allocated memory after GC.Collect: {GC.GetTotalMemory(false)}");
}
//set sync status to failed{3}
var connectorLogs = new ConnectorLogs()
{
sync_ended_at = DateTime.UtcNow,
sync_logs = new List <string>()
{
HttpUtility.UrlEncode($"{DateTime.UtcNow:yyyy-MM-dd HH:mm:ss.fff zzz} [{LogLevel.Error}]: {failedState.Exception} {Environment.NewLine}")
}
};
SyncRepository.UpdateSyncInfo(id: connectorId, ccid: ccid, status: 3, connectorLogs: connectorLogs);
}
}
}
/// <summary>
/// Action: OnPerformed
/// Description: It is the final filter to used for updating sync status to 2.
/// Sync status 2 means completed.
/// </summary>
/// <param name="context"></param>
void IServerFilter.OnPerformed(PerformedContext context)
{
if (context.Canceled == false && context.Exception == null)
{
Console.WriteLine(string.Format("Job `{0}` has been performed", context.BackgroundJob?.Id));
var ccid = context.BackgroundJob.Job.Args.ElementAt(2) as string;
int connectorId = (int)context.BackgroundJob.Job.Args.ElementAt(1);
if (!string.IsNullOrEmpty(ccid) && connectorId > 0)
{
if (GC.GetTotalMemory(false) >= 67108864)
{
Console.WriteLine($"GC.Generation: 2, max allocated memory: {GC.GetTotalMemory(false)}");
GC.Collect(2);
GC.WaitForPendingFinalizers();
GC.Collect(2);
Console.WriteLine($"Max allocated memory after GC.Collect: {GC.GetTotalMemory(false)}");
}
if (GC.GetTotalMemory(false) >= 33554432)
{
Console.WriteLine($"GC.Generation: 1, max allocated memory: {GC.GetTotalMemory(false)}");
GC.Collect(1);
GC.WaitForPendingFinalizers();
GC.Collect(1);
Console.WriteLine($"Max allocated memory after GC.Collect: {GC.GetTotalMemory(false)}");
}
if (GC.GetTotalMemory(false) >= 20971520)
{
Console.WriteLine($"GC.Generation: 0, max allocated memory: {GC.GetTotalMemory(false)}");
GC.Collect();
GC.WaitForPendingFinalizers();
GC.Collect();
Console.WriteLine($"Max allocated memory after GC.Collect: {GC.GetTotalMemory(false)}");
}
//set sync status to completed{2}
var connectorLogs = new ConnectorLogs()
{
sync_ended_at = DateTime.UtcNow,
sync_logs = new List <string>()
};
SyncRepository.UpdateSyncInfo(id: connectorId, ccid: ccid, status: 2, connectorLogs: connectorLogs);
}
}
}
/// <summary>
/// Action: OnCreated
/// Description: It is the second filter to update the sync status to 1 for all jobs validated by previous filter. But jobs are not to be started yet.
/// Sync status 1 means pending/ongoing.
/// </summary>
/// <param name="context"></param>
void IClientFilter.OnCreated(CreatedContext context)
{
if (context.Canceled == false || context.Exception == null)
{
Console.WriteLine(string.Format("Job is based on method `{0}` has been created with id `{1}`", context.Job.Method.Name, context.BackgroundJob?.Id));
var ccid = context.Job.Args.ElementAt(2) as string;
int connectorId = (int)context.Job.Args.ElementAt(1);
string jobId = context.BackgroundJob?.Id;
if (!string.IsNullOrEmpty(ccid) && connectorId > 0)
{
//set sync status to progress{1}.
var connectorLogs = new ConnectorLogs()
{
sync_started_at = DateTime.UtcNow,
sync_ended_at = null,
sync_logs = new List <string>()
};
SyncRepository.UpdateSyncInfo(id: connectorId, ccid: ccid, status: 1, count: 0, jobid: jobId, connectorLogs: connectorLogs, totaluniquecount: 0, sync_updated_count: 0, deduped_count: 0, total_records_count: 0);
}
}
}
public async Task ResetScheduledJobs()
{
try
{
//cancel token
var cancellationToken = (new CancellationTokenSource()).Token;
await Task.Run(async() =>
{
Connectors[] connectors = null;
Console.WriteLine("RSJ:Get connectors starts");
//get sync connectors
using (ConnectionFactory connectionFactory = new ConnectionFactory(ConfigVars.Instance.connectionString))
{
StringBuilder sb = new StringBuilder();
sb.Append(string.Format("SELECT c.* FROM \"{0}\".\"Connectors\" c", Constants.ADDON_DB_DEFAULT_SCHEMA));
sb.Append(string.Format(" LEFT JOIN \"{0}\".\"Resources\" r ON c.ccid=r.uuid", Constants.ADDON_DB_DEFAULT_SCHEMA));
//sb.Append(" WHERE r.plan NOT IN (" + $"{string.Join(",", ConfigVars.Instance.addonPrivatePlanLevels.Select(p => $"'{p}'").ToArray())}" + ");");
connectors = connectionFactory.DbConnection.Query <Connectors>(sb.ToString()).ToArray();
}
Console.WriteLine("RSJ:Get connectors ended");
if (connectors != null && connectors.Length > 0)
{
Console.WriteLine("RSJ:Connectors Count: {0}", connectors.Length);
for (int i = 0; i < connectors.Length; ++i)
{
//Cancel current task if cancel requested (eg: when system getting shutdown)
if (cancellationToken != null && cancellationToken.IsCancellationRequested)
{
cancellationToken.ThrowIfCancellationRequested();
return;
}
var isRecurringJobNeedRestart = false;
var connector = connectors[i];
if (connector.sync_status == 1)
{
//one time schedule job delete
if (!string.IsNullOrEmpty(connector.job_id) && (connector.schedule_type == ScheduleType.MANUAL_SYNC))
{
//delete old jobs
DeleteJob(connector.job_id);
}
//set sync status to failed{3}
var connectorLogs = new ConnectorLogs()
{
sync_ended_at = DateTime.UtcNow,
sync_logs = new List <string>()
{
RestSharp.Extensions.MonoHttp.HttpUtility.UrlEncode($"Records synced: {connector.sync_count} {Environment.NewLine}"),
RestSharp.Extensions.MonoHttp.HttpUtility.UrlEncode($"{DateTime.UtcNow:yyyy-MM-dd HH:mm:ss.fff zzz} [{Microsoft.Extensions.Logging.LogLevel.Information}]: {"Restarted sync after Heroku Dyno Restart, no action needed on your part"} {Environment.NewLine}")
}
};
//Update job status
Console.WriteLine("RSJ:Update connector sync status starts");
if (connector.schedule_type == ScheduleType.MANUAL_SYNC)
{
SyncRepository.UpdateSyncInfo(id: connector.connector_id, ccid: connector.ccid, status: 9, connectorLogs: connectorLogs);
Console.WriteLine("RSJ:Newly schedule job manual sync:{0}-{1}", connector.ccid, connector.connector_id);
await ScheduleJob(connector.ccid, connector.connector_id, connector.connector_type, connector.schedule_type, connector.custom_schedule_in_minutes).ConfigureAwait(false);
}
else
{
isRecurringJobNeedRestart = true;
SyncRepository.UpdateSyncInfo(id: connector.connector_id, ccid: connector.ccid, status: 9, connectorLogs: connectorLogs);
}
Console.WriteLine("RSJ:Update connector sync status ended");
}
if (connector.schedule_type != ScheduleType.MANUAL_SYNC)
{
string recurringJobId = await IsScheduledJob(connector.ccid, connector.connector_id, connector.schedule_type, connector.custom_schedule_in_minutes).ConfigureAwait(false);
Console.WriteLine("RSJ:Recurring Job ID:{0}", recurringJobId);
Console.WriteLine("RSJ:isRecurringJobNeedRestart:{0}", isRecurringJobNeedRestart);
if (string.IsNullOrEmpty(recurringJobId))
{
Console.WriteLine("RSJ:Newly schedule job:{0}-{1}", connector.ccid, connector.connector_id);
await ScheduleJob(connector.ccid, connector.connector_id, connector.connector_type, connector.schedule_type, connector.custom_schedule_in_minutes).ConfigureAwait(false);
}
else if (isRecurringJobNeedRestart)
{
Console.WriteLine("RSJ:scheduled job triggered immediately:{0}", recurringJobId);
RecurringJob.Trigger(recurringJobId.Replace("recurring-job:", ""));
}
}
}
}
}, cancellationToken);
}
catch (Exception ex)
{
Console.WriteLine("Error {0}", ex.Message);
}
}
/// <summary>
/// Method: ScheduleJob
/// Description: It is used to schedule background sync process for connector
/// </summary>
/// <param name="connector"></param>
/// <param name="id"></param>
/// <returns>status as int</returns>
public async Task <int> ScheduleJob(string ccid, int id, ConnectorType connectorType, ScheduleType scheduleType, int?customScheduleInMinutes = 1200)
{
var syncStatus = 1;
try
{
if (string.IsNullOrEmpty(ccid) || id == 0)
{
return(0);
}
IJobCancellationToken token = JobCancellationToken.Null;
//token = new JobCancellationToken(true);
var jobKey = string.Empty;
if (scheduleType != ScheduleType.MANUAL_SYNC)// && scheduleType != ScheduleType.STREAMING_SYNC)
{
jobKey = Math.Abs(Guid.NewGuid().ToInt()).ToString();
}
if (ConnectorType.Azure_SQL == connectorType)
{
//switch (scheduleType)
//{
// case ScheduleType.EVERY_15_MINS:
// RecurringJob.AddOrUpdate<DERepository>(jobKey, service => service.AddDataRowsToSqlConnector(token, id, ccid), Cron.MinuteInterval(15), null, "critical");
// break;
// case ScheduleType.CUSTOM:
// RecurringJob.AddOrUpdate<DERepository>(jobKey, service => service.AddDataRowsToSqlConnector(token, id, ccid), Cron.MinuteInterval(customScheduleInMinutes), null, "critical");
// break;
// case ScheduleType.EVERY_60_MINS:
// RecurringJob.AddOrUpdate<DERepository>(jobKey, service => service.AddDataRowsToSqlConnector(token, id, ccid), Cron.Hourly, null, "critical");
// break;
// case ScheduleType.ONCE_DAILY:
// RecurringJob.AddOrUpdate<DERepository>(jobKey, service => service.AddDataRowsToSqlConnector(token, id, ccid), Cron.Daily, null, "critical");
// break;
// case ScheduleType.TWICE_DAILY:
// RecurringJob.AddOrUpdate<DERepository>(jobKey, service => service.AddDataRowsToSqlConnector(token, id, ccid), Cron.HourInterval(11), null, "critical");
// break;
// case ScheduleType.TWICE_WEEKLY:
// RecurringJob.AddOrUpdate<DERepository>(jobKey, service => service.AddDataRowsToSqlConnector(token, id, ccid), Cron.DayInterval(2), null, "critical");
// break;
// case ScheduleType.ONCE_WEEKLY:
// RecurringJob.AddOrUpdate<DERepository>(jobKey, (service) => service.AddDataRowsToSqlConnector(token, id, ccid), Cron.Weekly, null, "critical");
// break;
// case ScheduleType.MANUAL_SYNC:
// jobKey = BackgroundJob.Enqueue<DERepository>(service => service.AddDataRowsToSqlConnector(token, id, ccid));
// break;
// case ScheduleType.STREAMING_SYNC:
// jobKey = string.Empty;
// break;
//}
}
else
{
switch (scheduleType)
{
case ScheduleType.EVERY_15_MINS:
RecurringJob.AddOrUpdate <ISyncRepository>(jobKey, service => service.DeDupRowsFromDatabaseTable(token, id, ccid), DedupCron.MinuteInterval(15), TimeZoneInfo.Local, Constants.JOB_QUEUE_NAME);
break;
case ScheduleType.CUSTOM:
RecurringJob.AddOrUpdate <ISyncRepository>(jobKey, service => service.DeDupRowsFromDatabaseTable(token, id, ccid), DedupCron.MinuteInterval((int)customScheduleInMinutes), TimeZoneInfo.Local, Constants.JOB_QUEUE_NAME);
break;
case ScheduleType.EVERY_60_MINS:
RecurringJob.AddOrUpdate <ISyncRepository>(jobKey, service => service.DeDupRowsFromDatabaseTable(token, id, ccid), DedupCron.Hourly, TimeZoneInfo.Local, Constants.JOB_QUEUE_NAME);
break;
case ScheduleType.ONCE_DAILY:
RecurringJob.AddOrUpdate <ISyncRepository>(jobKey, service => service.DeDupRowsFromDatabaseTable(token, id, ccid), DedupCron.Daily, TimeZoneInfo.Local, Constants.JOB_QUEUE_NAME);
break;
case ScheduleType.TWICE_DAILY:
RecurringJob.AddOrUpdate <ISyncRepository>(jobKey, service => service.DeDupRowsFromDatabaseTable(token, id, ccid), DedupCron.HourInterval(11), TimeZoneInfo.Local, Constants.JOB_QUEUE_NAME);
break;
case ScheduleType.TWICE_WEEKLY:
RecurringJob.AddOrUpdate <ISyncRepository>(jobKey, service => service.DeDupRowsFromDatabaseTable(token, id, ccid), DedupCron.DayInterval(2), TimeZoneInfo.Local, Constants.JOB_QUEUE_NAME);
break;
case ScheduleType.ONCE_WEEKLY:
RecurringJob.AddOrUpdate <ISyncRepository>(jobKey, (service) => service.DeDupRowsFromDatabaseTable(token, id, ccid), DedupCron.Weekly, TimeZoneInfo.Local, Constants.JOB_QUEUE_NAME);
break;
case ScheduleType.MANUAL_SYNC:
jobKey = BackgroundJob.Enqueue <ISyncRepository>(service => service.DeDupRowsFromDatabaseTable(token, id, ccid));
break;
//case ScheduleType.STREAMING_SYNC:
// jobKey = string.Empty;
// break;
}
}
if (!string.IsNullOrEmpty(jobKey))
{
SyncRepository.UpdateSyncInfo(id: id, ccid: ccid, jobid: jobKey);
}
}
catch (Exception ex)
{
syncStatus = 0;
Console.WriteLine("Error: {0}", ex.Message);
}
return(await Task.FromResult(syncStatus));
}