/// <summary> /// Action: OnStateElection /// Description: It is the error filter to capture any error occur while performing the data sync. If any error occured then it will update sync status to 3. /// Sync status 3 means failed. /// </summary> /// <param name="context"></param> void IElectStateFilter.OnStateElection(ElectStateContext context) { Console.WriteLine(string.Format("Job `{0}` has been changed to state `{1}`", context.BackgroundJob?.Id, context.CandidateState.Name)); //Get current state var failedState = context.CandidateState as FailedState; if (failedState != null) { Console.WriteLine(string.Format("Job `{0}` has been failed due to an exception `{1}`", context.BackgroundJob.Id, failedState.Exception)); var ccid = context.BackgroundJob.Job.Args.ElementAt(2) as string; int connectorId = (int)context.BackgroundJob.Job.Args.ElementAt(1); if (!string.IsNullOrEmpty(ccid) && connectorId > 0) { if (GC.GetTotalMemory(false) >= 67108864) { Console.WriteLine($"GC.Generation: 2, max allocated memory: {GC.GetTotalMemory(false)}"); GC.Collect(2); GC.WaitForPendingFinalizers(); GC.Collect(2); Console.WriteLine($"Max allocated memory after GC.Collect: {GC.GetTotalMemory(false)}"); } if (GC.GetTotalMemory(false) >= 33554432) { Console.WriteLine($"GC.Generation: 1, max allocated memory: {GC.GetTotalMemory(false)}"); GC.Collect(1); GC.WaitForPendingFinalizers(); GC.Collect(1); Console.WriteLine($"Max allocated memory after GC.Collect: {GC.GetTotalMemory(false)}"); } if (GC.GetTotalMemory(false) >= 20971520) { Console.WriteLine($"GC.Generation: 0, max allocated memory: {GC.GetTotalMemory(false)}"); GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect(); Console.WriteLine($"Max allocated memory after GC.Collect: {GC.GetTotalMemory(false)}"); } //set sync status to failed{3} var connectorLogs = new ConnectorLogs() { sync_ended_at = DateTime.UtcNow, sync_logs = new List <string>() { HttpUtility.UrlEncode($"{DateTime.UtcNow:yyyy-MM-dd HH:mm:ss.fff zzz} [{LogLevel.Error}]: {failedState.Exception} {Environment.NewLine}") } }; SyncRepository.UpdateSyncInfo(id: connectorId, ccid: ccid, status: 3, connectorLogs: connectorLogs); } } }
/// <summary> /// Action: OnPerformed /// Description: It is the final filter to used for updating sync status to 2. /// Sync status 2 means completed. /// </summary> /// <param name="context"></param> void IServerFilter.OnPerformed(PerformedContext context) { if (context.Canceled == false && context.Exception == null) { Console.WriteLine(string.Format("Job `{0}` has been performed", context.BackgroundJob?.Id)); var ccid = context.BackgroundJob.Job.Args.ElementAt(2) as string; int connectorId = (int)context.BackgroundJob.Job.Args.ElementAt(1); if (!string.IsNullOrEmpty(ccid) && connectorId > 0) { if (GC.GetTotalMemory(false) >= 67108864) { Console.WriteLine($"GC.Generation: 2, max allocated memory: {GC.GetTotalMemory(false)}"); GC.Collect(2); GC.WaitForPendingFinalizers(); GC.Collect(2); Console.WriteLine($"Max allocated memory after GC.Collect: {GC.GetTotalMemory(false)}"); } if (GC.GetTotalMemory(false) >= 33554432) { Console.WriteLine($"GC.Generation: 1, max allocated memory: {GC.GetTotalMemory(false)}"); GC.Collect(1); GC.WaitForPendingFinalizers(); GC.Collect(1); Console.WriteLine($"Max allocated memory after GC.Collect: {GC.GetTotalMemory(false)}"); } if (GC.GetTotalMemory(false) >= 20971520) { Console.WriteLine($"GC.Generation: 0, max allocated memory: {GC.GetTotalMemory(false)}"); GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect(); Console.WriteLine($"Max allocated memory after GC.Collect: {GC.GetTotalMemory(false)}"); } //set sync status to completed{2} var connectorLogs = new ConnectorLogs() { sync_ended_at = DateTime.UtcNow, sync_logs = new List <string>() }; SyncRepository.UpdateSyncInfo(id: connectorId, ccid: ccid, status: 2, connectorLogs: connectorLogs); } } }
/// <summary> /// Action: OnCreated /// Description: It is the second filter to update the sync status to 1 for all jobs validated by previous filter. But jobs are not to be started yet. /// Sync status 1 means pending/ongoing. /// </summary> /// <param name="context"></param> void IClientFilter.OnCreated(CreatedContext context) { if (context.Canceled == false || context.Exception == null) { Console.WriteLine(string.Format("Job is based on method `{0}` has been created with id `{1}`", context.Job.Method.Name, context.BackgroundJob?.Id)); var ccid = context.Job.Args.ElementAt(2) as string; int connectorId = (int)context.Job.Args.ElementAt(1); string jobId = context.BackgroundJob?.Id; if (!string.IsNullOrEmpty(ccid) && connectorId > 0) { //set sync status to progress{1}. var connectorLogs = new ConnectorLogs() { sync_started_at = DateTime.UtcNow, sync_ended_at = null, sync_logs = new List <string>() }; SyncRepository.UpdateSyncInfo(id: connectorId, ccid: ccid, status: 1, count: 0, jobid: jobId, connectorLogs: connectorLogs, totaluniquecount: 0, sync_updated_count: 0, deduped_count: 0, total_records_count: 0); } } }
public async Task ResetScheduledJobs() { try { //cancel token var cancellationToken = (new CancellationTokenSource()).Token; await Task.Run(async() => { Connectors[] connectors = null; Console.WriteLine("RSJ:Get connectors starts"); //get sync connectors using (ConnectionFactory connectionFactory = new ConnectionFactory(ConfigVars.Instance.connectionString)) { StringBuilder sb = new StringBuilder(); sb.Append(string.Format("SELECT c.* FROM \"{0}\".\"Connectors\" c", Constants.ADDON_DB_DEFAULT_SCHEMA)); sb.Append(string.Format(" LEFT JOIN \"{0}\".\"Resources\" r ON c.ccid=r.uuid", Constants.ADDON_DB_DEFAULT_SCHEMA)); //sb.Append(" WHERE r.plan NOT IN (" + $"{string.Join(",", ConfigVars.Instance.addonPrivatePlanLevels.Select(p => $"'{p}'").ToArray())}" + ");"); connectors = connectionFactory.DbConnection.Query <Connectors>(sb.ToString()).ToArray(); } Console.WriteLine("RSJ:Get connectors ended"); if (connectors != null && connectors.Length > 0) { Console.WriteLine("RSJ:Connectors Count: {0}", connectors.Length); for (int i = 0; i < connectors.Length; ++i) { //Cancel current task if cancel requested (eg: when system getting shutdown) if (cancellationToken != null && cancellationToken.IsCancellationRequested) { cancellationToken.ThrowIfCancellationRequested(); return; } var isRecurringJobNeedRestart = false; var connector = connectors[i]; if (connector.sync_status == 1) { //one time schedule job delete if (!string.IsNullOrEmpty(connector.job_id) && (connector.schedule_type == ScheduleType.MANUAL_SYNC)) { //delete old jobs DeleteJob(connector.job_id); } //set sync status to failed{3} var connectorLogs = new ConnectorLogs() { sync_ended_at = DateTime.UtcNow, sync_logs = new List <string>() { RestSharp.Extensions.MonoHttp.HttpUtility.UrlEncode($"Records synced: {connector.sync_count} {Environment.NewLine}"), RestSharp.Extensions.MonoHttp.HttpUtility.UrlEncode($"{DateTime.UtcNow:yyyy-MM-dd HH:mm:ss.fff zzz} [{Microsoft.Extensions.Logging.LogLevel.Information}]: {"Restarted sync after Heroku Dyno Restart, no action needed on your part"} {Environment.NewLine}") } }; //Update job status Console.WriteLine("RSJ:Update connector sync status starts"); if (connector.schedule_type == ScheduleType.MANUAL_SYNC) { SyncRepository.UpdateSyncInfo(id: connector.connector_id, ccid: connector.ccid, status: 9, connectorLogs: connectorLogs); Console.WriteLine("RSJ:Newly schedule job manual sync:{0}-{1}", connector.ccid, connector.connector_id); await ScheduleJob(connector.ccid, connector.connector_id, connector.connector_type, connector.schedule_type, connector.custom_schedule_in_minutes).ConfigureAwait(false); } else { isRecurringJobNeedRestart = true; SyncRepository.UpdateSyncInfo(id: connector.connector_id, ccid: connector.ccid, status: 9, connectorLogs: connectorLogs); } Console.WriteLine("RSJ:Update connector sync status ended"); } if (connector.schedule_type != ScheduleType.MANUAL_SYNC) { string recurringJobId = await IsScheduledJob(connector.ccid, connector.connector_id, connector.schedule_type, connector.custom_schedule_in_minutes).ConfigureAwait(false); Console.WriteLine("RSJ:Recurring Job ID:{0}", recurringJobId); Console.WriteLine("RSJ:isRecurringJobNeedRestart:{0}", isRecurringJobNeedRestart); if (string.IsNullOrEmpty(recurringJobId)) { Console.WriteLine("RSJ:Newly schedule job:{0}-{1}", connector.ccid, connector.connector_id); await ScheduleJob(connector.ccid, connector.connector_id, connector.connector_type, connector.schedule_type, connector.custom_schedule_in_minutes).ConfigureAwait(false); } else if (isRecurringJobNeedRestart) { Console.WriteLine("RSJ:scheduled job triggered immediately:{0}", recurringJobId); RecurringJob.Trigger(recurringJobId.Replace("recurring-job:", "")); } } } } }, cancellationToken); } catch (Exception ex) { Console.WriteLine("Error {0}", ex.Message); } }
/// <summary> /// Method: ScheduleJob /// Description: It is used to schedule background sync process for connector /// </summary> /// <param name="connector"></param> /// <param name="id"></param> /// <returns>status as int</returns> public async Task <int> ScheduleJob(string ccid, int id, ConnectorType connectorType, ScheduleType scheduleType, int?customScheduleInMinutes = 1200) { var syncStatus = 1; try { if (string.IsNullOrEmpty(ccid) || id == 0) { return(0); } IJobCancellationToken token = JobCancellationToken.Null; //token = new JobCancellationToken(true); var jobKey = string.Empty; if (scheduleType != ScheduleType.MANUAL_SYNC)// && scheduleType != ScheduleType.STREAMING_SYNC) { jobKey = Math.Abs(Guid.NewGuid().ToInt()).ToString(); } if (ConnectorType.Azure_SQL == connectorType) { //switch (scheduleType) //{ // case ScheduleType.EVERY_15_MINS: // RecurringJob.AddOrUpdate<DERepository>(jobKey, service => service.AddDataRowsToSqlConnector(token, id, ccid), Cron.MinuteInterval(15), null, "critical"); // break; // case ScheduleType.CUSTOM: // RecurringJob.AddOrUpdate<DERepository>(jobKey, service => service.AddDataRowsToSqlConnector(token, id, ccid), Cron.MinuteInterval(customScheduleInMinutes), null, "critical"); // break; // case ScheduleType.EVERY_60_MINS: // RecurringJob.AddOrUpdate<DERepository>(jobKey, service => service.AddDataRowsToSqlConnector(token, id, ccid), Cron.Hourly, null, "critical"); // break; // case ScheduleType.ONCE_DAILY: // RecurringJob.AddOrUpdate<DERepository>(jobKey, service => service.AddDataRowsToSqlConnector(token, id, ccid), Cron.Daily, null, "critical"); // break; // case ScheduleType.TWICE_DAILY: // RecurringJob.AddOrUpdate<DERepository>(jobKey, service => service.AddDataRowsToSqlConnector(token, id, ccid), Cron.HourInterval(11), null, "critical"); // break; // case ScheduleType.TWICE_WEEKLY: // RecurringJob.AddOrUpdate<DERepository>(jobKey, service => service.AddDataRowsToSqlConnector(token, id, ccid), Cron.DayInterval(2), null, "critical"); // break; // case ScheduleType.ONCE_WEEKLY: // RecurringJob.AddOrUpdate<DERepository>(jobKey, (service) => service.AddDataRowsToSqlConnector(token, id, ccid), Cron.Weekly, null, "critical"); // break; // case ScheduleType.MANUAL_SYNC: // jobKey = BackgroundJob.Enqueue<DERepository>(service => service.AddDataRowsToSqlConnector(token, id, ccid)); // break; // case ScheduleType.STREAMING_SYNC: // jobKey = string.Empty; // break; //} } else { switch (scheduleType) { case ScheduleType.EVERY_15_MINS: RecurringJob.AddOrUpdate <ISyncRepository>(jobKey, service => service.DeDupRowsFromDatabaseTable(token, id, ccid), DedupCron.MinuteInterval(15), TimeZoneInfo.Local, Constants.JOB_QUEUE_NAME); break; case ScheduleType.CUSTOM: RecurringJob.AddOrUpdate <ISyncRepository>(jobKey, service => service.DeDupRowsFromDatabaseTable(token, id, ccid), DedupCron.MinuteInterval((int)customScheduleInMinutes), TimeZoneInfo.Local, Constants.JOB_QUEUE_NAME); break; case ScheduleType.EVERY_60_MINS: RecurringJob.AddOrUpdate <ISyncRepository>(jobKey, service => service.DeDupRowsFromDatabaseTable(token, id, ccid), DedupCron.Hourly, TimeZoneInfo.Local, Constants.JOB_QUEUE_NAME); break; case ScheduleType.ONCE_DAILY: RecurringJob.AddOrUpdate <ISyncRepository>(jobKey, service => service.DeDupRowsFromDatabaseTable(token, id, ccid), DedupCron.Daily, TimeZoneInfo.Local, Constants.JOB_QUEUE_NAME); break; case ScheduleType.TWICE_DAILY: RecurringJob.AddOrUpdate <ISyncRepository>(jobKey, service => service.DeDupRowsFromDatabaseTable(token, id, ccid), DedupCron.HourInterval(11), TimeZoneInfo.Local, Constants.JOB_QUEUE_NAME); break; case ScheduleType.TWICE_WEEKLY: RecurringJob.AddOrUpdate <ISyncRepository>(jobKey, service => service.DeDupRowsFromDatabaseTable(token, id, ccid), DedupCron.DayInterval(2), TimeZoneInfo.Local, Constants.JOB_QUEUE_NAME); break; case ScheduleType.ONCE_WEEKLY: RecurringJob.AddOrUpdate <ISyncRepository>(jobKey, (service) => service.DeDupRowsFromDatabaseTable(token, id, ccid), DedupCron.Weekly, TimeZoneInfo.Local, Constants.JOB_QUEUE_NAME); break; case ScheduleType.MANUAL_SYNC: jobKey = BackgroundJob.Enqueue <ISyncRepository>(service => service.DeDupRowsFromDatabaseTable(token, id, ccid)); break; //case ScheduleType.STREAMING_SYNC: // jobKey = string.Empty; // break; } } if (!string.IsNullOrEmpty(jobKey)) { SyncRepository.UpdateSyncInfo(id: id, ccid: ccid, jobid: jobKey); } } catch (Exception ex) { syncStatus = 0; Console.WriteLine("Error: {0}", ex.Message); } return(await Task.FromResult(syncStatus)); }