/// <summary> /// Manages job threads against any submission of status /// </summary> /// <param name="id"></param> private void CheckJobState(string id, bool ensureSend) { if (_databaseJobMap.ContainsKey(id)) { // send status to config server switch (_databaseJobMap[id].ExecutionState.Status) { case RecoveryStatus.Failure: case RecoveryStatus.Completed: case RecoveryStatus.Cancelled: foreach (RecoveryJobBase job in _databaseJobMap[id].JobList) { try { // close all opened backup files switch (job.JobType) { case RecoveryJobType.DataBackup: if (_databaseJobMap[id].ExecutionState.Status == RecoveryStatus.Cancelled) { _databaseJobMap[job.JobIdentifier].RecoveryPersistenceManager.CloseBackupFile(job.Database, RecoveryFileState.Cancelled); } else if (_databaseJobMap[id].ExecutionState.Status == RecoveryStatus.Failure) { _databaseJobMap[job.JobIdentifier].RecoveryPersistenceManager.CloseBackupFile(job.Database, RecoveryFileState.Failed); } else if (_databaseJobMap[id].ExecutionState.Status == RecoveryStatus.Completed) { _databaseJobMap[job.JobIdentifier].RecoveryPersistenceManager.CloseBackupFile(job.Database, RecoveryFileState.Completed); } break; //ASSUMPTION:commented out under the assumption that during diffbackup oplog and dif job will create same file, //so only oplog job will close the file. // //case RecoveryJobType.DifferentialBackup: // _databaseJobMap[_job.JobIdentifier].RecoveryPersistenceManager.CloseBackupFile(RecoveryFileNames.Diflog); // break; // } try { if (job.State != ThreadState.Unstarted && job.State != ThreadState.Stopped) { job.Stop(); } } catch (ThreadAbortException) { Thread.ResetAbort(); // ignore it } try { job.Dispose(); } catch (ThreadAbortException) { Thread.ResetAbort(); // ignore it } try { #region rename folder JobInfoObject infoObj = _databaseJobMap[job.JobIdentifier]; switch (job.JobType) { case RecoveryJobType.DataBackup: if (infoObj.JobType == RecoveryJobType.DataBackup) { if (_databaseJobMap[id].ExecutionState.Status == RecoveryStatus.Failure) { infoObj.RenameRootFolder(RecoveryFileState.Failed); } else if (_databaseJobMap[id].ExecutionState.Status == RecoveryStatus.Cancelled) { infoObj.RenameRootFolder(RecoveryFileState.Cancelled); } else { infoObj.RenameRootFolder(RecoveryFileState.Completed); } } break; } #endregion } catch (Exception ex) { if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsErrorEnabled) { LoggerManager.Instance.RecoveryLogger.Error("ShardRecoveryManager.CheckJobState()", id + " : " + job.JobType + " : Renaming Folder : " + ex.ToString()); } } if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsInfoEnabled) { LoggerManager.Instance.RecoveryLogger.Info("ShardRecoveryManager.CheckJobState()", id + " : " + job.JobType + " : End"); } } catch (Exception exp) { if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsErrorEnabled) { LoggerManager.Instance.RecoveryLogger.Error("ShardRecoveryManager.CheckJobState()", id + " : " + job.JobType + " : " + exp.ToString()); } } } _databaseJobMap[id].ExecutionState.StopTime = DateTime.Now; _databaseJobMap[id].RecoveryPersistenceManager.SharedQueue.CompleteAdding(); _databaseJobMap[id].RecoveryPersistenceManager.IsJobActive = false; _databaseJobMap[id].ExecutionState.MessageTime = DateTime.Now; _context.ConfigurationSession.SubmitShardJobStatus(_databaseJobMap[id].ExecutionState); break; case RecoveryStatus.Executing: case RecoveryStatus.uninitiated: case RecoveryStatus.Waiting: if (ensureSend) { _databaseJobMap[id].ExecutionState.MessageTime = DateTime.Now; if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsInfoEnabled) { LoggerManager.Instance.RecoveryLogger.Info("ShardRecoveryManager.SubmitState()", _databaseJobMap[id].ExecutionState.ToString()); } _context.ConfigurationSession.SubmitShardJobStatus(_databaseJobMap[id].ExecutionState); } else { try { // check status of datajob and diffjob RecoveryJobBase dataJob = _databaseJobMap[id].JobList.Where(x => (x.JobType == RecoveryJobType.DataBackup) || (x.JobType == RecoveryJobType.DataRestore)).First(); RecoveryJobBase oplogJob = null; if (dataJob != null) { if (dataJob.ExecutionStatus.Status == RecoveryStatus.Completed) { if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsInfoEnabled) { LoggerManager.Instance.RecoveryLogger.Info("ShardRecoveryManager.CheckJobState()", id + "WARNING: Else case executed for DIFbackup"); } try { if (dataJob.JobType == RecoveryJobType.DataBackup) { //close file _databaseJobMap[dataJob.JobIdentifier].RecoveryPersistenceManager.CloseBackupFile(dataJob.Database, RecoveryFileState.Completed); } // stop data job dataJob.Stop(); } catch (ThreadAbortException) { Thread.ResetAbort(); // ignore it } _databaseJobMap[id].RecoveryPersistenceManager.SharedQueue.Consumed = false; _databaseJobMap[id].RecoveryPersistenceManager.SharedQueue.PauseProducing = false; SendStatus(id); } } } catch (Exception exp) { if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsErrorEnabled) { LoggerManager.Instance.RecoveryLogger.Error("ShardRecoveryManager.SubmitState_StartOplog()", id + " : " + exp.ToString()); } } } break; } } else { if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsInfoEnabled) { LoggerManager.Instance.RecoveryLogger.Info("ShardRecoveryManager.SubmitState()", "Shard does not contain a job against id: " + id); } } }
private RecoveryOperationStatus CancelRecoveryJob(RecoveryOperation opContext) { RecoveryOperationStatus status = new RecoveryOperationStatus(RecoveryStatus.Success); status.JobIdentifier = opContext.JobIdentifer; if (opContext != null) { try { #region End/Cancel job if (_databaseJobMap.ContainsKey(opContext.JobIdentifer)) { List <RecoveryJobBase> jobList = _databaseJobMap[opContext.JobIdentifer].JobList; foreach (RecoveryJobBase job in jobList) { if (opContext.OpCode == RecoveryOpCodes.CancelJob) { switch (job.JobType) { case RecoveryJobType.DataBackup: _databaseJobMap[opContext.JobIdentifer].RecoveryPersistenceManager.CloseBackupFile(job.Database, RecoveryFileState.Cancelled); break; //NOTE: diffbackup job is ignored under the assumption oplog file will close the file for it } } //if (job.JobType != RecoveryJobType.DifferentialRestore) { try { if (job.State != ThreadState.Unstarted && job.State != ThreadState.Stopped && job.State != ThreadState.WaitSleepJoin) { job.Stop(); } } catch (ThreadAbortException) { Thread.ResetAbort(); } catch (Exception exp) { } try { job.Dispose(); } catch (ThreadAbortException) { Thread.ResetAbort(); } catch (Exception exp) { } } //else //{ // _databaseJobMap[opContext.JobIdentifer].RecoveryPersistenceManager.IsJobActive = false; // _databaseJobMap[opContext.JobIdentifer].RecoveryPersistenceManager.Dispose(); //} } #region rename folder JobInfoObject infoObj = _databaseJobMap[opContext.JobIdentifer]; if (infoObj.JobType == RecoveryJobType.DataBackup) { switch (opContext.OpCode) { case RecoveryOpCodes.CancelJob: infoObj.RenameRootFolder(RecoveryFileState.Cancelled); break; } } #endregion // removing job from databaseJobMap _databaseJobMap[opContext.JobIdentifer].RecoveryPersistenceManager.IsJobActive = false; _databaseJobMap[opContext.JobIdentifer].Dispose(); _databaseJobMap.Remove(opContext.JobIdentifer); } #endregion } catch (Exception exp) { if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsErrorEnabled) { LoggerManager.Instance.RecoveryLogger.Error("ShardRecoveryManager.CancelRecoveryJob()", opContext.JobIdentifer + "_" + _databaseJobMap[opContext.JobIdentifer].ExecutionState.JobType + " : " + exp.ToString()); } if (_databaseJobMap.ContainsKey(opContext.JobIdentifer)) { _databaseJobMap[opContext.JobIdentifer].ExecutionState.Status = RecoveryStatus.Failure; _databaseJobMap[opContext.JobIdentifer].ExecutionState.Message = "Failure during Cancel phase"; // commented because the config server will not have any job handler to recieve this message //CheckJobState(opContext.JobIdentifer, true); } status.Status = RecoveryStatus.Failure; status.Message = exp.ToString(); } } if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsInfoEnabled) { LoggerManager.Instance.RecoveryLogger.Info("ShardRecoveryManager.CancelRecoveryJob()", status.ToString()); } return(status); }