private JobInfoObject CreateBackupJob(RecoveryOperation opContext) { JobInfoObject infoObject = null; RecoveryJobBase dataJob = null; RecoveryJobBase opLogJob = null; #region BackupJob SubmitBackupOpParams bckpParam = (SubmitBackupOpParams)opContext.Parameter; // create handler object infoObject = new JobInfoObject(opContext.JobIdentifer, _context.LocalShardName, _context.LocalAddress.ip, _context.ClusterName, RecoveryJobType.DataBackup, bckpParam.PersistenceConfiguration.FilePath); foreach (string db in bckpParam.PersistenceConfiguration.DbCollectionMap.Keys) { //assuming one database is sent dataJob = new DatabaseBackupJob(opContext.JobIdentifer, _context, db, bckpParam.PersistenceConfiguration.DbCollectionMap[db][db].ToList(), infoObject.RecoveryPersistenceManager, _context.ClusterName); dataJob.RegisterProgressHandler(this); infoObject.AddJob(dataJob); // set persistence configuration if (bckpParam.PersistenceConfiguration.FileName == null) { bckpParam.PersistenceConfiguration.FileName = new List <string>(); bckpParam.PersistenceConfiguration.FileName.Add(db);//add name of all databases for shard level job } infoObject.RecoveryPersistenceManager.SetJobConfiguration(RecoveryJobType.DataBackup, bckpParam.PersistenceConfiguration, db, 1); } #endregion return(infoObject); }
private void SubmitRecoveryJob(RecoveryOperation opContext) { if (opContext != null) { try { if (!_databaseJobMap.ContainsKey(opContext.JobIdentifer)) { JobInfoObject infoObject = null; switch (opContext.OpCode) { case RecoveryOpCodes.SubmitBackupJob: infoObject = CreateBackupJob(opContext); break; case RecoveryOpCodes.SubmitRestoreJob: infoObject = CreateRestoreJob(opContext); break; } if (infoObject != null) { _databaseJobMap.Add(opContext.JobIdentifer, infoObject); if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsInfoEnabled) { LoggerManager.Instance.RecoveryLogger.Info("ShardRecoveryManager.SubmitRecoveryJob()", opContext.JobIdentifer + "_" + opContext.OpCode + " Submitted"); } } } } catch (Exception exp) { if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsErrorEnabled) { LoggerManager.Instance.RecoveryLogger.Error("ShardRecoveryManager.SubmitRecoveryJob()", opContext.JobIdentifer + "_" + opContext.OpCode + " : " + exp.ToString()); } if (_databaseJobMap.ContainsKey(opContext.JobIdentifer)) { _databaseJobMap[opContext.JobIdentifer].ExecutionState.Status = RecoveryStatus.Failure; _databaseJobMap[opContext.JobIdentifer].ExecutionState.Message = "Failure during submission phase_" + opContext.OpCode; CheckJobState(opContext.JobIdentifer, true); } } } }
//M_TODO: must Specify DB name in case of shard job, so a separate job is created, for now this will run multiple times and override job if more //than one db is specified. private JobInfoObject CreateRestoreJob(RecoveryOperation opContext) { JobInfoObject infoObject = null; RecoveryJobBase dataJob = null; #region RestoreJob SubmitRestoreOpParams resParam = (SubmitRestoreOpParams)opContext.Parameter; foreach (string db in resParam.PersistenceConfiguration.DbCollectionMap.Keys) { // create handler object // Note: this is kept inside the loop with the assumption that a seperate info object is to be kept for each database in complete cluster job infoObject = new JobInfoObject(opContext.JobIdentifer, _context.LocalShardName, _context.LocalAddress.ip, _context.ClusterName, RecoveryJobType.DataRestore, resParam.PersistenceConfiguration.FilePath); string destination = resParam.PersistenceConfiguration.DbCollectionMap[db].Keys.First(); // create DataJob dataJob = new DatabaseRestoreJob(opContext.JobIdentifer, _context, destination, resParam.PersistenceConfiguration.DbCollectionMap[db][destination].ToList <string>(), infoObject.RecoveryPersistenceManager, _context.ClusterName); dataJob.RegisterProgressHandler(this); infoObject.AddJob(dataJob); // set persistence configuration if (resParam.PersistenceConfiguration.FileName == null) { resParam.PersistenceConfiguration.FileName = new List <string>(); resParam.PersistenceConfiguration.FileName.Add(db);//add name of all databases for shard level job } infoObject.RecoveryPersistenceManager.SetJobConfiguration(RecoveryJobType.DataRestore, resParam.PersistenceConfiguration, db, 1); } #endregion return(infoObject); }
/// <summary> /// Manages job threads against any submission of status /// </summary> /// <param name="id"></param> private void CheckJobState(string id, bool ensureSend) { if (_databaseJobMap.ContainsKey(id)) { // send status to config server switch (_databaseJobMap[id].ExecutionState.Status) { case RecoveryStatus.Failure: case RecoveryStatus.Completed: case RecoveryStatus.Cancelled: foreach (RecoveryJobBase job in _databaseJobMap[id].JobList) { try { // close all opened backup files switch (job.JobType) { case RecoveryJobType.DataBackup: if (_databaseJobMap[id].ExecutionState.Status == RecoveryStatus.Cancelled) { _databaseJobMap[job.JobIdentifier].RecoveryPersistenceManager.CloseBackupFile(job.Database, RecoveryFileState.Cancelled); } else if (_databaseJobMap[id].ExecutionState.Status == RecoveryStatus.Failure) { _databaseJobMap[job.JobIdentifier].RecoveryPersistenceManager.CloseBackupFile(job.Database, RecoveryFileState.Failed); } else if (_databaseJobMap[id].ExecutionState.Status == RecoveryStatus.Completed) { _databaseJobMap[job.JobIdentifier].RecoveryPersistenceManager.CloseBackupFile(job.Database, RecoveryFileState.Completed); } break; //ASSUMPTION:commented out under the assumption that during diffbackup oplog and dif job will create same file, //so only oplog job will close the file. // //case RecoveryJobType.DifferentialBackup: // _databaseJobMap[_job.JobIdentifier].RecoveryPersistenceManager.CloseBackupFile(RecoveryFileNames.Diflog); // break; // } try { if (job.State != ThreadState.Unstarted && job.State != ThreadState.Stopped) { job.Stop(); } } catch (ThreadAbortException) { Thread.ResetAbort(); // ignore it } try { job.Dispose(); } catch (ThreadAbortException) { Thread.ResetAbort(); // ignore it } try { #region rename folder JobInfoObject infoObj = _databaseJobMap[job.JobIdentifier]; switch (job.JobType) { case RecoveryJobType.DataBackup: if (infoObj.JobType == RecoveryJobType.DataBackup) { if (_databaseJobMap[id].ExecutionState.Status == RecoveryStatus.Failure) { infoObj.RenameRootFolder(RecoveryFileState.Failed); } else if (_databaseJobMap[id].ExecutionState.Status == RecoveryStatus.Cancelled) { infoObj.RenameRootFolder(RecoveryFileState.Cancelled); } else { infoObj.RenameRootFolder(RecoveryFileState.Completed); } } break; } #endregion } catch (Exception ex) { if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsErrorEnabled) { LoggerManager.Instance.RecoveryLogger.Error("ShardRecoveryManager.CheckJobState()", id + " : " + job.JobType + " : Renaming Folder : " + ex.ToString()); } } if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsInfoEnabled) { LoggerManager.Instance.RecoveryLogger.Info("ShardRecoveryManager.CheckJobState()", id + " : " + job.JobType + " : End"); } } catch (Exception exp) { if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsErrorEnabled) { LoggerManager.Instance.RecoveryLogger.Error("ShardRecoveryManager.CheckJobState()", id + " : " + job.JobType + " : " + exp.ToString()); } } } _databaseJobMap[id].ExecutionState.StopTime = DateTime.Now; _databaseJobMap[id].RecoveryPersistenceManager.SharedQueue.CompleteAdding(); _databaseJobMap[id].RecoveryPersistenceManager.IsJobActive = false; _databaseJobMap[id].ExecutionState.MessageTime = DateTime.Now; _context.ConfigurationSession.SubmitShardJobStatus(_databaseJobMap[id].ExecutionState); break; case RecoveryStatus.Executing: case RecoveryStatus.uninitiated: case RecoveryStatus.Waiting: if (ensureSend) { _databaseJobMap[id].ExecutionState.MessageTime = DateTime.Now; if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsInfoEnabled) { LoggerManager.Instance.RecoveryLogger.Info("ShardRecoveryManager.SubmitState()", _databaseJobMap[id].ExecutionState.ToString()); } _context.ConfigurationSession.SubmitShardJobStatus(_databaseJobMap[id].ExecutionState); } else { try { // check status of datajob and diffjob RecoveryJobBase dataJob = _databaseJobMap[id].JobList.Where(x => (x.JobType == RecoveryJobType.DataBackup) || (x.JobType == RecoveryJobType.DataRestore)).First(); RecoveryJobBase oplogJob = null; if (dataJob != null) { if (dataJob.ExecutionStatus.Status == RecoveryStatus.Completed) { if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsInfoEnabled) { LoggerManager.Instance.RecoveryLogger.Info("ShardRecoveryManager.CheckJobState()", id + "WARNING: Else case executed for DIFbackup"); } try { if (dataJob.JobType == RecoveryJobType.DataBackup) { //close file _databaseJobMap[dataJob.JobIdentifier].RecoveryPersistenceManager.CloseBackupFile(dataJob.Database, RecoveryFileState.Completed); } // stop data job dataJob.Stop(); } catch (ThreadAbortException) { Thread.ResetAbort(); // ignore it } _databaseJobMap[id].RecoveryPersistenceManager.SharedQueue.Consumed = false; _databaseJobMap[id].RecoveryPersistenceManager.SharedQueue.PauseProducing = false; SendStatus(id); } } } catch (Exception exp) { if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsErrorEnabled) { LoggerManager.Instance.RecoveryLogger.Error("ShardRecoveryManager.SubmitState_StartOplog()", id + " : " + exp.ToString()); } } } break; } } else { if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsInfoEnabled) { LoggerManager.Instance.RecoveryLogger.Info("ShardRecoveryManager.SubmitState()", "Shard does not contain a job against id: " + id); } } }
private RecoveryOperationStatus CancelRecoveryJob(RecoveryOperation opContext) { RecoveryOperationStatus status = new RecoveryOperationStatus(RecoveryStatus.Success); status.JobIdentifier = opContext.JobIdentifer; if (opContext != null) { try { #region End/Cancel job if (_databaseJobMap.ContainsKey(opContext.JobIdentifer)) { List <RecoveryJobBase> jobList = _databaseJobMap[opContext.JobIdentifer].JobList; foreach (RecoveryJobBase job in jobList) { if (opContext.OpCode == RecoveryOpCodes.CancelJob) { switch (job.JobType) { case RecoveryJobType.DataBackup: _databaseJobMap[opContext.JobIdentifer].RecoveryPersistenceManager.CloseBackupFile(job.Database, RecoveryFileState.Cancelled); break; //NOTE: diffbackup job is ignored under the assumption oplog file will close the file for it } } //if (job.JobType != RecoveryJobType.DifferentialRestore) { try { if (job.State != ThreadState.Unstarted && job.State != ThreadState.Stopped && job.State != ThreadState.WaitSleepJoin) { job.Stop(); } } catch (ThreadAbortException) { Thread.ResetAbort(); } catch (Exception exp) { } try { job.Dispose(); } catch (ThreadAbortException) { Thread.ResetAbort(); } catch (Exception exp) { } } //else //{ // _databaseJobMap[opContext.JobIdentifer].RecoveryPersistenceManager.IsJobActive = false; // _databaseJobMap[opContext.JobIdentifer].RecoveryPersistenceManager.Dispose(); //} } #region rename folder JobInfoObject infoObj = _databaseJobMap[opContext.JobIdentifer]; if (infoObj.JobType == RecoveryJobType.DataBackup) { switch (opContext.OpCode) { case RecoveryOpCodes.CancelJob: infoObj.RenameRootFolder(RecoveryFileState.Cancelled); break; } } #endregion // removing job from databaseJobMap _databaseJobMap[opContext.JobIdentifer].RecoveryPersistenceManager.IsJobActive = false; _databaseJobMap[opContext.JobIdentifer].Dispose(); _databaseJobMap.Remove(opContext.JobIdentifer); } #endregion } catch (Exception exp) { if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsErrorEnabled) { LoggerManager.Instance.RecoveryLogger.Error("ShardRecoveryManager.CancelRecoveryJob()", opContext.JobIdentifer + "_" + _databaseJobMap[opContext.JobIdentifer].ExecutionState.JobType + " : " + exp.ToString()); } if (_databaseJobMap.ContainsKey(opContext.JobIdentifer)) { _databaseJobMap[opContext.JobIdentifer].ExecutionState.Status = RecoveryStatus.Failure; _databaseJobMap[opContext.JobIdentifer].ExecutionState.Message = "Failure during Cancel phase"; // commented because the config server will not have any job handler to recieve this message //CheckJobState(opContext.JobIdentifer, true); } status.Status = RecoveryStatus.Failure; status.Message = exp.ToString(); } } if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsInfoEnabled) { LoggerManager.Instance.RecoveryLogger.Info("ShardRecoveryManager.CancelRecoveryJob()", status.ToString()); } return(status); }