private JobInfoObject CreateBackupJob(RecoveryOperation opContext)
{
JobInfoObject infoObject = null;
RecoveryJobBase dataJob = null;
RecoveryJobBase opLogJob = null;
#region BackupJob
SubmitBackupOpParams bckpParam = (SubmitBackupOpParams)opContext.Parameter;
// create handler object
infoObject = new JobInfoObject(opContext.JobIdentifer, _context.LocalShardName, _context.LocalAddress.ip, _context.ClusterName,
RecoveryJobType.DataBackup, bckpParam.PersistenceConfiguration.FilePath);
foreach (string db in bckpParam.PersistenceConfiguration.DbCollectionMap.Keys)
{
//assuming one database is sent
dataJob = new DatabaseBackupJob(opContext.JobIdentifer, _context, db, bckpParam.PersistenceConfiguration.DbCollectionMap[db][db].ToList(), infoObject.RecoveryPersistenceManager, _context.ClusterName);
dataJob.RegisterProgressHandler(this);
infoObject.AddJob(dataJob);
// set persistence configuration
if (bckpParam.PersistenceConfiguration.FileName == null)
{
bckpParam.PersistenceConfiguration.FileName = new List <string>();
bckpParam.PersistenceConfiguration.FileName.Add(db);//add name of all databases for shard level job
}
infoObject.RecoveryPersistenceManager.SetJobConfiguration(RecoveryJobType.DataBackup, bckpParam.PersistenceConfiguration, db, 1);
}
#endregion
return(infoObject);
}
private void SubmitRecoveryJob(RecoveryOperation opContext)
{
if (opContext != null)
{
try
{
if (!_databaseJobMap.ContainsKey(opContext.JobIdentifer))
{
JobInfoObject infoObject = null;
switch (opContext.OpCode)
{
case RecoveryOpCodes.SubmitBackupJob:
infoObject = CreateBackupJob(opContext);
break;
case RecoveryOpCodes.SubmitRestoreJob:
infoObject = CreateRestoreJob(opContext);
break;
}
if (infoObject != null)
{
_databaseJobMap.Add(opContext.JobIdentifer, infoObject);
if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsInfoEnabled)
{
LoggerManager.Instance.RecoveryLogger.Info("ShardRecoveryManager.SubmitRecoveryJob()", opContext.JobIdentifer + "_" + opContext.OpCode + " Submitted");
}
}
}
}
catch (Exception exp)
{
if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsErrorEnabled)
{
LoggerManager.Instance.RecoveryLogger.Error("ShardRecoveryManager.SubmitRecoveryJob()", opContext.JobIdentifer + "_" + opContext.OpCode + " : " + exp.ToString());
}
if (_databaseJobMap.ContainsKey(opContext.JobIdentifer))
{
_databaseJobMap[opContext.JobIdentifer].ExecutionState.Status = RecoveryStatus.Failure;
_databaseJobMap[opContext.JobIdentifer].ExecutionState.Message = "Failure during submission phase_" + opContext.OpCode;
CheckJobState(opContext.JobIdentifer, true);
}
}
}
}
//M_TODO: must Specify DB name in case of shard job, so a separate job is created, for now this will run multiple times and override job if more
//than one db is specified.
private JobInfoObject CreateRestoreJob(RecoveryOperation opContext)
{
JobInfoObject infoObject = null;
RecoveryJobBase dataJob = null;
#region RestoreJob
SubmitRestoreOpParams resParam = (SubmitRestoreOpParams)opContext.Parameter;
foreach (string db in resParam.PersistenceConfiguration.DbCollectionMap.Keys)
{
// create handler object
// Note: this is kept inside the loop with the assumption that a seperate info object is to be kept for each database in complete cluster job
infoObject = new JobInfoObject(opContext.JobIdentifer, _context.LocalShardName, _context.LocalAddress.ip, _context.ClusterName,
RecoveryJobType.DataRestore, resParam.PersistenceConfiguration.FilePath);
string destination = resParam.PersistenceConfiguration.DbCollectionMap[db].Keys.First();
// create DataJob
dataJob = new DatabaseRestoreJob(opContext.JobIdentifer, _context, destination, resParam.PersistenceConfiguration.DbCollectionMap[db][destination].ToList <string>(), infoObject.RecoveryPersistenceManager, _context.ClusterName);
dataJob.RegisterProgressHandler(this);
infoObject.AddJob(dataJob);
// set persistence configuration
if (resParam.PersistenceConfiguration.FileName == null)
{
resParam.PersistenceConfiguration.FileName = new List <string>();
resParam.PersistenceConfiguration.FileName.Add(db);//add name of all databases for shard level job
}
infoObject.RecoveryPersistenceManager.SetJobConfiguration(RecoveryJobType.DataRestore, resParam.PersistenceConfiguration, db, 1);
}
#endregion
return(infoObject);
}
/// <summary>
/// Manages job threads against any submission of status
/// </summary>
/// <param name="id"></param>
private void CheckJobState(string id, bool ensureSend)
{
if (_databaseJobMap.ContainsKey(id))
{
// send status to config server
switch (_databaseJobMap[id].ExecutionState.Status)
{
case RecoveryStatus.Failure:
case RecoveryStatus.Completed:
case RecoveryStatus.Cancelled:
foreach (RecoveryJobBase job in _databaseJobMap[id].JobList)
{
try
{
// close all opened backup files
switch (job.JobType)
{
case RecoveryJobType.DataBackup:
if (_databaseJobMap[id].ExecutionState.Status == RecoveryStatus.Cancelled)
{
_databaseJobMap[job.JobIdentifier].RecoveryPersistenceManager.CloseBackupFile(job.Database, RecoveryFileState.Cancelled);
}
else if (_databaseJobMap[id].ExecutionState.Status == RecoveryStatus.Failure)
{
_databaseJobMap[job.JobIdentifier].RecoveryPersistenceManager.CloseBackupFile(job.Database, RecoveryFileState.Failed);
}
else if (_databaseJobMap[id].ExecutionState.Status == RecoveryStatus.Completed)
{
_databaseJobMap[job.JobIdentifier].RecoveryPersistenceManager.CloseBackupFile(job.Database, RecoveryFileState.Completed);
}
break;
//ASSUMPTION:commented out under the assumption that during diffbackup oplog and dif job will create same file,
//so only oplog job will close the file.
//
//case RecoveryJobType.DifferentialBackup:
// _databaseJobMap[_job.JobIdentifier].RecoveryPersistenceManager.CloseBackupFile(RecoveryFileNames.Diflog);
// break;
//
}
try
{
if (job.State != ThreadState.Unstarted && job.State != ThreadState.Stopped)
{
job.Stop();
}
}
catch (ThreadAbortException)
{
Thread.ResetAbort(); // ignore it
}
try
{
job.Dispose();
}
catch (ThreadAbortException)
{
Thread.ResetAbort(); // ignore it
}
try
{
#region rename folder
JobInfoObject infoObj = _databaseJobMap[job.JobIdentifier];
switch (job.JobType)
{
case RecoveryJobType.DataBackup:
if (infoObj.JobType == RecoveryJobType.DataBackup)
{
if (_databaseJobMap[id].ExecutionState.Status == RecoveryStatus.Failure)
{
infoObj.RenameRootFolder(RecoveryFileState.Failed);
}
else if (_databaseJobMap[id].ExecutionState.Status == RecoveryStatus.Cancelled)
{
infoObj.RenameRootFolder(RecoveryFileState.Cancelled);
}
else
{
infoObj.RenameRootFolder(RecoveryFileState.Completed);
}
}
break;
}
#endregion
}
catch (Exception ex)
{
if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsErrorEnabled)
{
LoggerManager.Instance.RecoveryLogger.Error("ShardRecoveryManager.CheckJobState()", id + " : " + job.JobType + " : Renaming Folder : " + ex.ToString());
}
}
if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsInfoEnabled)
{
LoggerManager.Instance.RecoveryLogger.Info("ShardRecoveryManager.CheckJobState()", id + " : " + job.JobType + " : End");
}
}
catch (Exception exp)
{
if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsErrorEnabled)
{
LoggerManager.Instance.RecoveryLogger.Error("ShardRecoveryManager.CheckJobState()", id + " : " + job.JobType + " : " + exp.ToString());
}
}
}
_databaseJobMap[id].ExecutionState.StopTime = DateTime.Now;
_databaseJobMap[id].RecoveryPersistenceManager.SharedQueue.CompleteAdding();
_databaseJobMap[id].RecoveryPersistenceManager.IsJobActive = false;
_databaseJobMap[id].ExecutionState.MessageTime = DateTime.Now;
_context.ConfigurationSession.SubmitShardJobStatus(_databaseJobMap[id].ExecutionState);
break;
case RecoveryStatus.Executing:
case RecoveryStatus.uninitiated:
case RecoveryStatus.Waiting:
if (ensureSend)
{
_databaseJobMap[id].ExecutionState.MessageTime = DateTime.Now;
if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsInfoEnabled)
{
LoggerManager.Instance.RecoveryLogger.Info("ShardRecoveryManager.SubmitState()", _databaseJobMap[id].ExecutionState.ToString());
}
_context.ConfigurationSession.SubmitShardJobStatus(_databaseJobMap[id].ExecutionState);
}
else
{
try
{
// check status of datajob and diffjob
RecoveryJobBase dataJob = _databaseJobMap[id].JobList.Where(x => (x.JobType == RecoveryJobType.DataBackup) ||
(x.JobType == RecoveryJobType.DataRestore)).First();
RecoveryJobBase oplogJob = null;
if (dataJob != null)
{
if (dataJob.ExecutionStatus.Status == RecoveryStatus.Completed)
{
if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsInfoEnabled)
{
LoggerManager.Instance.RecoveryLogger.Info("ShardRecoveryManager.CheckJobState()", id + "WARNING: Else case executed for DIFbackup");
}
try
{
if (dataJob.JobType == RecoveryJobType.DataBackup)
{
//close file
_databaseJobMap[dataJob.JobIdentifier].RecoveryPersistenceManager.CloseBackupFile(dataJob.Database, RecoveryFileState.Completed);
}
// stop data job
dataJob.Stop();
}
catch (ThreadAbortException)
{
Thread.ResetAbort(); // ignore it
}
_databaseJobMap[id].RecoveryPersistenceManager.SharedQueue.Consumed = false;
_databaseJobMap[id].RecoveryPersistenceManager.SharedQueue.PauseProducing = false;
SendStatus(id);
}
}
}
catch (Exception exp)
{
if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsErrorEnabled)
{
LoggerManager.Instance.RecoveryLogger.Error("ShardRecoveryManager.SubmitState_StartOplog()", id + " : " + exp.ToString());
}
}
}
break;
}
}
else
{
if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsInfoEnabled)
{
LoggerManager.Instance.RecoveryLogger.Info("ShardRecoveryManager.SubmitState()", "Shard does not contain a job against id: " + id);
}
}
}
private RecoveryOperationStatus CancelRecoveryJob(RecoveryOperation opContext)
{
RecoveryOperationStatus status = new RecoveryOperationStatus(RecoveryStatus.Success);
status.JobIdentifier = opContext.JobIdentifer;
if (opContext != null)
{
try
{
#region End/Cancel job
if (_databaseJobMap.ContainsKey(opContext.JobIdentifer))
{
List <RecoveryJobBase> jobList = _databaseJobMap[opContext.JobIdentifer].JobList;
foreach (RecoveryJobBase job in jobList)
{
if (opContext.OpCode == RecoveryOpCodes.CancelJob)
{
switch (job.JobType)
{
case RecoveryJobType.DataBackup:
_databaseJobMap[opContext.JobIdentifer].RecoveryPersistenceManager.CloseBackupFile(job.Database, RecoveryFileState.Cancelled);
break;
//NOTE: diffbackup job is ignored under the assumption oplog file will close the file for it
}
}
//if (job.JobType != RecoveryJobType.DifferentialRestore)
{
try
{
if (job.State != ThreadState.Unstarted && job.State != ThreadState.Stopped && job.State != ThreadState.WaitSleepJoin)
{
job.Stop();
}
}
catch (ThreadAbortException)
{
Thread.ResetAbort();
}
catch (Exception exp)
{ }
try
{
job.Dispose();
}
catch (ThreadAbortException)
{
Thread.ResetAbort();
}
catch (Exception exp)
{ }
}
//else
//{
// _databaseJobMap[opContext.JobIdentifer].RecoveryPersistenceManager.IsJobActive = false;
// _databaseJobMap[opContext.JobIdentifer].RecoveryPersistenceManager.Dispose();
//}
}
#region rename folder
JobInfoObject infoObj = _databaseJobMap[opContext.JobIdentifer];
if (infoObj.JobType == RecoveryJobType.DataBackup)
{
switch (opContext.OpCode)
{
case RecoveryOpCodes.CancelJob:
infoObj.RenameRootFolder(RecoveryFileState.Cancelled);
break;
}
}
#endregion
// removing job from databaseJobMap
_databaseJobMap[opContext.JobIdentifer].RecoveryPersistenceManager.IsJobActive = false;
_databaseJobMap[opContext.JobIdentifer].Dispose();
_databaseJobMap.Remove(opContext.JobIdentifer);
}
#endregion
}
catch (Exception exp)
{
if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsErrorEnabled)
{
LoggerManager.Instance.RecoveryLogger.Error("ShardRecoveryManager.CancelRecoveryJob()", opContext.JobIdentifer + "_" + _databaseJobMap[opContext.JobIdentifer].ExecutionState.JobType + " : " + exp.ToString());
}
if (_databaseJobMap.ContainsKey(opContext.JobIdentifer))
{
_databaseJobMap[opContext.JobIdentifer].ExecutionState.Status = RecoveryStatus.Failure;
_databaseJobMap[opContext.JobIdentifer].ExecutionState.Message = "Failure during Cancel phase";
// commented because the config server will not have any job handler to recieve this message
//CheckJobState(opContext.JobIdentifer, true);
}
status.Status = RecoveryStatus.Failure;
status.Message = exp.ToString();
}
}
if (LoggerManager.Instance.RecoveryLogger != null && LoggerManager.Instance.RecoveryLogger.IsInfoEnabled)
{
LoggerManager.Instance.RecoveryLogger.Info("ShardRecoveryManager.CancelRecoveryJob()", status.ToString());
}
return(status);
}