/// <summary>
        /// Trigger the identified <see cref="IJob" /> (Execute it
        /// now) - with a volatile trigger.
        /// </summary>
        public virtual void TriggerJobWithVolatileTrigger(SchedulingContext ctxt, string jobName, string groupName,
                                                          JobDataMap data)
        {
            ValidateState();

            if (groupName == null)
            {
                groupName = SchedulerConstants.DefaultGroup;
            }

            Trigger trig =
                new SimpleTrigger(NewTriggerId(), SchedulerConstants.DefaultManualTriggers, jobName, groupName, DateTime.UtcNow,
                                  null, 0, TimeSpan.Zero);
            trig.Volatile = true;
            trig.ComputeFirstFireTimeUtc(null);
            if (data != null)
            {
                trig.JobDataMap = data;
            }

            bool collision = true;
            while (collision)
            {
                try
                {
                    resources.JobStore.StoreTrigger(ctxt, trig, false);
                    collision = false;
                }
                catch (ObjectAlreadyExistsException)
                {
                    trig.Name = NewTriggerId();
                }
            }

            NotifySchedulerThread(trig.GetNextFireTimeUtc());
            NotifySchedulerListenersScheduled(trig);
        }
        protected virtual void ClusterRecover(ConnectionAndTransactionHolder conn, IList failedInstances)
        {
            if (failedInstances.Count > 0)
            {
                long recoverIds = DateTime.UtcNow.Ticks;

                LogWarnIfNonZero(failedInstances.Count,
                                 "ClusterManager: detected " + failedInstances.Count + " failed or restarted instances.");
                try
                {
                    foreach (SchedulerStateRecord rec in failedInstances)
                    {
                        Log.Info("ClusterManager: Scanning for instance \"" + rec.SchedulerInstanceId +
                                 "\"'s failed in-progress jobs.");

                        IList firedTriggerRecs =
                            Delegate.SelectInstancesFiredTriggerRecords(conn, rec.SchedulerInstanceId);

                        int acquiredCount = 0;
                        int recoveredCount = 0;
                        int otherCount = 0;

                        ISet triggerKeys = new HashSet();

                        foreach (FiredTriggerRecord ftRec in firedTriggerRecs)
                        {
                            Key tKey = ftRec.TriggerKey;
                            Key jKey = ftRec.JobKey;

                            triggerKeys.Add(tKey);

                            // release blocked triggers..
                            if (ftRec.FireInstanceState.Equals(StateBlocked))
                            {
                                Delegate.UpdateTriggerStatesForJobFromOtherState(conn, jKey.Name, jKey.Group,
                                                                                 StateWaiting,
                                                                                 StateBlocked);
                            }
                            else if (ftRec.FireInstanceState.Equals(StatePausedBlocked))
                            {
                                Delegate.UpdateTriggerStatesForJobFromOtherState(conn, jKey.Name, jKey.Group,
                                                                                 StatePaused,
                                                                                 StatePausedBlocked);
                            }

                            // release acquired triggers..
                            if (ftRec.FireInstanceState.Equals(StateAcquired))
                            {
                                Delegate.UpdateTriggerStateFromOtherState(conn, tKey.Name, tKey.Group, StateWaiting,
                                                                          StateAcquired);
                                acquiredCount++;
                            }
                            else if (ftRec.JobRequestsRecovery)
                            {
                                // handle jobs marked for recovery that were not fully
                                // executed..
                                if (JobExists(conn, jKey.Name, jKey.Group))
                                {
                                    DateTime tempAux = new DateTime(ftRec.FireTimestamp);
                                    SimpleTrigger rcvryTrig =
                                        new SimpleTrigger(
                                            "recover_" + rec.SchedulerInstanceId + "_" + Convert.ToString(recoverIds++, CultureInfo.InvariantCulture),
                                            SchedulerConstants.DefaultRecoveryGroup, tempAux);
                                    rcvryTrig.Volatile = ftRec.TriggerIsVolatile;
                                    rcvryTrig.JobName = jKey.Name;
                                    rcvryTrig.JobGroup = jKey.Group;
                                    rcvryTrig.MisfireInstruction = MisfireInstruction.SimpleTrigger.FireNow;
                                    rcvryTrig.Priority = ftRec.Priority;
                                    JobDataMap jd = Delegate.SelectTriggerJobDataMap(conn, tKey.Name, tKey.Group);
                                    jd.Put(SchedulerConstants.FailedJobOriginalTriggerName, tKey.Name);
                                    jd.Put(SchedulerConstants.FailedJobOriginalTriggerGroup, tKey.Group);
                                    jd.Put(SchedulerConstants.FailedJobOriginalTriggerFiretimeInMillisecoonds, Convert.ToString(ftRec.FireTimestamp, CultureInfo.InvariantCulture));
                                    rcvryTrig.JobDataMap = jd;

                                    rcvryTrig.ComputeFirstFireTimeUtc(null);
                                    StoreTrigger(conn, null, rcvryTrig, null, false, StateWaiting, false, true);
                                    recoveredCount++;
                                }
                                else
                                {
                                    Log.Warn("ClusterManager: failed job '" + jKey +
                                             "' no longer exists, cannot schedule recovery.");
                                    otherCount++;
                                }
                            }
                            else
                            {
                                otherCount++;
                            }

                            // free up stateful job's triggers
                            if (ftRec.JobIsStateful)
                            {
                                Delegate.UpdateTriggerStatesForJobFromOtherState(conn, jKey.Name, jKey.Group,
                                                                                 StateWaiting,
                                                                                 StateBlocked);
                                Delegate.UpdateTriggerStatesForJobFromOtherState(conn, jKey.Name, jKey.Group,
                                                                                 StatePaused,
                                                                                 StatePausedBlocked);
                            }
                        }

                        Delegate.DeleteFiredTriggers(conn, rec.SchedulerInstanceId);


                        // Check if any of the fired triggers we just deleted were the last fired trigger
                        // records of a COMPLETE trigger.
                        int completeCount = 0;
                        foreach (Key triggerKey in triggerKeys)
                        {
                            if (
                                Delegate.SelectTriggerState(conn, triggerKey.Name, triggerKey.Group).Equals(
                                    StateComplete))
                            {
                                IList firedTriggers =
                                    Delegate.SelectFiredTriggerRecords(conn, triggerKey.Name, triggerKey.Group);
                                if (firedTriggers.Count == 0)
                                {
                                    SchedulingContext schedulingContext = new SchedulingContext();
                                    schedulingContext.InstanceId = instanceId;

                                    if (RemoveTrigger(conn, schedulingContext, triggerKey.Name, triggerKey.Group))
                                    {
                                        completeCount++;
                                    }
                                }
                            }
                        }
                        LogWarnIfNonZero(acquiredCount,
                                         "ClusterManager: ......Freed " + acquiredCount + " acquired trigger(s).");
                        LogWarnIfNonZero(completeCount,
                                         "ClusterManager: ......Deleted " + completeCount + " complete triggers(s).");
                        LogWarnIfNonZero(recoveredCount,
                                         "ClusterManager: ......Scheduled " + recoveredCount +
                                         " recoverable job(s) for recovery.");
                        LogWarnIfNonZero(otherCount,
                                         "ClusterManager: ......Cleaned-up " + otherCount + " other failed job(s).");


                        if (rec.SchedulerInstanceId.Equals(InstanceId) == false)
                        {
                            Delegate.DeleteSchedulerState(conn, rec.SchedulerInstanceId);
                        }
                    }
                }
                catch (Exception e)
                {
                    throw new JobPersistenceException("Failure recovering jobs: " + e.Message, e);
                }
            }
        }