public StateMachineController(LifeCycleState lifeCycleState, SchedulerState schedulerState,
ConnectivityState connectivityState)
{
_lifeCycleState = lifeCycleState;
_schedulerState = schedulerState;
_connectivityState = connectivityState;
}
public static void Start(CancellationToken ct)
{
LogManager.WriteToLog(new LogMessage()
{
Message = "JobScheduler Running .."
});
while (true)
{
Global.SchedulerStopHandle.WaitOne();
State = SchedulerState.Running;
if (ct.IsCancellationRequested)
{
State = SchedulerState.Stopped;
LogManager.WriteToLog(new LogMessage()
{
Message = "JobScheduler Stopped."
});
ct.ThrowIfCancellationRequested();
}
Job j = JobQueue.Dequeue();
if (j != null)
{
ExceuteJob(j);
}
Thread.Sleep(500);
}
}
protected void DoSchedulerStateChanged(SchedulerState state)
{
try
{
if (SystemContext.RemoteManageServer != null)
{
SystemContext.RemoteManageServer.SyncSchedulerState(this, null);
}
if (OnSchedulerStateChanged != null)
{
if (CLocalSystem.MainForm != null)
{
MethodInvoker form_invoker = delegate
{
OnSchedulerStateChanged(SystemContext, Name, state);
};
CLocalSystem.MainForm.Invoke(form_invoker);
}
else
{
OnSchedulerStateChanged(SystemContext, Name, state);
}
}
}
catch (Exception e)
{
System.Console.Out.WriteLine("CScheduler.DoSchedulerStateChanged Exception:{0}", e);
}
}
private static PageToCrawl GetNextInner(SchedulerState state, int attempt)
{
if (attempt > 10)
{
return(null);
}
var pageToCrawlKey = PageToCrawlKey(state.SiteName);
var trans = CreateTransaction(state);
trans.AddCondition(Condition.KeyExists(pageToCrawlKey));
var taskJson = trans.ListRightPopAsync(pageToCrawlKey);
var committed = trans.ExecuteAsync().Result;
if (!committed)
{
Thread.Sleep(500);
return(GetNextInner(state, attempt + 1));
}
var json = taskJson.Result;
var pageToCrawl = JsonConvert.DeserializeObject <PageToCrawl>(json);
pageToCrawl.PageBag = new ExpandoObject();
return(pageToCrawl);
}
private IIncrementalSchedulingState LoadOrReuseInternal(
FileEnvelopeId atomicSaveToken,
PipGraph pipGraph,
IConfiguration configuration,
ContentHash preserveOutputSalt,
string incrementalSchedulingStatePath,
SchedulerState schedulerState)
{
Contract.Requires(pipGraph != null);
Contract.Requires(!string.IsNullOrWhiteSpace(incrementalSchedulingStatePath));
Contract.Assert(m_analysisMode || configuration != null);
if (!m_analysisMode && schedulerState != null && schedulerState.IncrementalSchedulingState != null)
{
IIncrementalSchedulingState reusedState = schedulerState.IncrementalSchedulingState.Reuse(m_loggingContext, pipGraph, configuration, preserveOutputSalt, m_tempDirectoryCleaner);
if (reusedState != null)
{
return(reusedState);
}
}
return(GraphAgnosticIncrementalSchedulingState.Load(
m_loggingContext,
atomicSaveToken,
pipGraph,
configuration,
preserveOutputSalt,
incrementalSchedulingStatePath,
analysisModeOnly: m_analysisMode,
tempDirectoryCleaner: m_tempDirectoryCleaner));
}
static void Init()
{
setInitSchedule();
readyQueue = readyQueue.Add(0, new Set <Bar>(schedule["start"]));
bars = schedule.Values;
//bars = readyQueue[0];
state = SchedulerState.WAITING;
}
public async Task SetSchedulerState(SchedulerState state, CancellationToken cancellationToken)
{
using var session = Store.OpenAsyncSession();
var scheduler = await session.LoadAsync <Scheduler>(InstanceName, cancellationToken);
scheduler.State = state;
await session.SaveChangesAsync(cancellationToken);
}
public WidgetContainerAutoFetchState Clone()
{
WidgetContainerAutoFetchState state = new WidgetContainerAutoFetchState
{
SchedulerState = SchedulerState.Clone()
};
return(state);
}
public Scheduler(Dispatcher dispatcher)
{
_dispatcher = dispatcher;
_lock = new object();
_state = SchedulerState.Terminated;
_queue = new PriorityQueue <Schedule>();
_lastScheduletId = 0;
_scheduleIds = new HashSet <uint>();
}
/// <summary>
/// Initializes a new instance of the <see cref="SchedulerContext"/> class.
/// </summary>
/// <param name="scheduler">The scheduler which this context belongs to.</param>
public SchedulerContext(IScheduler scheduler)
{
_SchedulerState = SchedulerState.Inactive;
_Callback = null;
_ParamCallback = null;
_Scheduler = scheduler;
_IsDisposed = false;
_AutoResetEvent = new AutoResetEvent(false);
}
public Scheduler(Dispatcher dispatcher)
{
_dispatcher = dispatcher;
_lock = new object();
_state = SchedulerState.Terminated;
_queue = new PriorityQueue<Schedule>();
_lastScheduletId = 0;
_scheduleIds = new HashSet<uint>();
}
/// <summary>
/// Changes the scheduler state of this <see cref="SchedulerContext"/>.
/// </summary>
public void ChangeSchedulerState(SchedulerState state)
{
_SchedulerState = state;
if (_SchedulerState == SchedulerState.Active)
{
_AutoResetEvent.Set();
}
}
public void TestSetup()
{
state = new SchedulerState(Task.CompletedTask, CancellationToken.None);
job = Substitute.For <IStreamJob>();
launcher = Substitute.For <IJobLauncher>();
handler = new JobHandler(launcher);
}
/// <summary>
/// Same behavior as <see cref="LoadOrReuse"/>, but the file envelopeId is ignored
/// </summary>
/// <remarks>
/// Only available when the factory is constructed with analysis mode on
/// </remarks>
public IIncrementalSchedulingState LoadOrReuseIgnoringFileEnvelope(
PipGraph pipGraph,
IConfiguration configuration,
ContentHash preserveOutputSalt,
string incrementalSchedulingStatePath,
SchedulerState schedulerState)
{
Contract.Assert(m_analysisMode);
return(LoadOrReuseInternal(FileEnvelopeId.Invalid, pipGraph, configuration, preserveOutputSalt, incrementalSchedulingStatePath, schedulerState));
}
public DashboardAutoFetchState Clone()
{
DashboardAutoFetchState state = new DashboardAutoFetchState
{
SchedulerState = SchedulerState.Clone(),
IsDominatingSchedule = IsDominatingSchedule
};
return(state);
}
/// <summary>
/// Runs the scheduler using the instance member PipGraph and Configuration objects. This will also carry over
/// any state from any previous run such as the cache
/// </summary>
public ScheduleRunResult RunScheduler(SchedulerTestHooks testHooks = null, SchedulerState schedulerState = null, RootFilter filter = null, TempCleaner tempCleaner = null)
{
if (m_graphWasModified || m_lastGraph == null)
{
m_lastGraph = PipGraphBuilder.Build();
XAssert.IsNotNull(m_lastGraph, "Failed to build pip graph");
}
m_graphWasModified = false;
return(RunSchedulerSpecific(m_lastGraph, testHooks, schedulerState, filter, tempCleaner));
}
public static void Clear(SchedulerState state)
{
var database = state.Connection.GetDatabase();
var server = state.Connection.GetServer(state.Connection.GetEndPoints().First());
foreach (var key in server.Keys(state.Database, pattern: CrawledPageKey(state.SiteName, "*")))
{
database.KeyDelete(key);
}
database.KeyDelete(PageToCrawlKey(state.SiteName));
}
/// <summary>
/// Loads an existing instance of <see cref="IIncrementalSchedulingState"/> from a given file or reuse it from a given <see cref="SchedulerState"/>.
/// </summary>
public IIncrementalSchedulingState LoadOrReuse(
FileEnvelopeId atomicSaveToken,
PipGraph pipGraph,
IConfiguration configuration,
ContentHash preserveOutputSalt,
string incrementalSchedulingStatePath,
SchedulerState schedulerState)
{
Contract.Requires(atomicSaveToken.IsValid);
return(LoadOrReuseInternal(atomicSaveToken, pipGraph, configuration, preserveOutputSalt, incrementalSchedulingStatePath, schedulerState));
}
public void Stop()
{
lock (_lock)
{
if (_state != SchedulerState.Running)
{
return;
}
_state = SchedulerState.Closing;
}
}
public Scheduler(Dispatcher dispatcher)
{
if (dispatcher == null)
throw new ArgumentNullException("dispatcher");
_dispatcher = dispatcher;
_lock = new object();
_state = SchedulerState.Terminated;
_queue = new PriorityQueue<Schedule>();
_lastScheduletId = 0;
_scheduleIds = new HashSet<uint>();
_lastScheduletId = Constants.SchedulerStartId;
}
public void SetUp()
{
var config = new ConfigurationOptions()
{
EndPoints = { { _host, _port } }, Password = _password, DefaultDatabase = _database, AllowAdmin = true
};
_state = SchedulerFunc.Create(config, "Test", _database);
var server = _state.Connection.GetServer(_state.Connection.GetEndPoints().First());
server.FlushDatabase(_state.Database);
}
public static void Add(SchedulerState state, PageToCrawl page)
{
var json = JsonConvert.SerializeObject(page);
var url = page.Uri.AbsoluteUri;
var trans = CreateTransaction(state);
var crawledPageKey = CrawledPageKey(state.SiteName, url);
var pageToCrawlKey = PageToCrawlKey(state.SiteName);
trans.AddCondition(Condition.KeyNotExists(crawledPageKey));
trans.StringSetAsync(crawledPageKey, "");
trans.ListLeftPushAsync(pageToCrawlKey, json);
trans.ExecuteAsync().Wait();
}
public void TestSetup()
{
source1 = new TaskCompletionSource <bool>();
source2 = new TaskCompletionSource <bool>();
source3 = new TaskCompletionSource <bool>();
source4 = new TaskCompletionSource <bool>();
cancellation = new CancellationTokenSource();
cancellation.Token.Register(() => source4.TrySetResult(true));
state = new SchedulerState(source4.Task, cancellation.Token);
waiter = new JobWaiter(100.Milliseconds(), 2);
}
static void StartExecute([Domain("bars")] Bar b)
{
if (TimeUpdate.time >= b.earliestStartTime &&
TimeUpdate.time < (b.deadline - b.duration))
{
state = SchedulerState.EXECUTING;
UpdateReadyQueue(b, TimeUpdate.time);
Process.state = ProcessState.DEFERRED;
}
else
{
TimeUpdate.tickEnabled = true;
}
}
public void Synchronize(SchedulerState state)
{
foreach (var streamState in statesProvider.GetStates())
{
if (state.AllJobs.ContainsKey(streamState.Name))
{
continue;
}
var newJob = state.AllJobs[streamState.Name] = jobFactory.CreateJob(streamState);
jobLauncher.LaunchWaitJob(newJob, state);
}
}
public void HandleCompletedJob(Task completedJobTask, SchedulerState state)
{
if (completedJobTask is Task <SendingJobResult> sendingJobTask)
{
state.SendingJobs.Remove(completedJobTask);
launcher.LaunchWaitJob(sendingJobTask.GetAwaiter().GetResult().Job, state);
}
if (completedJobTask is Task <WaitingJobResult> waitingJobTask)
{
state.WaitingJobs.Remove(completedJobTask);
launcher.LaunchSendJob(waitingJobTask.GetAwaiter().GetResult().Job, state);
}
completedJobTask.GetAwaiter().GetResult();
}
public static EngineState CreateNew(EngineSchedule engineSchedule)
{
Contract.Requires(engineSchedule != null);
var schedulerState = new SchedulerState(engineSchedule.Scheduler);
return(new EngineState(
engineSchedule.Scheduler.PipGraph.GraphId,
engineSchedule.Context.StringTable,
engineSchedule.Context.PathTable,
engineSchedule.Context.SymbolTable,
engineSchedule.Context.QualifierTable,
engineSchedule.PipTable,
engineSchedule.Scheduler.PipGraph,
engineSchedule.MountPathExpander,
schedulerState,
engineSchedule.Context.HistoricTableSizes));
}
public void Shutdown()
{
lock (_lock)
{
if (_state == SchedulerState.Terminated)
{
return;
}
#if _DEBUG_SCHEDULER
Logger.Debug(string.Format("Scheduler: Stoping {0} scheduler.", Name));
#endif
_state = SchedulerState.Terminated;
_queue.Clear();
_scheduleIds.Clear();
_lastScheduletId = 0;
Monitor.Pulse(_lock);
}
}
public void Start()
{
lock (_lock)
{
if (_state == SchedulerState.Running)
{
return;
}
#if _DEBUG_SCHEDULER
Logger.Debug(string.Format("Scheduler: Starting {0} scheduler.", Name));
#endif
_state = SchedulerState.Running;
_thread = new Thread(Run)
{
IsBackground = false
};
_thread.Start();
}
}
public void TestSetup()
{
state1 = Substitute.For <IStreamState>();
state2 = Substitute.For <IStreamState>();
state3 = Substitute.For <IStreamState>();
state1.Name.Returns("state1");
state2.Name.Returns("state2");
state3.Name.Returns("state3");
statesProvider = Substitute.For <IStreamStatesProvider>();
statesProvider.GetStates().Returns(new[] { state1, state2, state3 });
jobFactory = Substitute.For <IStreamJobFactory>();
jobLauncher = Substitute.For <IJobLauncher>();
state = new SchedulerState(Task.CompletedTask, CancellationToken.None);
synchronizer = new StateSynchronizer(statesProvider, jobFactory, jobLauncher);
}
private List <Task> GetTasksToWaitFor(SchedulerState state)
{
var result = new List <Task>();
result.AddRange(state.SendingJobs);
if (result.Count < maxJobParallelism)
{
result.AddRange(state.WaitingJobs);
}
if (result.Count == 0)
{
result.Add(Task.Delay(defaultWait));
}
result.Add(state.CancellationTask);
return(result);
}
private EngineState(
Guid graphId,
StringTable stringTable,
PathTable pathTable,
SymbolTable symbolTable,
QualifierTable qualifierTable,
PipTable pipTable,
PipGraph pipGraph,
MountPathExpander mountPathExpander,
SchedulerState schedulerState,
HistoricTableSizes historicTableSizes,
FileContentTable fileContentTable)
{
Contract.Requires(graphId != default(Guid), "GraphId is not unique enough to be represented in EngineState");
Contract.Requires(stringTable != null);
Contract.Requires(pathTable != null);
Contract.Requires(symbolTable != null);
Contract.Requires(qualifierTable != null);
Contract.Requires(stringTable == pathTable.StringTable);
Contract.Requires(pathTable.StringTable == symbolTable.StringTable);
Contract.Requires(pathTable.StringTable == qualifierTable.StringTable);
Contract.Requires(pipTable != null);
Contract.Requires(!pipTable.IsDisposed);
Contract.Requires(pipGraph != null);
Contract.Requires(mountPathExpander != null);
Contract.Requires(schedulerState != null);
Contract.Requires(historicTableSizes != null);
Contract.Requires(fileContentTable != null);
m_stringTable = stringTable;
m_pathTable = pathTable;
m_symbolTable = symbolTable;
m_qualifierTable = qualifierTable;
m_pipTable = pipTable;
m_pipGraph = pipGraph;
m_mountPathExpander = mountPathExpander;
m_schedulerState = schedulerState;
m_graphId = graphId;
m_historicTableSizes = historicTableSizes;
m_fileContentTable = fileContentTable;
}
static void Match([Domain("bars")] Bar b)
{
Set <Bar> readyBars;
if (readyQueue.ContainsKey(TimeUpdate.time))
{
readyBars = readyQueue[TimeUpdate.time];
bars = readyBars;
if (readyBars.Contains(b))
{
state = SchedulerState.WAITING;
}
else
{
state = SchedulerState.COMPLETED;
}
}
else
{
state = SchedulerState.COMPLETED;
}
}
public void Shutdown()
{
lock (_lock)
{
if (_state == SchedulerState.Terminated)
throw new InvalidOperationException("The scheduler is already terminated.");
#if DEBUG_SCHEDULER
Logger.Debug("Shutting down scheduler.");
#endif
_state = SchedulerState.Terminated;
_queue.Clear();
_scheduleIds.Clear();
_lastScheduletId = 0;
Monitor.Pulse(_lock);
}
}
public void Start()
{
lock (_lock)
{
if (_state == SchedulerState.Running)
return;
#if _DEBUG_SCHEDULER
Logger.Debug(string.Format("Scheduler: Starting {0} scheduler.", Name));
#endif
_state = SchedulerState.Running;
_thread = new Thread(Run) { IsBackground = false };
_thread.Start();
}
}
public void Stop()
{
lock (_lock)
{
if (_state != SchedulerState.Running)
return;
_state = SchedulerState.Closing;
}
}
public void Stop()
{
lock (_lock)
{
if (_state != SchedulerState.Running)
throw new InvalidOperationException("The scheduler is already stoped.");
#if DEBUG_SCHEDULER
Logger.Debug("Stoping scheduler.");
#endif
_state = SchedulerState.Closing;
}
}
/// <summary>
/// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
/// </summary>
public void Dispose()
{
// change state to inactive
_SchedulerState = SchedulerState.Inactive;
// remove callback delegate invocation list
_Callback = null;
_ParamCallback = null;
// set IsDisposed flag
_IsDisposed = true;
}
/// <summary>
/// Changes the scheduler state of this <see cref="SchedulerContext"/>.
/// </summary>
public void ChangeSchedulerState(SchedulerState state)
{
_SchedulerState = state;
if (_SchedulerState == SchedulerState.Active) {
_AutoResetEvent.Set();
}
}
public void Shutdown()
{
lock (_lock)
{
if (_state == SchedulerState.Terminated)
return;
#if _DEBUG_SCHEDULER
Logger.Debug(string.Format("Scheduler: Stoping {0} scheduler.", Name));
#endif
_state = SchedulerState.Terminated;
_queue.Clear();
_scheduleIds.Clear();
_lastScheduletId = 0;
Monitor.Pulse(_lock);
}
}
static void Completed()
{
bars = readyQueue[TimeUpdate.time];
state = SchedulerState.WAITING;
}
static void Init()
{
setInitSchedule();
readyQueue = readyQueue.Add(0,new Set<Bar>(schedule["start"]));
bars = schedule.Values;
//bars = readyQueue[0];
state = SchedulerState.WAITING;
}
static void Match([Domain("bars")]Bar b)
{
Set<Bar> readyBars;
if (readyQueue.ContainsKey(TimeUpdate.time))
{
readyBars = readyQueue[TimeUpdate.time];
bars = readyBars;
if (readyBars.Contains(b))
{
state = SchedulerState.WAITING;
}
else
state = SchedulerState.COMPLETED;
}
else
state = SchedulerState.COMPLETED;
}
static void StartExecute([Domain("bars")]Bar b)
{
if (TimeUpdate.time >= b.earliestStartTime &&
TimeUpdate.time < (b.deadline - b.duration))
{
state = SchedulerState.EXECUTING;
UpdateReadyQueue(b, TimeUpdate.time);
Process.state = ProcessState.DEFERRED;
}
else
{
TimeUpdate.tickEnabled = true;
}
}
public void Start()
{
lock (_lock)
{
if (_state == SchedulerState.Running)
throw new InvalidOperationException("The scheduler is already running.");
if (_state == SchedulerState.Closing)
{
_state = SchedulerState.Running;
#if DEBUG_SCHEDULER
Logger.Debug("Resuming scheduler.");
#endif
}
else
{
#if DEBUG_SCHEDULER
Logger.Debug("Starting scheduler.");
#endif
_state = SchedulerState.Running;
_thread = new Thread(Run) { IsBackground = false };
_thread.Start();
}
}
}
