public PackagesController(INugetApi nugetApi, IJobQueueFactory queueFactory, IGetSetQuerier <Package> redisQuery, ILatestPackagesIndex latestPackages) { this.nugetApi = nugetApi; this.packageCrawlerJobQueue = queueFactory.ForQueueName(Constants.Redis.PackageCrawlerJobQueueName); this.redisQuerier = redisQuery; this.latestPackages = latestPackages; }
public override bool OnStart() { try { // Set the maximum number of concurrent connections ServicePointManager.DefaultConnectionLimit = 12; RoleStartupHelper.StartDiagnostics(); InitializeBlockStorage(); StartBlockUpdateService(); StartStoreWorkerService(); StoreConfiguration.DefaultStoreConfiguration.StoreManagerType = typeof(AzureStoreManager); var jobQueueConnectionString = RoleEnvironment.GetConfigurationSettingValue("ManagementDatabaseConnectionString"); Trace.TraceInformation("Initializing SQL job queue"); _jobQueue = new SqlJobQueue(jobQueueConnectionString, RoleEnvironment.CurrentRoleInstance.Id); Trace.TraceInformation("Job queue initialized"); EventLog.WriteEntry("AzureEventLogs", "Worker role started", EventLogEntryType.Information); return(base.OnStart()); } catch (Exception ex) { Trace.TraceError("WorkerRole OnStart failed: " + ex); EventLog.WriteEntry("AzureEventLogs", "OnStart failed for StoreWorker role: Exception:" + ex.GetType().FullName + ", Message:" + ex.Message, EventLogEntryType.Error, 0); Trace.TraceError("Sleeping for long enough to allow errors to flush"); Thread.Sleep(TimeSpan.FromMinutes(1.5)); Trace.TraceError("Aborting role startup"); return(false); } }
public const int ClientTimeOutInSeconds = 10; ////TODO: This should not be hardcoded and must be moved to congig public async Task <bool> StartAsync(IJobQueue job) { var webScrapeJob = (WebScrapeJob)job; using (HttpClient client = new HttpClient()) { // Timeout if site doesn't respond client.Timeout = TimeSpan.FromSeconds(ClientTimeOutInSeconds); var html = await client.GetStringAsync(webScrapeJob.Url); // Console.WriteLine(html); var doc = new HtmlDocument(); doc.LoadHtml(html); webScrapeJob.Scrapes = new List <string>(); if (!string.IsNullOrEmpty(webScrapeJob.Selector)) { foreach (var selectedNode in doc.DocumentNode.QuerySelectorAll(webScrapeJob.Selector)) { webScrapeJob.Scrapes.Add(selectedNode.InnerHtml); } } return(true); } }
public static void CreateGanttChartForRessource(JobInProgress jobInProgress, IJobQueue scopeQueue, Agent agent) { List <GanttChartItem> ganttData = new List <GanttChartItem>(); if (jobInProgress.IsSet) { ganttData.AddRange(CreateGanttProcessingQueueLog(new[] { jobInProgress.GanttItem }, true, "Processing", jobInProgress, agent)); // add from scope } ganttData.AddRange(CreateGanttProcessingQueueLog(jobInProgress.GanttItems.ToArray(), false, "ReadyElement", jobInProgress, agent)); var jobs = scopeQueue.GetAllJobs().OrderBy(x => x.Job.Priority(agent.CurrentTime)).ToList(); ganttData.AddRange(CreateGanttProcessingQueueLog(jobs.ToArray(), false, "ScopeQueue", jobInProgress, agent)); CustomFileWriter.WriteToFile($"Logs//ResourceScheduleAt-{agent.CurrentTime}.log", JsonConvert.SerializeObject(ganttData).Replace("[", "").Replace("]", ",")); if (jobs.Count > 0) { if (jobs.First().ScopeConfirmation.GetScopeStart() < jobInProgress.ResourceIsBusyUntil) { agent.DebugMessage("Seems wrong"); } } }
public IJobQueue Register() { IConsumerContext ctx = Turbine.Consumer.AppUtility.GetConsumerContext(); Guid consumerId = ctx.Id; String hostname = ctx.Hostname; Debug.WriteLine(String.Format("Register as {0}, {1}", consumerId, hostname), this.GetType().Name); // // NOTE: Amazon InstanceID and AMI removed // using (TurbineModelContainer container = new TurbineModelContainer()) { container.JobConsumers.AddObject(new JobConsumer() { Id = consumerId, hostname = hostname, AMI = "", instance = "", processId = Process.GetCurrentProcess().Id.ToString(), status = "up", keepalive = DateTime.UtcNow } ); container.SaveChanges(); } queue = new DBJobQueue(); return(queue); }
public IJobQueue Register(IConsumerRun run) { this.run = run; IConsumerContext ctx = Turbine.Consumer.AppUtility.GetConsumerContext(); //Guid consumerId = ctx.Id; //Guid consumerId = Guid.NewGuid(); String hostname = ctx.Hostname; String appName = run.SupportedApplications.ElementAtOrDefault(0); Debug.WriteLine(String.Format("Register({0}) as {1}, {2}", appName, run.ConsumerId, hostname), this.GetType().Name); using (ProducerContext db = new ProducerContext()) { // TODO: Registering as a single application is dubious. // IF support multiple apps in single consumer need to expose that via the database ( update SCHEMA!! ) var app = db.Applications.Single(a => a.Name == appName); var consumer = new Turbine.Data.Entities.JobConsumer { Application = app, Id = run.ConsumerId, hostname = hostname, processId = System.Diagnostics.Process.GetCurrentProcess().Id, keepalive = DateTime.UtcNow, status = "up" }; db.Consumers.Add(consumer); db.SaveChanges(); } queue = AppUtility.GetJobQueue(run); //((DBJobQueue)queue).consumerId = consumerId; return(queue); }
public JobRequestConsumer(IJobQueue queue, IServiceBus bus, int delaySeconds, IWindsorContainer container) { this.queue = queue; this.bus = bus; this.delaySeconds = delaySeconds; this.container = container; }
public static async Task <IProjectManager> Load(string projectsDir, IJobQueue jobQueue, CancellationToken cancellationToken) { var projects = new Dictionary <Guid, IProject>(); Directory.CreateDirectory(projectsDir); foreach (var projectDir in Directory.EnumerateDirectories(projectsDir)) { cancellationToken.ThrowIfCancellationRequested(); if (!Guid.TryParse(Path.GetFileName(projectDir), out var id)) { continue; } var configStr = await File.ReadAllTextAsync(Path.Combine(projectDir, "project.json"), cancellationToken); var config = JsonConvert.DeserializeObject <ProjectConfig>(configStr); var project = new Project(Path.GetFullPath(projectDir), id, jobQueue, config); projects[id] = project; } return(new ProjectManager(projectsDir, jobQueue, projects, cancellationToken)); }
public TaskScheduler(Func <IJobQueue> JobQueueFactory, ITimeProvider timeProvider = null, Thread mainThread = null) { MainThread = mainThread ?? Thread.CurrentThread; TimeProvider = timeProvider ?? Time.DefaultTimeProvider; BackgroundTaskOnComplete = _BackgroundTaskOnComplete; OnResolvedDispatcher = _OnResolvedDispatcher; OnResolvedDispatcher_SkipQueue = _OnResolvedDispatcher_SkipQueue; mOnResolvedDispatcher_SkipQueue = GetType().GetMethod("_OnResolvedDispatcher_SkipQueue_Generic", BindingFlags.Instance | BindingFlags.NonPublic); _JobQueue = JobQueueFactory(); _SleepWorker = new Internal.WorkerThread <PriorityQueue <SleepItem> >( SleepWorkerThreadFunc, ThreadPriority.AboveNormal, "TaskScheduler Sleep Provider" ); SynchronizationContext = new TaskSchedulerSynchronizationContext(this); if (!_Default.IsValueCreated) { _Default.Value = this; WorkItemQueueTarget.SetDefaultIfNone(this); } else { _Default.Value = null; } }
public Project(string dir, Guid id, IJobQueue jobQueue, ProjectConfig config) { ProjectDir = dir; this.jobQueue = jobQueue; Id = id; Config = config; }
public TaskScheduler(Func <IJobQueue> JobQueueFactory) { _JobQueue = JobQueueFactory(); _SleepWorker = new Internal.WorkerThread <PriorityQueue <SleepItem> >( SleepWorkerThreadFunc, ThreadPriority.AboveNormal, "TaskScheduler Sleep Provider" ); }
private ProjectManager(string projectsDir, IJobQueue jobQueue, IDictionary <Guid, IProject> projects, CancellationToken cancellationToken) { this.projectsDir = projectsDir; this.jobQueue = jobQueue; this.projects = new ConcurrentDictionary <Guid, IProject>(projects); this.cancellationToken = cancellationToken; }
/// <summary> /// Initializes a new instance of the <see cref="JobScheduler"/> class. /// </summary> /// <param name="jobQueue">The job queue.</param> /// <param name="getTimeFactory">The get time factory.</param> /// <param name="log">The log factory.</param> public JobScheduler(IJobQueue jobQueue, IGetTimeFactory getTimeFactory, ILogger log) { _jobQueue = jobQueue; _getTime = getTimeFactory; _log = log; }
/// <summary> /// Initializes a new instance of the <see cref="JobScheduler"/> class. /// </summary> /// <param name="jobQueue">The job queue.</param> /// <param name="getTimeFactory">The get time factory.</param> /// <param name="logFactory">The log factory.</param> public JobScheduler(IJobQueue jobQueue, IGetTimeFactory getTimeFactory, ILogFactory logFactory) { _jobQueue = jobQueue; _getTime = getTimeFactory; _logFactory = logFactory; }
public QueueConfiguration(IJobQueue @default, IDictionary<Type, IJobQueue> activitySpecificQueues) { if (@default == null) throw new ArgumentNullException("default"); if (activitySpecificQueues == null) throw new ArgumentNullException("activitySpecificQueues"); Default = @default; ActivitySpecificQueues = activitySpecificQueues; }
public FileSystemWatcherService(ILogger <FileSystemWatcherService> logger, IOptionsMonitor <FolderSettings> folderSettings, IJobQueue jobQueue, IJobBuilder job) { _logger = logger; _folderSettings = folderSettings; _jobQueue = jobQueue; _job = job; }
public void SetUp() { _tokenSource = new CancellationTokenSource(); _testJobQueueMonitor = new TestMonitor(); _jobQueueConfig = new JobQueueConfig(_tokenSource.Token, MaxConsurrentJobs); _jobQueue = new JobQueue(_testJobQueueMonitor, _jobQueueConfig); Task.Factory.StartNew(_jobQueue.TryRun); }
public SessionDebugManager(JobExecutor jobExecutor, IJobQueue jobQueue, LaunchAndAttachFlow launchAndAttachFlow, IDebugSession debugSession) { _jobExecutor = jobExecutor; _jobQueue = jobQueue; _launchAndAttachFlow = launchAndAttachFlow; Session = debugSession; }
public JobsHostedService( ILogger <JobsHostedService> logger, IOptions <JobsContext> options, IJobQueue jobQueue) { Logger = logger; Options = options; JobQueue = jobQueue; }
public TelegramResponseHandler(TelegramBotClient client, IJenkinsService jenkinsService, IJobQueue jobQueue, IList <MenuItem> menu, IList <long> allowedChatIds, string jenkinsProtocol, string jenkinsFqdn) { _client = client ?? throw new ArgumentNullException(nameof(client)); _jenkinsService = jenkinsService ?? throw new ArgumentNullException(nameof(jenkinsService)); _menu = menu ?? throw new ArgumentNullException(nameof(menu)); _jenkinsProtocol = jenkinsProtocol ?? throw new ArgumentNullException(nameof(jenkinsProtocol)); _jenkinsFqdn = jenkinsFqdn ?? throw new ArgumentNullException(nameof(jenkinsFqdn)); _jobQueue = jobQueue ?? throw new ArgumentNullException(nameof(jobQueue)); _allowedChatIds = allowedChatIds; }
public Factory(IJobQueue jobQueue, RefreshVariableJob.Factory refreshVariableJobFactory, IDebugSessionContext debugSessionContext, VariableExpander.Factory variableExpanderFactory) { _jobQueue = jobQueue; _refreshVariableJobFactory = refreshVariableJobFactory; _debugSessionContext = debugSessionContext; _variableExpanderFactory = variableExpanderFactory; }
public BreakpointView(BindBreakpointJob.Factory bindBreakpointJobFactory, BreakpointDeleteJob.Factory breakpointDeleteJobFactory, IDebugSessionContext debugSessionContext, IJobQueue jobQueue, JoinableTaskContext taskContext) { _bindBreakpointJobFactory = bindBreakpointJobFactory; _breakpointDeleteJobFactory = breakpointDeleteJobFactory; _debugSessionContext = debugSessionContext; _jobQueue = jobQueue; _taskContext = taskContext; }
public WriteOnlyTransaction( IConnectionProvider connectionProvider, IJobQueue queue) { Guard.ThrowIfNull(connectionProvider, nameof(connectionProvider)); Guard.ThrowIfNull(queue, nameof(queue)); _connectionProvider = connectionProvider; _queue = queue; _commandQueue = new Queue <Action <NpgsqlConnection, NpgsqlTransaction> >(); }
VariableEntry(IVariableDataSource dataSource, IJobQueue jobQueue, RefreshVariableJob.Factory refreshVariableJobFactory, IDebugSessionContext debugSessionContext, IVariableExpander variableExpander) { _dataSource = dataSource; _jobQueue = jobQueue; _refreshVariableJobFactory = refreshVariableJobFactory; _debugSessionContext = debugSessionContext; _variableExpander = variableExpander; }
public JobOrchestrator(IDebugSessionContext debugSessionContext, IJobQueue queue, ProgramStoppedJob.Factory programStoppedJobFactory, BroadcastDebugEventJob.Factory broadcastDebugEventJobFactory, ProgramTerminatedJob.Factory programTerminatedJobFactory) { _debugSessionContext = debugSessionContext; _queue = queue; _programStoppedJobFactory = programStoppedJobFactory; _broadcastDebugEventJobFactory = broadcastDebugEventJobFactory; _programTerminatedJobFactory = programTerminatedJobFactory; }
private UdpClient(UdpTransportConfiguration configuration, List <Socket> multicastSockets, List <Socket> unicastSockets, IJobQueue <UdpUnicastJob> unicastJobQueue, IObjectPool <byte[]> sendReceiveBufferPool, IAuditAggregator <double> inboundBytesAggregator, IAuditAggregator <double> outboundBytesAggregator, IAuditAggregator <double> inboundReceiveProcessDispatchLatencyAggregator) { this.configuration = configuration; this.multicastSockets = multicastSockets; this.unicastSockets = unicastSockets; this.unicastJobQueue = unicastJobQueue; this.sendReceiveBufferPool = sendReceiveBufferPool; this.inboundBytesAggregator = inboundBytesAggregator; this.outboundBytesAggregator = outboundBytesAggregator; this.inboundReceiveProcessDispatchLatencyAggregator = inboundReceiveProcessDispatchLatencyAggregator; this.sendArgsPool = ObjectPool.CreateStackBacked(() => new SocketAsyncEventArgs()); }
/// <summary> /// Initializes a new instance of the <see cref="QueueExecutor"/> /// class. /// </summary> /// <param name="queue">The <see cref="IJobQueue"/> this /// <see cref="QueueExecutor"/> will execute the elements /// within.</param> /// <exception cref="ArgumentNullException">queue is null.</exception> public QueueExecutor(IJobQueue queue) { if (queue == null) { throw new ArgumentNullException("queue"); } AutoStart = false; _stop = false; _queue = queue; _queue.JobAdded += job_added; }
public StorageConnection( IConnectionProvider connectionProvider, IJobQueue queue, PostgreSqlStorageOptions options) { Guard.ThrowIfNull(connectionProvider, nameof(connectionProvider)); Guard.ThrowIfNull(queue, nameof(queue)); Guard.ThrowIfNull(options, nameof(options)); _connectionProvider = connectionProvider; _queue = queue; }
public void Initialize() { _cancellationTokenSource = new CancellationTokenSource(); _cancellationToken = _cancellationTokenSource.Token; _jobQueue = new JobQueue(); _jobId = Guid.NewGuid(); _jobManager = new JobManager(); _consumer = new Consumer(_jobQueue, _cancellationToken); Task.Run(() => { _consumer.ConsumeJobs(); }); }
public ObserveAndNotifyJob(IJobQueue jobQueue, Func <bool> predicate, Action callback, string logDetails) { _jobQueue = jobQueue; _predicate = predicate; _callback = callback; _logDetails = logDetails ?? ""; _timer = new Timer(POLL_INTERVAL_MS); _timer.AutoReset = false; _timer.Elapsed += TimerExpired; }
public void FillQueue(IJobQueue queue) { var submissionInfos = new WaitingSubmissionsSortedByTimeAsc() .Load(session) .List <Submission>() .Select(x => new SubmissionInfo(x)); foreach (var submissionInfo in submissionInfos) { queue.Enqueue(submissionInfo, Priority.Normal); } }
public Endpoint( IOptions <EndpointConfiguration <TEndpoint> > configuration, ISerializer <TInput> inputSerializer, ISerializer <TOutput> outputSerializer, IJobRepository repository, IJobQueue queue) { _configuration = configuration.Value; _inputSerializer = inputSerializer; _outputSerializer = outputSerializer; _repository = repository; _queue = queue; }
public PackageManagerProxy(IPackageManagerProvider packageManagerProvider, IPackageManagerSettings packageManagerSettings, IClientMessenger clientMessenger, IJobQueue jobQueue, IJobFactory jobFactory, IPackageList packageList) { _packageManagerProvider = packageManagerProvider; _packageManagerSettings = packageManagerSettings; _clientMessenger = clientMessenger; _jobQueue = jobQueue; _jobFactory = jobFactory; _packageList = packageList; }
public JobPump( IDispatcher dispatcher, IEventStream eventStream, IJobQueue queue) { if(dispatcher == null) throw new ArgumentNullException("dispatcher"); if(eventStream == null) throw new ArgumentNullException("eventStream"); if (queue == null) throw new ArgumentNullException("queue"); _dispatcher = dispatcher; _eventStream = eventStream; _queue = queue; _throttle = queue.Configuration.MaxWorkers == 0 ? int.MaxValue : queue.Configuration.MaxWorkers; }
private BrightstarCluster() { try { CloudStorageAccount.SetConfigurationSettingPublisher((key, publisher) => publisher(RoleEnvironment.GetConfigurationSettingValue(key))); _clients = new List<Tuple<string, IStoreWorkerService>>(); UpdateClientList(); var storageAccount = CloudStorageAccount.FromConfigurationSetting(AzureConstants.BlockStoreConnectionStringName); _blobClient = storageAccount.CreateCloudBlobClient(); RoleEnvironment.Changing += HandleRoleEnvironmentChanging; _jobQueue = new SqlJobQueue( RoleEnvironment.GetConfigurationSettingValue(AzureConstants.ManagementDatabaseConnectionStringName), RoleEnvironment.CurrentRoleInstance.Id); } catch (Exception ex) { Trace.TraceError("Error initializing BrightstarCluster: {0}", ex); } }
public MaxWorkers() { var readTask = Task.FromResult((Dependable.Job)_world.NewJob); _queue = Substitute.For<IJobQueue>(); _queue.Configuration.Returns(new ActivityConfiguration().WithMaxWorkers(1)); _queue.Read().Returns(c => { if (_readInvocations.Count != 0) _readInvocations.Dequeue().SetResult(new object()); return readTask; }); _world.Dispatcher.Dispatch(null, null).ReturnsForAnyArgs(c => { _dispatchInvocations.Dequeue().SetResult(new object()); return _dispatchResults.Dequeue().Task; }); }
public JobsModule(IJobQueue jobQueue) { Get["/jobs"] = _ => Response.AsJson(jobQueue.Jobs); }
public Scheduler(IJobQueue jobQueue) { this.jobQueue = jobQueue; }
public static JobPump NewJobPump(this World world, IJobQueue queue) { if (world == null) throw new ArgumentNullException("world"); if (queue == null) throw new ArgumentNullException("queue"); return new JobPump(world.Dispatcher, world.EventStream, queue); }
public virtual void WorkWithQueue(IJobQueue jobQueue) { _queue = jobQueue; }