public PackagesController(INugetApi nugetApi, IJobQueueFactory queueFactory, IGetSetQuerier <Package> redisQuery, ILatestPackagesIndex latestPackages)
{
this.nugetApi = nugetApi;
this.packageCrawlerJobQueue = queueFactory.ForQueueName(Constants.Redis.PackageCrawlerJobQueueName);
this.redisQuerier = redisQuery;
this.latestPackages = latestPackages;
}
public override bool OnStart()
{
try
{
// Set the maximum number of concurrent connections
ServicePointManager.DefaultConnectionLimit = 12;
RoleStartupHelper.StartDiagnostics();
InitializeBlockStorage();
StartBlockUpdateService();
StartStoreWorkerService();
StoreConfiguration.DefaultStoreConfiguration.StoreManagerType = typeof(AzureStoreManager);
var jobQueueConnectionString = RoleEnvironment.GetConfigurationSettingValue("ManagementDatabaseConnectionString");
Trace.TraceInformation("Initializing SQL job queue");
_jobQueue = new SqlJobQueue(jobQueueConnectionString, RoleEnvironment.CurrentRoleInstance.Id);
Trace.TraceInformation("Job queue initialized");
EventLog.WriteEntry("AzureEventLogs",
"Worker role started",
EventLogEntryType.Information);
return(base.OnStart());
}
catch (Exception ex)
{
Trace.TraceError("WorkerRole OnStart failed: " + ex);
EventLog.WriteEntry("AzureEventLogs",
"OnStart failed for StoreWorker role: Exception:" + ex.GetType().FullName +
", Message:" + ex.Message,
EventLogEntryType.Error, 0);
Trace.TraceError("Sleeping for long enough to allow errors to flush");
Thread.Sleep(TimeSpan.FromMinutes(1.5));
Trace.TraceError("Aborting role startup");
return(false);
}
}
public const int ClientTimeOutInSeconds = 10; ////TODO: This should not be hardcoded and must be moved to congig
public async Task <bool> StartAsync(IJobQueue job)
{
var webScrapeJob = (WebScrapeJob)job;
using (HttpClient client = new HttpClient())
{
// Timeout if site doesn't respond
client.Timeout = TimeSpan.FromSeconds(ClientTimeOutInSeconds);
var html = await client.GetStringAsync(webScrapeJob.Url);
// Console.WriteLine(html);
var doc = new HtmlDocument();
doc.LoadHtml(html);
webScrapeJob.Scrapes = new List <string>();
if (!string.IsNullOrEmpty(webScrapeJob.Selector))
{
foreach (var selectedNode in doc.DocumentNode.QuerySelectorAll(webScrapeJob.Selector))
{
webScrapeJob.Scrapes.Add(selectedNode.InnerHtml);
}
}
return(true);
}
}
public static void CreateGanttChartForRessource(JobInProgress jobInProgress, IJobQueue scopeQueue, Agent agent)
{
List <GanttChartItem> ganttData = new List <GanttChartItem>();
if (jobInProgress.IsSet)
{
ganttData.AddRange(CreateGanttProcessingQueueLog(new[] { jobInProgress.GanttItem }, true, "Processing", jobInProgress, agent));
// add from scope
}
ganttData.AddRange(CreateGanttProcessingQueueLog(jobInProgress.GanttItems.ToArray(), false, "ReadyElement", jobInProgress, agent));
var jobs = scopeQueue.GetAllJobs().OrderBy(x => x.Job.Priority(agent.CurrentTime)).ToList();
ganttData.AddRange(CreateGanttProcessingQueueLog(jobs.ToArray(), false, "ScopeQueue", jobInProgress, agent));
CustomFileWriter.WriteToFile($"Logs//ResourceScheduleAt-{agent.CurrentTime}.log",
JsonConvert.SerializeObject(ganttData).Replace("[", "").Replace("]", ","));
if (jobs.Count > 0)
{
if (jobs.First().ScopeConfirmation.GetScopeStart() < jobInProgress.ResourceIsBusyUntil)
{
agent.DebugMessage("Seems wrong");
}
}
}
public IJobQueue Register()
{
IConsumerContext ctx = Turbine.Consumer.AppUtility.GetConsumerContext();
Guid consumerId = ctx.Id;
String hostname = ctx.Hostname;
Debug.WriteLine(String.Format("Register as {0}, {1}", consumerId, hostname), this.GetType().Name);
//
// NOTE: Amazon InstanceID and AMI removed
//
using (TurbineModelContainer container = new TurbineModelContainer())
{
container.JobConsumers.AddObject(new JobConsumer()
{
Id = consumerId,
hostname = hostname,
AMI = "",
instance = "",
processId = Process.GetCurrentProcess().Id.ToString(),
status = "up",
keepalive = DateTime.UtcNow
}
);
container.SaveChanges();
}
queue = new DBJobQueue();
return(queue);
}
public IJobQueue Register(IConsumerRun run)
{
this.run = run;
IConsumerContext ctx = Turbine.Consumer.AppUtility.GetConsumerContext();
//Guid consumerId = ctx.Id;
//Guid consumerId = Guid.NewGuid();
String hostname = ctx.Hostname;
String appName = run.SupportedApplications.ElementAtOrDefault(0);
Debug.WriteLine(String.Format("Register({0}) as {1}, {2}", appName, run.ConsumerId, hostname), this.GetType().Name);
using (ProducerContext db = new ProducerContext())
{
// TODO: Registering as a single application is dubious.
// IF support multiple apps in single consumer need to expose that via the database ( update SCHEMA!! )
var app = db.Applications.Single(a => a.Name == appName);
var consumer = new Turbine.Data.Entities.JobConsumer
{
Application = app,
Id = run.ConsumerId,
hostname = hostname,
processId = System.Diagnostics.Process.GetCurrentProcess().Id,
keepalive = DateTime.UtcNow,
status = "up"
};
db.Consumers.Add(consumer);
db.SaveChanges();
}
queue = AppUtility.GetJobQueue(run);
//((DBJobQueue)queue).consumerId = consumerId;
return(queue);
}
public JobRequestConsumer(IJobQueue queue, IServiceBus bus, int delaySeconds, IWindsorContainer container)
{
this.queue = queue;
this.bus = bus;
this.delaySeconds = delaySeconds;
this.container = container;
}
public static async Task <IProjectManager> Load(string projectsDir, IJobQueue jobQueue, CancellationToken cancellationToken)
{
var projects = new Dictionary <Guid, IProject>();
Directory.CreateDirectory(projectsDir);
foreach (var projectDir in Directory.EnumerateDirectories(projectsDir))
{
cancellationToken.ThrowIfCancellationRequested();
if (!Guid.TryParse(Path.GetFileName(projectDir), out var id))
{
continue;
}
var configStr = await File.ReadAllTextAsync(Path.Combine(projectDir, "project.json"), cancellationToken);
var config = JsonConvert.DeserializeObject <ProjectConfig>(configStr);
var project = new Project(Path.GetFullPath(projectDir), id, jobQueue, config);
projects[id] = project;
}
return(new ProjectManager(projectsDir, jobQueue, projects, cancellationToken));
}
public TaskScheduler(Func <IJobQueue> JobQueueFactory, ITimeProvider timeProvider = null, Thread mainThread = null)
{
MainThread = mainThread ?? Thread.CurrentThread;
TimeProvider = timeProvider ?? Time.DefaultTimeProvider;
BackgroundTaskOnComplete = _BackgroundTaskOnComplete;
OnResolvedDispatcher = _OnResolvedDispatcher;
OnResolvedDispatcher_SkipQueue = _OnResolvedDispatcher_SkipQueue;
mOnResolvedDispatcher_SkipQueue = GetType().GetMethod("_OnResolvedDispatcher_SkipQueue_Generic", BindingFlags.Instance | BindingFlags.NonPublic);
_JobQueue = JobQueueFactory();
_SleepWorker = new Internal.WorkerThread <PriorityQueue <SleepItem> >(
SleepWorkerThreadFunc, ThreadPriority.AboveNormal, "TaskScheduler Sleep Provider"
);
SynchronizationContext = new TaskSchedulerSynchronizationContext(this);
if (!_Default.IsValueCreated)
{
_Default.Value = this;
WorkItemQueueTarget.SetDefaultIfNone(this);
}
else
{
_Default.Value = null;
}
}
public Project(string dir, Guid id, IJobQueue jobQueue, ProjectConfig config)
{
ProjectDir = dir;
this.jobQueue = jobQueue;
Id = id;
Config = config;
}
public TaskScheduler(Func <IJobQueue> JobQueueFactory)
{
_JobQueue = JobQueueFactory();
_SleepWorker = new Internal.WorkerThread <PriorityQueue <SleepItem> >(
SleepWorkerThreadFunc, ThreadPriority.AboveNormal, "TaskScheduler Sleep Provider"
);
}
private ProjectManager(string projectsDir, IJobQueue jobQueue, IDictionary <Guid, IProject> projects, CancellationToken cancellationToken)
{
this.projectsDir = projectsDir;
this.jobQueue = jobQueue;
this.projects = new ConcurrentDictionary <Guid, IProject>(projects);
this.cancellationToken = cancellationToken;
}
/// <summary>
/// Initializes a new instance of the <see cref="JobScheduler"/> class.
/// </summary>
/// <param name="jobQueue">The job queue.</param>
/// <param name="getTimeFactory">The get time factory.</param>
/// <param name="log">The log factory.</param>
public JobScheduler(IJobQueue jobQueue,
IGetTimeFactory getTimeFactory,
ILogger log)
{
_jobQueue = jobQueue;
_getTime = getTimeFactory;
_log = log;
}
/// <summary>
/// Initializes a new instance of the <see cref="JobScheduler"/> class.
/// </summary>
/// <param name="jobQueue">The job queue.</param>
/// <param name="getTimeFactory">The get time factory.</param>
/// <param name="logFactory">The log factory.</param>
public JobScheduler(IJobQueue jobQueue,
IGetTimeFactory getTimeFactory,
ILogFactory logFactory)
{
_jobQueue = jobQueue;
_getTime = getTimeFactory;
_logFactory = logFactory;
}
public QueueConfiguration(IJobQueue @default, IDictionary<Type, IJobQueue> activitySpecificQueues)
{
if (@default == null) throw new ArgumentNullException("default");
if (activitySpecificQueues == null) throw new ArgumentNullException("activitySpecificQueues");
Default = @default;
ActivitySpecificQueues = activitySpecificQueues;
}
public FileSystemWatcherService(ILogger <FileSystemWatcherService> logger,
IOptionsMonitor <FolderSettings> folderSettings, IJobQueue jobQueue, IJobBuilder job)
{
_logger = logger;
_folderSettings = folderSettings;
_jobQueue = jobQueue;
_job = job;
}
public void SetUp()
{
_tokenSource = new CancellationTokenSource();
_testJobQueueMonitor = new TestMonitor();
_jobQueueConfig = new JobQueueConfig(_tokenSource.Token, MaxConsurrentJobs);
_jobQueue = new JobQueue(_testJobQueueMonitor, _jobQueueConfig);
Task.Factory.StartNew(_jobQueue.TryRun);
}
public SessionDebugManager(JobExecutor jobExecutor, IJobQueue jobQueue,
LaunchAndAttachFlow launchAndAttachFlow,
IDebugSession debugSession)
{
_jobExecutor = jobExecutor;
_jobQueue = jobQueue;
_launchAndAttachFlow = launchAndAttachFlow;
Session = debugSession;
}
public JobsHostedService(
ILogger <JobsHostedService> logger,
IOptions <JobsContext> options,
IJobQueue jobQueue)
{
Logger = logger;
Options = options;
JobQueue = jobQueue;
}
public TelegramResponseHandler(TelegramBotClient client, IJenkinsService jenkinsService, IJobQueue jobQueue, IList <MenuItem> menu, IList <long> allowedChatIds, string jenkinsProtocol, string jenkinsFqdn)
{
_client = client ?? throw new ArgumentNullException(nameof(client));
_jenkinsService = jenkinsService ?? throw new ArgumentNullException(nameof(jenkinsService));
_menu = menu ?? throw new ArgumentNullException(nameof(menu));
_jenkinsProtocol = jenkinsProtocol ?? throw new ArgumentNullException(nameof(jenkinsProtocol));
_jenkinsFqdn = jenkinsFqdn ?? throw new ArgumentNullException(nameof(jenkinsFqdn));
_jobQueue = jobQueue ?? throw new ArgumentNullException(nameof(jobQueue));
_allowedChatIds = allowedChatIds;
}
public Factory(IJobQueue jobQueue,
RefreshVariableJob.Factory refreshVariableJobFactory,
IDebugSessionContext debugSessionContext,
VariableExpander.Factory variableExpanderFactory)
{
_jobQueue = jobQueue;
_refreshVariableJobFactory = refreshVariableJobFactory;
_debugSessionContext = debugSessionContext;
_variableExpanderFactory = variableExpanderFactory;
}
public BreakpointView(BindBreakpointJob.Factory bindBreakpointJobFactory,
BreakpointDeleteJob.Factory breakpointDeleteJobFactory,
IDebugSessionContext debugSessionContext, IJobQueue jobQueue,
JoinableTaskContext taskContext)
{
_bindBreakpointJobFactory = bindBreakpointJobFactory;
_breakpointDeleteJobFactory = breakpointDeleteJobFactory;
_debugSessionContext = debugSessionContext;
_jobQueue = jobQueue;
_taskContext = taskContext;
}
public WriteOnlyTransaction(
IConnectionProvider connectionProvider,
IJobQueue queue)
{
Guard.ThrowIfNull(connectionProvider, nameof(connectionProvider));
Guard.ThrowIfNull(queue, nameof(queue));
_connectionProvider = connectionProvider;
_queue = queue;
_commandQueue = new Queue <Action <NpgsqlConnection, NpgsqlTransaction> >();
}
VariableEntry(IVariableDataSource dataSource, IJobQueue jobQueue,
RefreshVariableJob.Factory refreshVariableJobFactory,
IDebugSessionContext debugSessionContext,
IVariableExpander variableExpander)
{
_dataSource = dataSource;
_jobQueue = jobQueue;
_refreshVariableJobFactory = refreshVariableJobFactory;
_debugSessionContext = debugSessionContext;
_variableExpander = variableExpander;
}
public JobOrchestrator(IDebugSessionContext debugSessionContext, IJobQueue queue,
ProgramStoppedJob.Factory programStoppedJobFactory,
BroadcastDebugEventJob.Factory broadcastDebugEventJobFactory,
ProgramTerminatedJob.Factory programTerminatedJobFactory)
{
_debugSessionContext = debugSessionContext;
_queue = queue;
_programStoppedJobFactory = programStoppedJobFactory;
_broadcastDebugEventJobFactory = broadcastDebugEventJobFactory;
_programTerminatedJobFactory = programTerminatedJobFactory;
}
private UdpClient(UdpTransportConfiguration configuration, List <Socket> multicastSockets, List <Socket> unicastSockets, IJobQueue <UdpUnicastJob> unicastJobQueue, IObjectPool <byte[]> sendReceiveBufferPool, IAuditAggregator <double> inboundBytesAggregator, IAuditAggregator <double> outboundBytesAggregator, IAuditAggregator <double> inboundReceiveProcessDispatchLatencyAggregator)
{
this.configuration = configuration;
this.multicastSockets = multicastSockets;
this.unicastSockets = unicastSockets;
this.unicastJobQueue = unicastJobQueue;
this.sendReceiveBufferPool = sendReceiveBufferPool;
this.inboundBytesAggregator = inboundBytesAggregator;
this.outboundBytesAggregator = outboundBytesAggregator;
this.inboundReceiveProcessDispatchLatencyAggregator = inboundReceiveProcessDispatchLatencyAggregator;
this.sendArgsPool = ObjectPool.CreateStackBacked(() => new SocketAsyncEventArgs());
}
/// <summary>
/// Initializes a new instance of the <see cref="QueueExecutor"/>
/// class.
/// </summary>
/// <param name="queue">The <see cref="IJobQueue"/> this
/// <see cref="QueueExecutor"/> will execute the elements
/// within.</param>
/// <exception cref="ArgumentNullException">queue is null.</exception>
public QueueExecutor(IJobQueue queue)
{
if (queue == null)
{
throw new ArgumentNullException("queue");
}
AutoStart = false;
_stop = false;
_queue = queue;
_queue.JobAdded += job_added;
}
public StorageConnection(
IConnectionProvider connectionProvider,
IJobQueue queue,
PostgreSqlStorageOptions options)
{
Guard.ThrowIfNull(connectionProvider, nameof(connectionProvider));
Guard.ThrowIfNull(queue, nameof(queue));
Guard.ThrowIfNull(options, nameof(options));
_connectionProvider = connectionProvider;
_queue = queue;
}
public void Initialize()
{
_cancellationTokenSource = new CancellationTokenSource();
_cancellationToken = _cancellationTokenSource.Token;
_jobQueue = new JobQueue();
_jobId = Guid.NewGuid();
_jobManager = new JobManager();
_consumer = new Consumer(_jobQueue, _cancellationToken);
Task.Run(() => {
_consumer.ConsumeJobs();
});
}
public ObserveAndNotifyJob(IJobQueue jobQueue, Func <bool> predicate, Action callback,
string logDetails)
{
_jobQueue = jobQueue;
_predicate = predicate;
_callback = callback;
_logDetails = logDetails ?? "";
_timer = new Timer(POLL_INTERVAL_MS);
_timer.AutoReset = false;
_timer.Elapsed += TimerExpired;
}
public void FillQueue(IJobQueue queue)
{
var submissionInfos = new WaitingSubmissionsSortedByTimeAsc()
.Load(session)
.List <Submission>()
.Select(x => new SubmissionInfo(x));
foreach (var submissionInfo in submissionInfos)
{
queue.Enqueue(submissionInfo, Priority.Normal);
}
}
public Endpoint(
IOptions <EndpointConfiguration <TEndpoint> > configuration,
ISerializer <TInput> inputSerializer,
ISerializer <TOutput> outputSerializer,
IJobRepository repository,
IJobQueue queue)
{
_configuration = configuration.Value;
_inputSerializer = inputSerializer;
_outputSerializer = outputSerializer;
_repository = repository;
_queue = queue;
}
public PackageManagerProxy(IPackageManagerProvider packageManagerProvider,
IPackageManagerSettings packageManagerSettings,
IClientMessenger clientMessenger,
IJobQueue jobQueue,
IJobFactory jobFactory,
IPackageList packageList)
{
_packageManagerProvider = packageManagerProvider;
_packageManagerSettings = packageManagerSettings;
_clientMessenger = clientMessenger;
_jobQueue = jobQueue;
_jobFactory = jobFactory;
_packageList = packageList;
}
public JobPump(
IDispatcher dispatcher,
IEventStream eventStream,
IJobQueue queue)
{
if(dispatcher == null) throw new ArgumentNullException("dispatcher");
if(eventStream == null) throw new ArgumentNullException("eventStream");
if (queue == null) throw new ArgumentNullException("queue");
_dispatcher = dispatcher;
_eventStream = eventStream;
_queue = queue;
_throttle = queue.Configuration.MaxWorkers == 0 ? int.MaxValue : queue.Configuration.MaxWorkers;
}
private BrightstarCluster()
{
try
{
CloudStorageAccount.SetConfigurationSettingPublisher((key, publisher) => publisher(RoleEnvironment.GetConfigurationSettingValue(key)));
_clients = new List<Tuple<string, IStoreWorkerService>>();
UpdateClientList();
var storageAccount = CloudStorageAccount.FromConfigurationSetting(AzureConstants.BlockStoreConnectionStringName);
_blobClient = storageAccount.CreateCloudBlobClient();
RoleEnvironment.Changing += HandleRoleEnvironmentChanging;
_jobQueue =
new SqlJobQueue(
RoleEnvironment.GetConfigurationSettingValue(AzureConstants.ManagementDatabaseConnectionStringName),
RoleEnvironment.CurrentRoleInstance.Id);
}
catch (Exception ex)
{
Trace.TraceError("Error initializing BrightstarCluster: {0}", ex);
}
}
public MaxWorkers()
{
var readTask = Task.FromResult((Dependable.Job)_world.NewJob);
_queue = Substitute.For<IJobQueue>();
_queue.Configuration.Returns(new ActivityConfiguration().WithMaxWorkers(1));
_queue.Read().Returns(c =>
{
if (_readInvocations.Count != 0)
_readInvocations.Dequeue().SetResult(new object());
return readTask;
});
_world.Dispatcher.Dispatch(null, null).ReturnsForAnyArgs(c =>
{
_dispatchInvocations.Dequeue().SetResult(new object());
return _dispatchResults.Dequeue().Task;
});
}
public JobsModule(IJobQueue jobQueue)
{
Get["/jobs"] = _ => Response.AsJson(jobQueue.Jobs);
}
public Scheduler(IJobQueue jobQueue)
{
this.jobQueue = jobQueue;
}
public static JobPump NewJobPump(this World world, IJobQueue queue)
{
if (world == null) throw new ArgumentNullException("world");
if (queue == null) throw new ArgumentNullException("queue");
return new JobPump(world.Dispatcher, world.EventStream, queue);
}
public virtual void WorkWithQueue(IJobQueue jobQueue)
{
_queue = jobQueue;
}
