public Worker(WorkerConfiguration configuration) { _configuration = configuration; _isConnected = new BehaviorSubject <bool>(false); _cancel = new CancellationTokenSource(); }
public WorkerRegistrationFinalization FinalizeRegistration([FromBody] GatewayRegistrationCallback registrationCallback) { if (registrationCallback == null) { throw new ArgumentNullException(nameof(registrationCallback)); } bool success = false; Exception exception = null; try { success = WorkerConfiguration.AcceptNewDatabaseConnection(registrationCallback); } catch (Exception e) { exception = e; } return(new WorkerRegistrationFinalization { HasSuccessfullyConnectedToDatabase = success, ConnectionException = exception }); }
public GitCommandProcessor(WorkerConfiguration configuration, IProgressLog progressLog, IGitHubClientFactory gitHubClientFactory, IGitWrapperFactory gitWrapperFactory) { Configuration = configuration; ProgressLog = progressLog; _gitHubClientFactory = gitHubClientFactory; _gitWrapperFactory = gitWrapperFactory; }
private static LocatorParameters CreateLocatorParameters(WorkerConfiguration instance) { var credentials = string.IsNullOrEmpty(instance.SteamToken) ? LocatorCredentialsType.LoginToken : LocatorCredentialsType.Steam; var locatorParameters = new LocatorParameters { ProjectName = instance.ProjectName, CredentialsType = credentials }; switch (locatorParameters.CredentialsType) { case LocatorCredentialsType.LoginToken: locatorParameters.LoginToken.Token = instance.LoginToken; break; case LocatorCredentialsType.Steam: locatorParameters.Steam.DeploymentTag = instance.DeploymentTag; locatorParameters.Steam.Ticket = instance.SteamToken; break; default: throw new ArgumentOutOfRangeException(); } return(locatorParameters); }
private static ConnectionParameters CreateConnectionParameters(WorkerConfiguration instance) { var tcpParameters = new TcpNetworkParameters { MultiplexLevel = instance.TcpMultiplexLevel }; var raknetParameters = new RakNetNetworkParameters { HeartbeatTimeoutMillis = instance.RaknetHeartbeatTimeoutMillis }; var parameters = new ConnectionParameters { Network = { ConnectionType = instance.LinkProtocol, Tcp = tcpParameters, RakNet = raknetParameters, UseExternalIp = instance.UseExternalIp, }, WorkerType = WorkerTypeUtils.ToWorkerName(instance.WorkerPlatform), BuiltInMetricsReportPeriodMillis = 2000, EnableProtocolLoggingAtStartup = SpatialOS.Configuration.ProtocolLoggingOnStartup, ProtocolLogging = { LogPrefix = SpatialOS.Configuration.ProtocolLogPrefix, MaxLogFiles = 10, MaxLogFileSizeBytes = SpatialOS.Configuration.ProtocolLogMaxFileBytes }, ReceiveQueueCapacity = instance.ReceiveQueueCapacity, SendQueueCapacity = instance.SendQueueCapacity, }; return(parameters); }
/// <summary> /// Crawler uses WorkManager to get URLs to crawl through. /// </summary> public Crawler(WorkManager manager, WorkerConfiguration config, PluginManager plugins = null) { Config = config; Manager = manager; this.plugins = plugins; RecentDownloads = new ConcurrentSlidingBuffer <DownloadedWork>(config.MaxLoggedDownloads); }
public void TimerRemovalTest() { // test if timer correctly removes old entries // general test string robotsTxt = @" User-agent: * Disallow: /"; var timerInterval = TimeSpan.FromSeconds(2); var config = new WorkerConfiguration() { UserAgent = "CryCrawler", RespectRobotsExclusionStandard = true }; var robots = new RobotsHandler(config, new System.Net.Http.HttpClient(), timerInterval); var data = robots.RegisterRobotsTxt("test.com", robotsTxt).Result; var e = robots.IsUrlExcluded("http://test.com/something").Result; Assert.True(e); e = robots.IsUrlExcluded("http://test.com").Result; Assert.True(e); e = robots.IsUrlExcluded("http://test.com/lol/test").Result; Assert.True(e); Task.Delay((int)timerInterval.TotalMilliseconds + 100).Wait(); e = robots.IsUrlExcluded("http://test.com/something").Result; Assert.False(e); e = robots.IsUrlExcluded("http://test.com").Result; Assert.False(e); e = robots.IsUrlExcluded("http://test.com/lol/test").Result; Assert.False(e); }
void MakeEnvironment(string dbName, Action <CacheDatabase, WorkerConfiguration, WorkManager, int> action, bool depthSearch = false, int memLimit = 100000) { var database = new CacheDatabase(dbName); database.EnsureNew(); var config = new WorkerConfiguration(); config.DepthSearch = depthSearch; var wm = new WorkManager(config, database, memLimit); try { action(database, config, wm, memLimit); } finally { database.Dispose(); database.Delete(); } }
public ElasticsearchFixture() { var esUrl = Environment.GetEnvironmentVariable("ELASTICSEARCH_URL") ?? "http://localhost:9200"; var pool = new SingleNodeConnectionPool(new Uri(esUrl)); var connectionSettings = new ConnectionSettings(pool, sourceSerializer: JsonNetSerializer.Default); Client = new ElasticClient(connectionSettings); var indexSuffix = "_" + DateTime.UtcNow.Ticks; Configuration = new ApplicationConfiguration() { ElasticsearchUrl = esUrl, JobsIndexName = "test_jobs" + indexSuffix, UserIndexName = "test_usersettings" + indexSuffix, OwnerSettingsIndexName = "test_ownersettings" + indexSuffix, RepoSettingsIndexName = "test_reposettings" + indexSuffix, DatasetIndexName = "test_datasets" + indexSuffix, SchemaIndexName = "test_schemas" + indexSuffix, FileStorePath = "test_files" + indexSuffix }; WorkerConfiguration = new WorkerConfiguration { ElasticsearchUrl = esUrl, JobsIndexName = "test_jobs" + indexSuffix, UserIndexName = "test_usersettings" + indexSuffix, OwnerSettingsIndexName = "test_ownersettings" + indexSuffix, RepoSettingsIndexName = "test_reposettings" + indexSuffix, DatasetIndexName = "test_datasets" + indexSuffix, SchemaIndexName = "test_schemas" + indexSuffix, FileStorePath = "test_files" + indexSuffix, GitPath = "", RepoBaseDir = "test_repos" + indexSuffix, GitHubClientHeader = "datadock_test" }; }
/// <summary> /// Create a worker that will execute `model` using the best backend that is available for a given `device` type. /// `model` is the associated model. See ModelLoader.cs. /// `additionalOutputs` are the additional outputs to track but not directly specified by the model. /// `trimOutputs` are the outputs not discard even if they are specified by the model. /// `device` is the device type to run worker on. For example `WorkerFactory.Device.GPU` specifies the fast GPU path. /// `verbose` will log scheduling of layers execution to the console (default == false). /// </summary> public static IWorker CreateWorker(Model model, string[] additionalOutputs, string[] trimOutputs, Device device = Device.Auto, bool verbose = false) { var type = GetBestTypeForDevice(device); var workerConfiguration = new WorkerConfiguration(type, verbose); return(CreateWorker(type, model, additionalOutputs, trimOutputs, workerConfiguration)); }
public void SetupContainer() { var currentPath = Path.GetDirectoryName(SandboxHelper.GetAssemblyLocation(typeof(StartupTest).Assembly)); currentPath = Path.Combine(currentPath, "Sandbox"); WorkerConfiguration.SetConfiguration(currentPath, Guid.NewGuid()); }
public void ItCanReadFromJson() { var builder = new ConfigurationBuilder() .SetBasePath(Path.GetFullPath("data/config")) .AddJsonFile("testSettings.json") .AddEnvironmentVariables("DDX_"); var config = builder.Build(); var appConfig = new WorkerConfiguration(); config.Bind(appConfig); appConfig.ElasticsearchUrl.Should().Be("http://some.elasticsearch:9200/"); appConfig.DatasetIndexName.Should().Be("TestDatasets"); appConfig.JobsIndexName.Should().Be("TestJobs"); appConfig.OwnerSettingsIndexName.Should().Be("TestOwners"); appConfig.SchemaIndexName.Should().Be("TestSchemas"); appConfig.RepoSettingsIndexName.Should().Be("TestRepoSettings"); appConfig.UserIndexName.Should().Be("TestUsers"); appConfig.FileStorePath.Should().Be("/path/to/file/store"); appConfig.GitPath.Should().Be("/usr/bin/git"); appConfig.GitHubClientHeader.Should().Be("MyDataDock"); appConfig.PublishUrl.Should().Be("http://mydatadock.com/"); appConfig.RepoBaseDir.Should().Be("/data/repos"); appConfig.SignalRHubUrl.Should().Be("http://datadock.web/progress"); }
public void ItCanUseDefaultValues() { var builder = new ConfigurationBuilder() .SetBasePath(Path.GetFullPath("data/config")) .AddJsonFile("partialSettings.json") .AddEnvironmentVariables("DDX_"); // different prefix to avoid any test race conditions with the test that uses DD_ env vars var config = builder.Build(); var appConfig = new WorkerConfiguration(); config.Bind(appConfig); // Expected overrides appConfig.ElasticsearchUrl.Should().Be("http://some.elasticsearch:9200/"); appConfig.FileStorePath.Should().Be("/path/to/file/store"); // Expected defaults appConfig.DatasetIndexName.Should().Be("datasets"); appConfig.JobsIndexName.Should().Be("jobs"); appConfig.OwnerSettingsIndexName.Should().Be("ownersettings"); appConfig.SchemaIndexName.Should().Be("schemas"); appConfig.RepoSettingsIndexName.Should().Be("reposettings"); appConfig.UserIndexName.Should().Be("users"); appConfig.GitPath.Should().Be("git"); appConfig.GitHubClientHeader.Should().Be(""); appConfig.PublishUrl.Should().Be("http://datadock.io/"); appConfig.RepoBaseDir.Should().Be("/datadock/repositories"); appConfig.SignalRHubUrl.Should().Be("http://web/progress"); }
public void UserAgentMatchingTest() { // test if user agents get matched correctly string robotsTxt = @" User-agent: Googlebot Allow: /testdomain2 Allow: /advertisements Disallow: /artists User-agent: CryCrawler Disallow: /admin Allow: /*?lang= Disallow: /search/realtime User-agent: * Disallow: /testdomain "; var config = new WorkerConfiguration() { UserAgent = "CryCrawler", RespectRobotsExclusionStandard = true }; var robots = new RobotsHandler(config, new System.Net.Http.HttpClient()); var data = robots.RegisterRobotsTxt("test.com", robotsTxt).Result; Assert.Single(data.AllowedList); Assert.Equal(2, data.DisallowedList.Count); }
public async void ItWrapsTheSchemaInSourceMetadata() { var config = new WorkerConfiguration { PublishUrl = "http://datadock.io/" }; var proc = new ImportSchemaProcessor(config, _mockSchemeStore.Object, _mockFileStore.Object); WithJsonSchema(@"{ '@context': 'http://www.w3.org/ns/csvw' }"); #pragma warning disable 618 var job = new JobInfo #pragma warning restore 618 { JobType = JobType.SchemaCreate, UserId = "kal", OwnerId = "datadock", RepositoryId = "test", SchemaFileId = "schemaFileId" }; await proc.ProcessJob(job, new UserAccount(), _mockProgressLog.Object); _mockSchemeStore.Verify(s => s.CreateOrUpdateSchemaRecordAsync(It.Is <SchemaInfo>(p => p.OwnerId.Equals("datadock") && p.RepositoryId.Equals("test") && p.Schema.ContainsKey("@context") && (p.Schema["@context"] as JValue).Value <string>().Equals("http://www.w3.org/ns/csvw")))); }
public void GeneralTest2() { // general test string robotsTxt = @" User-agent: CryCrawler Disallow: / User-agent: * Disallow: /admin Disallow: /advertisements Disallow: /artists Disallow: /artist_commentaries Disallow: /artist_commentary_versions Disallow: /artist_versions Disallow: /bans Disallow: /comment_votes Disallow: /comments Disallow: /counts Disallow: /delayed_jobs Disallow: /dmails Disallow: /favorite Disallow: /iqdb_queries Disallow: /ip_bans Disallow: /janitor_trials"; var config = new WorkerConfiguration() { UserAgent = "CryCrawler", RespectRobotsExclusionStandard = true }; var robots = new RobotsHandler(config, new System.Net.Http.HttpClient()); var data = robots.RegisterRobotsTxt("test.com", robotsTxt).Result; Assert.Empty(data.AllowedList); Assert.Single(data.DisallowedList); Assert.Equal(0, data.WaitTime); Assert.DoesNotContain(RobotsHandler.GetRegexPattern("/admin"), data.DisallowedList); Assert.DoesNotContain(RobotsHandler.GetRegexPattern("/bans"), data.DisallowedList); Assert.DoesNotContain(RobotsHandler.GetRegexPattern("/artist_commentary_versions"), data.DisallowedList); Assert.DoesNotContain(RobotsHandler.GetRegexPattern("/favorite"), data.DisallowedList); Assert.DoesNotContain(RobotsHandler.GetRegexPattern("/janitor_trials"), data.DisallowedList); var e = robots.IsUrlExcluded("http://test2.com/admin").Result; Assert.False(e); e = robots.IsUrlExcluded("http://test.com/admin").Result; Assert.True(e); e = robots.IsUrlExcluded("http://test.com/admin?url=test").Result; Assert.True(e); e = robots.IsUrlExcluded("http://test.com/admin/test/admin?url=test").Result; Assert.True(e); e = robots.IsUrlExcluded("http://test.com/test/admin").Result; Assert.True(e); e = robots.IsUrlExcluded("http://test.com/admin2").Result; Assert.True(e); e = robots.IsUrlExcluded("http://test.com/").Result; Assert.True(e); e = robots.IsUrlExcluded("http://test.com").Result; Assert.True(e); }
/// <summary> /// Create a worker with explicitly specified backend `type` to execute the given `model`. /// `type` is backend type to use. For example `WorkerFactory.Type.Compute` specifies the fast GPU path. /// `model` is the associated model. See ModelLoader.cs. /// `additionalOutputs` are the additional outputs to track but not directly specified by the model. /// `trimOutputs` are the outputs not discard even if they are specified by the model. /// `verbose` will log scheduling of layers execution to the console. /// `compareAgainstType` if different than `type` model will be run on those two backend and result of every layer will be compared, checking for divergence. Great for debugging, but very slow because of the sync needed. /// `differenceAsError` if `compareAgainstType` is used difference will be reported as error is this is true or warning otherwise. /// </summary> public static IWorker CreateWorker(Type type, Model model, string[] additionalOutputs, string[] trimOutputs, bool verbose, Type compareAgainstType, CompareOpsUtils.LogLevel differenceLogLevel = CompareOpsUtils.LogLevel.Warning) { var workerConfiguration = new WorkerConfiguration(type, verbose); workerConfiguration.compareAgainstType = compareAgainstType; workerConfiguration.compareLogLevel = differenceLogLevel; return(BarracudaBackendsFactory.CreateWorker(type, model, additionalOutputs, trimOutputs, workerConfiguration)); }
/// <summary> /// Create a worker with explicitly specified backend `type` to execute the given `model`. /// `type` is backend type to use. For example `WorkerFactory.Type.Compute` specifies the fast GPU path. /// `model` is the associated model. See ModelLoader.cs. /// `verbose` will log scheduling of layers execution to the console. /// `compareAgainstType` if different than `type` model will be run on those two backend and result of every layer will be compared, checking for divergence. Great for debugging, but very slow because of the sync needed. /// `differenceAsError` if `compareAgainstType` is used difference will be reported as error is this is true or warning otherwise. /// </summary> public static IWorker CreateWorker(Type type, Model model, bool verbose, Type compareAgainstType, CompareOpsUtils.LogLevel differenceLogLevel = CompareOpsUtils.LogLevel.Warning) { var workerConfiguration = new WorkerConfiguration(type, verbose); workerConfiguration.compareAgainstType = compareAgainstType; workerConfiguration.compareLogLevel = differenceLogLevel; return(CreateWorker(type, model, additionalOutputs: null, trimOutputs: null, workerConfiguration)); }
/// <summary> /// Call this to apply any customisations to the Unity Worker. /// </summary> /// <remarks> /// It is invalid to call this after this.Connect is called. /// </remarks> public static void ApplyConfiguration(WorkerConfigurationData configData, IList <string> commandLineArguments = null) { if (connectionLifecycle != null) { throw new InvalidOperationException("ApplyConfiguration was called after Connect was called."); } configuration = new WorkerConfiguration(configData, commandLineArguments); }
public FileWatcherService(WorkerConfiguration workerConfiguration) { if (workerConfiguration == null) { throw new ArgumentNullException(nameof(workerConfiguration)); } _workerConfiguration = workerConfiguration; }
public ETLService( IPUBGApiCaller pubgApiCaller, IWriteOnlyMongoDbService writeOnlyMongoDbService, IOptions <WorkerConfiguration> options) { _pubgApiCaller = pubgApiCaller; _writeOnlyMongoDbService = writeOnlyMongoDbService; _workerConfiguration = options.Value; }
protected PublishingJobProcessor(WorkerConfiguration configuration, IOwnerSettingsStore ownerSettingsStore, IRepoSettingsStore repoSettingsStore, IGitHubClientFactory gitHubClientFactory) { Configuration = configuration; OwnerSettingsStore = ownerSettingsStore; RepoSettingsStore = repoSettingsStore; GitHubClientFactory = gitHubClientFactory; }
public RobotsHandler(WorkerConfiguration config, HttpClient client, TimeSpan overrideMaxEntryAge = default) { this.config = config; this.maxAge = overrideMaxEntryAge == default ? maxAgeDefault : overrideMaxEntryAge; http = client; timer = new Timer(TimeSpan.FromMinutes(5).TotalMinutes); timer.Elapsed += Timer_Elapsed; timer.Start(); }
private static WorkerConnectionParameters CreateWorkerConnectionParameters(WorkerConfiguration instance) { return(new WorkerConnectionParameters { ConnectionParameters = CreateConnectionParameters(instance), LocatorParameters = CreateLocatorParameters(instance), LocatorHost = instance.LocatorHost, ReceptionistHost = instance.ReceptionistHost, ReceptionistPort = instance.ReceptionistPort, WorkerId = instance.WorkerId, }); }
static EndpointConfiguration BuildWorker(Type worker) { var config = new WorkerConfiguration(ActorTypeName.Register(worker)); SetReentrancy(worker, config); SetKeepAliveTimeout(worker, config); SetReceiver(worker, config); SetAutorun(worker, config); SetStickiness(worker, config); SetInvoker(worker, config); return(config); }
public Worker( IOptions <WorkerConfiguration> workerConfigurationAccessor, ILogger <Worker> logger) { _workerConfiguration = workerConfigurationAccessor.Value; _logger = logger; _crontabSchedule = CrontabSchedule.Parse(_workerConfiguration.WorkerCronSchedule, new CrontabSchedule.ParseOptions { IncludingSeconds = true }); _nextRun = _crontabSchedule.GetNextOccurrence(DateTime.Now); }
public DeleteDatasetProcessor( WorkerConfiguration configuration, GitCommandProcessor gitProcessor, IGitHubClientFactory gitHubClientFactory, IOwnerSettingsStore ownerSettingsStore, IRepoSettingsStore repoSettingsStore, IDatasetStore datasetStore, IDataDockRepositoryFactory repositoryFactory) : base(configuration, ownerSettingsStore, repoSettingsStore, gitHubClientFactory) { _git = gitProcessor; _datasetStore = datasetStore; _repositoryFactory = repositoryFactory; }
public static IHostBuilder CreateHostBuilder(string[] args) => Host.CreateDefaultBuilder(args) .ConfigureServices((hostContext, services) => { IConfiguration configuration; var serviceProvider = services.BuildServiceProvider(); configuration = serviceProvider.GetRequiredService <IConfiguration>(); WorkerConfiguration workerConfiguration = new WorkerConfiguration(services, configuration); services.AddSingleton(workerConfiguration); workerConfiguration.Configure(); });
/// <inheritdoc /> // ReSharper disable once TooManyDependencies protected SteamBotWorker( SteamBotOptions options, DataStoreBase dataStore, Coordinator coordinator, WorkerConfiguration configuration, Logger logger) : base( options, new LoggerWrapper(configuration.Alias ?? configuration.WorkerType + " [" + options.Username + "]", logger) ) { DataStore = dataStore; Coordinator = coordinator; Configuration = configuration; options.Coordinator = coordinator; Options = options; }
public void PriorityMatchingTest() { // test if more detailed rules have priority when matching patterns string robotsTxt = @" User-agent: Googlebot Allow: /testdomain2 Allow: /advertisements Disallow: /artists User-agent: CryCrawler Disallow: /admin Disallow: /search Allow: /search/test User-agent: * Disallow: /testdomain "; var config = new WorkerConfiguration() { UserAgent = "CryCrawler", RespectRobotsExclusionStandard = true }; var robots = new RobotsHandler(config, new System.Net.Http.HttpClient()); var data = robots.RegisterRobotsTxt("test.com", robotsTxt).Result; Assert.Single(data.AllowedList); Assert.Equal(2, data.DisallowedList.Count); var e = robots.IsUrlExcluded("http://test.com/admin").Result; Assert.True(e); e = robots.IsUrlExcluded("http://test.com/search").Result; Assert.True(e); e = robots.IsUrlExcluded("http://test.com/search/test").Result; Assert.False(e); e = robots.IsUrlExcluded("http://test.com/search/test/anothertest").Result; Assert.False(e); e = robots.IsUrlExcluded("http://test.com/search/test?url=23").Result; Assert.False(e); e = robots.IsUrlExcluded("http://test.com/search/test2").Result; Assert.True(e); e = robots.IsUrlExcluded("http://test.com/search/nottest").Result; Assert.True(e); }