public Worker(WorkerConfiguration configuration)
{
_configuration = configuration;
_isConnected = new BehaviorSubject <bool>(false);
_cancel = new CancellationTokenSource();
}
public WorkerRegistrationFinalization FinalizeRegistration([FromBody] GatewayRegistrationCallback registrationCallback)
{
if (registrationCallback == null)
{
throw new ArgumentNullException(nameof(registrationCallback));
}
bool success = false;
Exception exception = null;
try
{
success = WorkerConfiguration.AcceptNewDatabaseConnection(registrationCallback);
}
catch (Exception e)
{
exception = e;
}
return(new WorkerRegistrationFinalization
{
HasSuccessfullyConnectedToDatabase = success,
ConnectionException = exception
});
}
public GitCommandProcessor(WorkerConfiguration configuration, IProgressLog progressLog, IGitHubClientFactory gitHubClientFactory, IGitWrapperFactory gitWrapperFactory)
{
Configuration = configuration;
ProgressLog = progressLog;
_gitHubClientFactory = gitHubClientFactory;
_gitWrapperFactory = gitWrapperFactory;
}
private static LocatorParameters CreateLocatorParameters(WorkerConfiguration instance)
{
var credentials = string.IsNullOrEmpty(instance.SteamToken) ? LocatorCredentialsType.LoginToken : LocatorCredentialsType.Steam;
var locatorParameters = new LocatorParameters
{
ProjectName = instance.ProjectName,
CredentialsType = credentials
};
switch (locatorParameters.CredentialsType)
{
case LocatorCredentialsType.LoginToken:
locatorParameters.LoginToken.Token = instance.LoginToken;
break;
case LocatorCredentialsType.Steam:
locatorParameters.Steam.DeploymentTag = instance.DeploymentTag;
locatorParameters.Steam.Ticket = instance.SteamToken;
break;
default:
throw new ArgumentOutOfRangeException();
}
return(locatorParameters);
}
private static ConnectionParameters CreateConnectionParameters(WorkerConfiguration instance)
{
var tcpParameters = new TcpNetworkParameters {
MultiplexLevel = instance.TcpMultiplexLevel
};
var raknetParameters = new RakNetNetworkParameters {
HeartbeatTimeoutMillis = instance.RaknetHeartbeatTimeoutMillis
};
var parameters = new ConnectionParameters
{
Network =
{
ConnectionType = instance.LinkProtocol,
Tcp = tcpParameters,
RakNet = raknetParameters,
UseExternalIp = instance.UseExternalIp,
},
WorkerType = WorkerTypeUtils.ToWorkerName(instance.WorkerPlatform),
BuiltInMetricsReportPeriodMillis = 2000,
EnableProtocolLoggingAtStartup = SpatialOS.Configuration.ProtocolLoggingOnStartup,
ProtocolLogging =
{
LogPrefix = SpatialOS.Configuration.ProtocolLogPrefix,
MaxLogFiles = 10,
MaxLogFileSizeBytes = SpatialOS.Configuration.ProtocolLogMaxFileBytes
},
ReceiveQueueCapacity = instance.ReceiveQueueCapacity,
SendQueueCapacity = instance.SendQueueCapacity,
};
return(parameters);
}
/// <summary>
/// Crawler uses WorkManager to get URLs to crawl through.
/// </summary>
public Crawler(WorkManager manager, WorkerConfiguration config, PluginManager plugins = null)
{
Config = config;
Manager = manager;
this.plugins = plugins;
RecentDownloads = new ConcurrentSlidingBuffer <DownloadedWork>(config.MaxLoggedDownloads);
}
public void TimerRemovalTest()
{
// test if timer correctly removes old entries
// general test
string robotsTxt = @"
User-agent: *
Disallow: /";
var timerInterval = TimeSpan.FromSeconds(2);
var config = new WorkerConfiguration()
{
UserAgent = "CryCrawler", RespectRobotsExclusionStandard = true
};
var robots = new RobotsHandler(config, new System.Net.Http.HttpClient(), timerInterval);
var data = robots.RegisterRobotsTxt("test.com", robotsTxt).Result;
var e = robots.IsUrlExcluded("http://test.com/something").Result;
Assert.True(e);
e = robots.IsUrlExcluded("http://test.com").Result;
Assert.True(e);
e = robots.IsUrlExcluded("http://test.com/lol/test").Result;
Assert.True(e);
Task.Delay((int)timerInterval.TotalMilliseconds + 100).Wait();
e = robots.IsUrlExcluded("http://test.com/something").Result;
Assert.False(e);
e = robots.IsUrlExcluded("http://test.com").Result;
Assert.False(e);
e = robots.IsUrlExcluded("http://test.com/lol/test").Result;
Assert.False(e);
}
void MakeEnvironment(string dbName,
Action <CacheDatabase, WorkerConfiguration, WorkManager, int> action,
bool depthSearch = false, int memLimit = 100000)
{
var database = new CacheDatabase(dbName);
database.EnsureNew();
var config = new WorkerConfiguration();
config.DepthSearch = depthSearch;
var wm = new WorkManager(config, database, memLimit);
try
{
action(database, config, wm, memLimit);
}
finally
{
database.Dispose();
database.Delete();
}
}
public ElasticsearchFixture()
{
var esUrl = Environment.GetEnvironmentVariable("ELASTICSEARCH_URL") ?? "http://localhost:9200";
var pool = new SingleNodeConnectionPool(new Uri(esUrl));
var connectionSettings = new ConnectionSettings(pool, sourceSerializer: JsonNetSerializer.Default);
Client = new ElasticClient(connectionSettings);
var indexSuffix = "_" + DateTime.UtcNow.Ticks;
Configuration = new ApplicationConfiguration()
{
ElasticsearchUrl = esUrl,
JobsIndexName = "test_jobs" + indexSuffix,
UserIndexName = "test_usersettings" + indexSuffix,
OwnerSettingsIndexName = "test_ownersettings" + indexSuffix,
RepoSettingsIndexName = "test_reposettings" + indexSuffix,
DatasetIndexName = "test_datasets" + indexSuffix,
SchemaIndexName = "test_schemas" + indexSuffix,
FileStorePath = "test_files" + indexSuffix
};
WorkerConfiguration = new WorkerConfiguration {
ElasticsearchUrl = esUrl,
JobsIndexName = "test_jobs" + indexSuffix,
UserIndexName = "test_usersettings" + indexSuffix,
OwnerSettingsIndexName = "test_ownersettings" + indexSuffix,
RepoSettingsIndexName = "test_reposettings" + indexSuffix,
DatasetIndexName = "test_datasets" + indexSuffix,
SchemaIndexName = "test_schemas" + indexSuffix,
FileStorePath = "test_files" + indexSuffix,
GitPath = "",
RepoBaseDir = "test_repos" + indexSuffix,
GitHubClientHeader = "datadock_test"
};
}
/// <summary>
/// Create a worker that will execute `model` using the best backend that is available for a given `device` type.
/// `model` is the associated model. See ModelLoader.cs.
/// `additionalOutputs` are the additional outputs to track but not directly specified by the model.
/// `trimOutputs` are the outputs not discard even if they are specified by the model.
/// `device` is the device type to run worker on. For example `WorkerFactory.Device.GPU` specifies the fast GPU path.
/// `verbose` will log scheduling of layers execution to the console (default == false).
/// </summary>
public static IWorker CreateWorker(Model model, string[] additionalOutputs, string[] trimOutputs, Device device = Device.Auto, bool verbose = false)
{
var type = GetBestTypeForDevice(device);
var workerConfiguration = new WorkerConfiguration(type, verbose);
return(CreateWorker(type, model, additionalOutputs, trimOutputs, workerConfiguration));
}
public void SetupContainer()
{
var currentPath = Path.GetDirectoryName(SandboxHelper.GetAssemblyLocation(typeof(StartupTest).Assembly));
currentPath = Path.Combine(currentPath, "Sandbox");
WorkerConfiguration.SetConfiguration(currentPath, Guid.NewGuid());
}
public void ItCanReadFromJson()
{
var builder = new ConfigurationBuilder()
.SetBasePath(Path.GetFullPath("data/config"))
.AddJsonFile("testSettings.json")
.AddEnvironmentVariables("DDX_");
var config = builder.Build();
var appConfig = new WorkerConfiguration();
config.Bind(appConfig);
appConfig.ElasticsearchUrl.Should().Be("http://some.elasticsearch:9200/");
appConfig.DatasetIndexName.Should().Be("TestDatasets");
appConfig.JobsIndexName.Should().Be("TestJobs");
appConfig.OwnerSettingsIndexName.Should().Be("TestOwners");
appConfig.SchemaIndexName.Should().Be("TestSchemas");
appConfig.RepoSettingsIndexName.Should().Be("TestRepoSettings");
appConfig.UserIndexName.Should().Be("TestUsers");
appConfig.FileStorePath.Should().Be("/path/to/file/store");
appConfig.GitPath.Should().Be("/usr/bin/git");
appConfig.GitHubClientHeader.Should().Be("MyDataDock");
appConfig.PublishUrl.Should().Be("http://mydatadock.com/");
appConfig.RepoBaseDir.Should().Be("/data/repos");
appConfig.SignalRHubUrl.Should().Be("http://datadock.web/progress");
}
public void ItCanUseDefaultValues()
{
var builder = new ConfigurationBuilder()
.SetBasePath(Path.GetFullPath("data/config"))
.AddJsonFile("partialSettings.json")
.AddEnvironmentVariables("DDX_"); // different prefix to avoid any test race conditions with the test that uses DD_ env vars
var config = builder.Build();
var appConfig = new WorkerConfiguration();
config.Bind(appConfig);
// Expected overrides
appConfig.ElasticsearchUrl.Should().Be("http://some.elasticsearch:9200/");
appConfig.FileStorePath.Should().Be("/path/to/file/store");
// Expected defaults
appConfig.DatasetIndexName.Should().Be("datasets");
appConfig.JobsIndexName.Should().Be("jobs");
appConfig.OwnerSettingsIndexName.Should().Be("ownersettings");
appConfig.SchemaIndexName.Should().Be("schemas");
appConfig.RepoSettingsIndexName.Should().Be("reposettings");
appConfig.UserIndexName.Should().Be("users");
appConfig.GitPath.Should().Be("git");
appConfig.GitHubClientHeader.Should().Be("");
appConfig.PublishUrl.Should().Be("http://datadock.io/");
appConfig.RepoBaseDir.Should().Be("/datadock/repositories");
appConfig.SignalRHubUrl.Should().Be("http://web/progress");
}
public void UserAgentMatchingTest()
{
// test if user agents get matched correctly
string robotsTxt = @"
User-agent: Googlebot
Allow: /testdomain2
Allow: /advertisements
Disallow: /artists
User-agent: CryCrawler
Disallow: /admin
Allow: /*?lang=
Disallow: /search/realtime
User-agent: *
Disallow: /testdomain
";
var config = new WorkerConfiguration()
{
UserAgent = "CryCrawler", RespectRobotsExclusionStandard = true
};
var robots = new RobotsHandler(config, new System.Net.Http.HttpClient());
var data = robots.RegisterRobotsTxt("test.com", robotsTxt).Result;
Assert.Single(data.AllowedList);
Assert.Equal(2, data.DisallowedList.Count);
}
public async void ItWrapsTheSchemaInSourceMetadata()
{
var config = new WorkerConfiguration {
PublishUrl = "http://datadock.io/"
};
var proc = new ImportSchemaProcessor(config, _mockSchemeStore.Object, _mockFileStore.Object);
WithJsonSchema(@"{ '@context': 'http://www.w3.org/ns/csvw' }");
#pragma warning disable 618
var job = new JobInfo
#pragma warning restore 618
{
JobType = JobType.SchemaCreate,
UserId = "kal",
OwnerId = "datadock",
RepositoryId = "test",
SchemaFileId = "schemaFileId"
};
await proc.ProcessJob(job, new UserAccount(), _mockProgressLog.Object);
_mockSchemeStore.Verify(s => s.CreateOrUpdateSchemaRecordAsync(It.Is <SchemaInfo>(p =>
p.OwnerId.Equals("datadock") &&
p.RepositoryId.Equals("test") &&
p.Schema.ContainsKey("@context") &&
(p.Schema["@context"] as JValue).Value <string>().Equals("http://www.w3.org/ns/csvw"))));
}
public void GeneralTest2()
{
// general test
string robotsTxt = @"
User-agent: CryCrawler
Disallow: /
User-agent: *
Disallow: /admin
Disallow: /advertisements
Disallow: /artists
Disallow: /artist_commentaries
Disallow: /artist_commentary_versions
Disallow: /artist_versions
Disallow: /bans
Disallow: /comment_votes
Disallow: /comments
Disallow: /counts
Disallow: /delayed_jobs
Disallow: /dmails
Disallow: /favorite
Disallow: /iqdb_queries
Disallow: /ip_bans
Disallow: /janitor_trials";
var config = new WorkerConfiguration()
{
UserAgent = "CryCrawler", RespectRobotsExclusionStandard = true
};
var robots = new RobotsHandler(config, new System.Net.Http.HttpClient());
var data = robots.RegisterRobotsTxt("test.com", robotsTxt).Result;
Assert.Empty(data.AllowedList);
Assert.Single(data.DisallowedList);
Assert.Equal(0, data.WaitTime);
Assert.DoesNotContain(RobotsHandler.GetRegexPattern("/admin"), data.DisallowedList);
Assert.DoesNotContain(RobotsHandler.GetRegexPattern("/bans"), data.DisallowedList);
Assert.DoesNotContain(RobotsHandler.GetRegexPattern("/artist_commentary_versions"), data.DisallowedList);
Assert.DoesNotContain(RobotsHandler.GetRegexPattern("/favorite"), data.DisallowedList);
Assert.DoesNotContain(RobotsHandler.GetRegexPattern("/janitor_trials"), data.DisallowedList);
var e = robots.IsUrlExcluded("http://test2.com/admin").Result;
Assert.False(e);
e = robots.IsUrlExcluded("http://test.com/admin").Result;
Assert.True(e);
e = robots.IsUrlExcluded("http://test.com/admin?url=test").Result;
Assert.True(e);
e = robots.IsUrlExcluded("http://test.com/admin/test/admin?url=test").Result;
Assert.True(e);
e = robots.IsUrlExcluded("http://test.com/test/admin").Result;
Assert.True(e);
e = robots.IsUrlExcluded("http://test.com/admin2").Result;
Assert.True(e);
e = robots.IsUrlExcluded("http://test.com/").Result;
Assert.True(e);
e = robots.IsUrlExcluded("http://test.com").Result;
Assert.True(e);
}
/// <summary>
/// Create a worker with explicitly specified backend `type` to execute the given `model`.
/// `type` is backend type to use. For example `WorkerFactory.Type.Compute` specifies the fast GPU path.
/// `model` is the associated model. See ModelLoader.cs.
/// `additionalOutputs` are the additional outputs to track but not directly specified by the model.
/// `trimOutputs` are the outputs not discard even if they are specified by the model.
/// `verbose` will log scheduling of layers execution to the console.
/// `compareAgainstType` if different than `type` model will be run on those two backend and result of every layer will be compared, checking for divergence. Great for debugging, but very slow because of the sync needed.
/// `differenceAsError` if `compareAgainstType` is used difference will be reported as error is this is true or warning otherwise.
/// </summary>
public static IWorker CreateWorker(Type type, Model model, string[] additionalOutputs, string[] trimOutputs, bool verbose, Type compareAgainstType, CompareOpsUtils.LogLevel differenceLogLevel = CompareOpsUtils.LogLevel.Warning)
{
var workerConfiguration = new WorkerConfiguration(type, verbose);
workerConfiguration.compareAgainstType = compareAgainstType;
workerConfiguration.compareLogLevel = differenceLogLevel;
return(BarracudaBackendsFactory.CreateWorker(type, model, additionalOutputs, trimOutputs, workerConfiguration));
}
/// <summary>
/// Create a worker with explicitly specified backend `type` to execute the given `model`.
/// `type` is backend type to use. For example `WorkerFactory.Type.Compute` specifies the fast GPU path.
/// `model` is the associated model. See ModelLoader.cs.
/// `verbose` will log scheduling of layers execution to the console.
/// `compareAgainstType` if different than `type` model will be run on those two backend and result of every layer will be compared, checking for divergence. Great for debugging, but very slow because of the sync needed.
/// `differenceAsError` if `compareAgainstType` is used difference will be reported as error is this is true or warning otherwise.
/// </summary>
public static IWorker CreateWorker(Type type, Model model, bool verbose, Type compareAgainstType, CompareOpsUtils.LogLevel differenceLogLevel = CompareOpsUtils.LogLevel.Warning)
{
var workerConfiguration = new WorkerConfiguration(type, verbose);
workerConfiguration.compareAgainstType = compareAgainstType;
workerConfiguration.compareLogLevel = differenceLogLevel;
return(CreateWorker(type, model, additionalOutputs: null, trimOutputs: null, workerConfiguration));
}
/// <summary>
/// Call this to apply any customisations to the Unity Worker.
/// </summary>
/// <remarks>
/// It is invalid to call this after this.Connect is called.
/// </remarks>
public static void ApplyConfiguration(WorkerConfigurationData configData, IList <string> commandLineArguments = null)
{
if (connectionLifecycle != null)
{
throw new InvalidOperationException("ApplyConfiguration was called after Connect was called.");
}
configuration = new WorkerConfiguration(configData, commandLineArguments);
}
public FileWatcherService(WorkerConfiguration workerConfiguration)
{
if (workerConfiguration == null)
{
throw new ArgumentNullException(nameof(workerConfiguration));
}
_workerConfiguration = workerConfiguration;
}
public ETLService(
IPUBGApiCaller pubgApiCaller,
IWriteOnlyMongoDbService writeOnlyMongoDbService,
IOptions <WorkerConfiguration> options)
{
_pubgApiCaller = pubgApiCaller;
_writeOnlyMongoDbService = writeOnlyMongoDbService;
_workerConfiguration = options.Value;
}
protected PublishingJobProcessor(WorkerConfiguration configuration,
IOwnerSettingsStore ownerSettingsStore,
IRepoSettingsStore repoSettingsStore,
IGitHubClientFactory gitHubClientFactory)
{
Configuration = configuration;
OwnerSettingsStore = ownerSettingsStore;
RepoSettingsStore = repoSettingsStore;
GitHubClientFactory = gitHubClientFactory;
}
public RobotsHandler(WorkerConfiguration config, HttpClient client, TimeSpan overrideMaxEntryAge = default)
{
this.config = config;
this.maxAge = overrideMaxEntryAge == default ? maxAgeDefault : overrideMaxEntryAge;
http = client;
timer = new Timer(TimeSpan.FromMinutes(5).TotalMinutes);
timer.Elapsed += Timer_Elapsed;
timer.Start();
}
private static WorkerConnectionParameters CreateWorkerConnectionParameters(WorkerConfiguration instance)
{
return(new WorkerConnectionParameters
{
ConnectionParameters = CreateConnectionParameters(instance),
LocatorParameters = CreateLocatorParameters(instance),
LocatorHost = instance.LocatorHost,
ReceptionistHost = instance.ReceptionistHost,
ReceptionistPort = instance.ReceptionistPort,
WorkerId = instance.WorkerId,
});
}
static EndpointConfiguration BuildWorker(Type worker)
{
var config = new WorkerConfiguration(ActorTypeName.Register(worker));
SetReentrancy(worker, config);
SetKeepAliveTimeout(worker, config);
SetReceiver(worker, config);
SetAutorun(worker, config);
SetStickiness(worker, config);
SetInvoker(worker, config);
return(config);
}
public Worker(
IOptions <WorkerConfiguration> workerConfigurationAccessor,
ILogger <Worker> logger)
{
_workerConfiguration = workerConfigurationAccessor.Value;
_logger = logger;
_crontabSchedule = CrontabSchedule.Parse(_workerConfiguration.WorkerCronSchedule,
new CrontabSchedule.ParseOptions {
IncludingSeconds = true
});
_nextRun = _crontabSchedule.GetNextOccurrence(DateTime.Now);
}
public DeleteDatasetProcessor(
WorkerConfiguration configuration,
GitCommandProcessor gitProcessor,
IGitHubClientFactory gitHubClientFactory,
IOwnerSettingsStore ownerSettingsStore,
IRepoSettingsStore repoSettingsStore,
IDatasetStore datasetStore,
IDataDockRepositoryFactory repositoryFactory) : base(configuration, ownerSettingsStore, repoSettingsStore,
gitHubClientFactory)
{
_git = gitProcessor;
_datasetStore = datasetStore;
_repositoryFactory = repositoryFactory;
}
public static IHostBuilder CreateHostBuilder(string[] args) =>
Host.CreateDefaultBuilder(args)
.ConfigureServices((hostContext, services) =>
{
IConfiguration configuration;
var serviceProvider = services.BuildServiceProvider();
configuration = serviceProvider.GetRequiredService <IConfiguration>();
WorkerConfiguration workerConfiguration = new WorkerConfiguration(services, configuration);
services.AddSingleton(workerConfiguration);
workerConfiguration.Configure();
});
/// <inheritdoc />
// ReSharper disable once TooManyDependencies
protected SteamBotWorker(
SteamBotOptions options,
DataStoreBase dataStore,
Coordinator coordinator,
WorkerConfiguration configuration,
Logger logger) :
base(
options,
new LoggerWrapper(configuration.Alias ?? configuration.WorkerType + " [" + options.Username + "]", logger)
)
{
DataStore = dataStore;
Coordinator = coordinator;
Configuration = configuration;
options.Coordinator = coordinator;
Options = options;
}
public void PriorityMatchingTest()
{
// test if more detailed rules have priority when matching patterns
string robotsTxt = @"
User-agent: Googlebot
Allow: /testdomain2
Allow: /advertisements
Disallow: /artists
User-agent: CryCrawler
Disallow: /admin
Disallow: /search
Allow: /search/test
User-agent: *
Disallow: /testdomain
";
var config = new WorkerConfiguration()
{
UserAgent = "CryCrawler", RespectRobotsExclusionStandard = true
};
var robots = new RobotsHandler(config, new System.Net.Http.HttpClient());
var data = robots.RegisterRobotsTxt("test.com", robotsTxt).Result;
Assert.Single(data.AllowedList);
Assert.Equal(2, data.DisallowedList.Count);
var e = robots.IsUrlExcluded("http://test.com/admin").Result;
Assert.True(e);
e = robots.IsUrlExcluded("http://test.com/search").Result;
Assert.True(e);
e = robots.IsUrlExcluded("http://test.com/search/test").Result;
Assert.False(e);
e = robots.IsUrlExcluded("http://test.com/search/test/anothertest").Result;
Assert.False(e);
e = robots.IsUrlExcluded("http://test.com/search/test?url=23").Result;
Assert.False(e);
e = robots.IsUrlExcluded("http://test.com/search/test2").Result;
Assert.True(e);
e = robots.IsUrlExcluded("http://test.com/search/nottest").Result;
Assert.True(e);
}
