//--- Constructors ---
public S3Storage(XDoc configuration, ILog log) {
_timerFactory = TaskTimerFactory.Create(this);
_log = log;
_publicKey = configuration["publickey"].AsText;
_privateKey = configuration["privatekey"].AsText;
_bucket = configuration["bucket"].AsText;
_prefix = configuration["prefix"].AsText;
if(string.IsNullOrEmpty(_publicKey)) {
throw new ArgumentException("Invalid Amazon S3 publickey");
}
if(string.IsNullOrEmpty(_privateKey)) {
throw new ArgumentException("Invalid Amazon S3 privatekey");
}
if(string.IsNullOrEmpty(_bucket)) {
throw new ArgumentException("Invalid Amazon S3 bucket");
}
if(string.IsNullOrEmpty(_prefix)) {
throw new ArgumentException("Invalid Amazon S3 prefix");
}
_tempDirectory = Path.Combine(Path.GetTempPath(), "s3_cache_" + XUri.EncodeSegment(_prefix));
if(Directory.Exists(_tempDirectory)) {
Directory.Delete(_tempDirectory, true);
}
Directory.CreateDirectory(_tempDirectory);
_allowRedirects = configuration["allowredirects"].AsBool ?? false;
_redirectTimeout = TimeSpan.FromSeconds(configuration["redirecttimeout"].AsInt ?? 60);
_cacheTtl = (configuration["cachetimeout"].AsInt ?? 60 * 60).Seconds();
// initialize S3 plug
_s3 = Plug.New("http://s3.amazonaws.com", TimeSpan.FromSeconds(configuration["timeout"].AsDouble ?? DEFAUTL_S3_TIMEOUT)).WithPreHandler(S3AuthenticationHeader).At(_bucket);
}
//--- Class methods ---
public static InstanceManager New(DekiWikiService dekiService, TaskTimerFactory timerFactory) {
InstanceManager mgr;
var srcString = dekiService.Config["wikis/@src"].AsText;
if(!string.IsNullOrEmpty(srcString)) {
XUri remoteDirUri;
if(!XUri.TryParse(srcString, out remoteDirUri)) {
//TODO: build a specialized exception out of this
throw new ApplicationException(string.Format("Configuration is not valid. wikis/@src ({0})is not a valid url!", srcString));
}
mgr = new RemoteInstanceManager(dekiService, timerFactory, remoteDirUri, dekiService.TempPath);
} else {
mgr = new LocalInstanceManager(dekiService, timerFactory);
}
mgr._maxInstances = dekiService.Config["wikis/@max"].AsInt ?? int.MaxValue;
var timeoutSecs = dekiService.Config["wikis/@ttl"].AsDouble;
if(timeoutSecs == null || timeoutSecs == 0) {
mgr._inactiveInstanceTimeOut = TimeSpan.MaxValue;
} else {
mgr._inactiveInstanceTimeOut = TimeSpan.FromSeconds(timeoutSecs.Value);
}
var retryInterval = dekiService.Config["wikis/@retry-interval"].AsDouble ?? 10;
mgr._abandonedInstanceRetryInterval = TimeSpan.FromSeconds(retryInterval);
mgr._minInstanceIdletime = TimeSpan.FromSeconds(dekiService.Config["wikis/@idletime"].AsDouble ?? 60);
return mgr;
}
//--- Constructors ---
public MemoryPubSubDispatchQueueRepository(TaskTimerFactory taskTimerFactory, TimeSpan retryTime)
{
if(taskTimerFactory == null) {
throw new ArgumentNullException("taskTimerFactory");
}
_taskTimerFactory = taskTimerFactory;
_retryTime = retryTime;
}
/// <summary>
/// Clone the current task environment with a provided dispatch queue and task timer factory.
/// </summary>
/// <param name="dispatchQueue">Dispatch queue to use for the environment.</param>
/// <param name="taskTimerFactory"><see cref="TaskTimer"/> factory to use for the environment.</param>
/// <returns>A new <see cref="TaskEnv"/> instance.</returns>
public static TaskEnv Clone(IDispatchQueue dispatchQueue, TaskTimerFactory taskTimerFactory)
{
if (dispatchQueue == null)
{
throw new ArgumentNullException("dispatchQueue");
}
return(new TaskEnv(_currentEnv, dispatchQueue, taskTimerFactory));
}
//--- Constructors ---
/// <summary>
/// Constructor for creating an instance.
/// </summary>
/// <param name="client">ISqsClient instance.</param>
/// <param name="queueName">Queue name.</param>
/// <param name="timerFactory">TimerFactory instance.</param>
public SqsQueueDelayedSendClient(ISqsClient client, SqsQueueName queueName, TaskTimerFactory timerFactory)
{
if(client == null) {
throw new ArgumentNullException("client");
}
this.QueueName = queueName;
_client = client;
_timedSendAccumulator = new TimedAccumulator<KeyValuePair<int, string>>(items => AsyncUtil.ForkBackgroundSender(() => BatchSendMessages(items)), SqsUtils.MAX_NUMBER_OF_BATCH_SEND_MESSAGES, AUTOFLUSH_TIME, timerFactory);
}
//--- Constructors ---
/// <summary>
/// Constructor for creating an instance.
/// </summary>
/// <param name="client">ISqsClient instance.</param>
/// <param name="timerFactory">TimeFactory instance.</param>
public SqsDelayedSendClient(ISqsClient client, TaskTimerFactory timerFactory)
{
if(client == null) {
throw new ArgumentNullException("client");
}
if(timerFactory == null) {
throw new ArgumentNullException("timerFactory");
}
_client = client;
_timerFactory = timerFactory;
}
//--- Class Methods ---
/// <summary>
/// Create a new factory.
/// </summary>
/// <param name="owner">
/// The entity that this factory belongs to. Owner is used for usage tracking and is stored as a weak reference, which means
/// that the factory will not prevent the owner from being garbage collected.
/// </param>
/// <returns>New factory instance.</returns>
public static TaskTimerFactory Create(object owner)
{
if (owner == null)
{
throw new ArgumentNullException("owner", "Cannot create TaskTimerFactory without an owner");
}
lock (_factories) {
var factory = new TaskTimerFactory(owner);
_factories.Add(factory);
return(factory);
}
}
private TaskEnv(TaskEnv env, IDispatchQueue dispatchQueue, TaskTimerFactory taskTimerFactory)
{
if (env != null)
{
foreach (var entry in env)
{
var cloneable = entry.Value as ITaskLifespan;
Add(entry.Key, cloneable == null ? entry.Value : cloneable.Clone());
}
}
_taskTimerFactory = taskTimerFactory;
_dispatchQueue = (dispatchQueue is ImmediateDispatchQueue) ? null : dispatchQueue;
}
//--- Constructors ---
public LocalInstanceManager(DekiWikiService dekiService, TaskTimerFactory timerFactory) : base(dekiService, timerFactory) {
var dekiServiceConfig = dekiService.Config;
if (dekiServiceConfig["wikis/config"].IsEmpty) {
//Not in cluster mode (no other wikis defined): run all hosts under default wiki.
AssociateHostnameWithWiki("*", DEFAULT_WIKI_ID);
} else {
foreach (XDoc wikiDoc in dekiServiceConfig["wikis/config"]) {
var wikiId = wikiDoc["@id"].AsText;
AssociateHostnameWithWiki(wikiDoc["host"].Select(hostDoc => hostDoc.Contents).ToArray(), wikiId);
}
}
}
/// <summary>
/// Execute an action in a new environment.
/// </summary>
/// <param name="handler">Action to execute.</param>
/// <param name="timerFactory">The <see cref="TaskTimer"/> factory to use in the execution environment.</param>
/// <returns><see langword="null"/> if the handler was executed sucessfully, the captured exception otherwise.</returns>
public static Exception ExecuteNew(Action handler, TaskTimerFactory timerFactory)
{
if (handler == null)
{
throw new ArgumentNullException("handler");
}
var env = New(timerFactory);
env.Acquire();
var exception = env.InvokeNow(handler);
env.Release();
return(exception);
}
//--- Constructors ---
public RemoteInstanceManager(DekiWikiService dekiService, TaskTimerFactory timerFactory, XUri directoryUri, string tempPath) : base(dekiService, timerFactory) {
// validate temp folder
_tempPath = tempPath;
if(!Directory.Exists(_tempPath)) {
throw new ArgumentException("temp folder does not exist", "tempPath");
}
// check remote directory
_directory = Plug.New(directoryUri);
var testMsg = _directory.GetAsync().Wait();
if(!testMsg.IsSuccessful) {
_log.WarnFormat("Error validating remote deki portal service at '{0}'", directoryUri);
}
}
//--- Construtors
public MemoryPubSubDispatchQueue(string location, TaskTimerFactory taskTimerFactory, TimeSpan retryTime, Func<DispatchItem, Result<bool>> handler)
{
if(string.IsNullOrEmpty(location)) {
throw new ArgumentNullException("location");
}
if(taskTimerFactory == null) {
throw new ArgumentNullException("taskTimerFactory");
}
if(handler == null) {
throw new ArgumentNullException("handler");
}
_location = location;
_retryTime = retryTime;
_queueTimer = taskTimerFactory.New(RetryDequeue, null);
_dequeueHandler = handler;
}
//--- Constructors ---
/// <summary>
/// Create new client instance
/// </summary>
/// <param name="config">Client configuration.</param>
/// <param name="timerFactory">Timer factory.</param>
public AwsS3Client(AwsS3ClientConfig config, TaskTimerFactory timerFactory)
{
_config = config;
_bucketPlug = Plug.New(_config.Endpoint.S3Uri)
.WithS3Authentication(_config.PrivateKey, _config.PublicKey)
.WithTimeout(_config.Timeout)
.At(_config.Bucket);
_rootPlug = _bucketPlug;
if(!string.IsNullOrEmpty(_config.RootPath)) {
_keyRootParts = _config.RootPath.Split(new[] { config.Delimiter }, StringSplitOptions.RemoveEmptyEntries);
if(_keyRootParts.Any()) {
_rootPlug = _rootPlug.At(_keyRootParts);
}
}
_expirationEntries = new ExpiringHashSet<string>(timerFactory);
_expirationEntries.EntryExpired += OnDelete;
}
//--- Construtors
public PersistentPubSubDispatchQueue(string queuePath, TaskTimerFactory taskTimerFactory, TimeSpan retryTime, Func<DispatchItem, Result<bool>> handler)
{
if(string.IsNullOrEmpty(queuePath)) {
throw new ArgumentNullException("location");
}
if(taskTimerFactory == null) {
throw new ArgumentNullException("taskTimerFactory");
}
if(handler == null) {
throw new ArgumentNullException("handler");
}
_queuePath = queuePath;
_retryTime = retryTime;
_queueTimer = taskTimerFactory.New(RetryDequeue, null);
_queue = new TransactionalQueue<DispatchItem>(new MultiFileQueueStream(queuePath), new DispatchItemSerializer()) {
DefaultCommitTimeout = TimeSpan.MaxValue
};
_dequeueHandler = handler;
Kick();
}
//--- Constructors ---
public PersistentPubSubDispatchQueueRepository(string queueRootPath, TaskTimerFactory taskTimerFactory, TimeSpan retryTime)
{
_queueRootPath = queueRootPath;
_taskTimerFactory = taskTimerFactory;
_retryTime = retryTime;
if(!Directory.Exists(_queueRootPath)) {
Directory.CreateDirectory(_queueRootPath);
}
foreach(var setFile in Directory.GetFiles(_queueRootPath, "*.xml")) {
PubSubSubscriptionSet set;
try {
var setDoc = XDocFactory.LoadFrom(setFile, MimeType.TEXT_XML);
var location = setDoc["@location"].AsText;
var accessKey = setDoc["@accesskey"].AsText;
set = new PubSubSubscriptionSet(setDoc, location, accessKey);
} catch(Exception e) {
_log.Warn(string.Format("unable to retrieve and re-register subscription for location", Path.GetFileNameWithoutExtension(setFile)), e);
continue;
}
_uninitializedSets.Add(set);
}
}
//--- Constructors ---
public SearchInstanceData(string indexPath, Analyzer analyzer, UpdateDelayQueue queue, TimeSpan commitInterval, TaskTimerFactory taskTimerFactory) {
_analyzer = analyzer;
_directory = FSDirectory.GetDirectory(indexPath);
// Note (arnec): Needed with SimpleFSLock, since a hard shutdown will have left the lock dangling
IndexWriter.Unlock(_directory);
try {
_writer = new IndexWriter(_directory, _analyzer, IndexWriter.MaxFieldLength.UNLIMITED);
} catch(CorruptIndexException e) {
_log.WarnFormat("The Search index at {0} is corrupt. You must repair or delete it before restarting the service. If you delete it, you must rebuild your index after service restart.", indexPath);
if(e.Message.StartsWith("Unknown format version")) {
_log.Warn("The index is considered corrupt because it's an unknown version. Did you accidentally downgrade your install?");
}
throw;
}
_reader = IndexReader.Open(_directory);
_searcher = new IndexSearcher(_reader);
_queue = queue;
_commitInterval = commitInterval;
_taskTimerFactory = taskTimerFactory;
if(_commitInterval != TimeSpan.Zero) {
_commitTimer = _taskTimerFactory.New(_commitInterval, Commit, null, TaskEnv.None);
}
}
//--- Constructors ---
public Listener(string queuename, Action<AwsSqsMessage> callback, IAwsSqsClient client, TaskTimerFactory timerFactory, TimeSpan interval)
{
_queuename = queuename;
_callback = callback;
_client = client;
_cache = new ExpiringHashSet<string>(timerFactory);
_cacheTimer = ((interval.TotalSeconds * 2 < 60) ? 60 : interval.TotalSeconds * 2 + 1).Seconds();
_pollTimer = timerFactory.New(tt => Coroutine.Invoke(PollSqs, new Result()).WhenDone(r => _pollTimer.Change(interval, TaskEnv.None)), null);
_pollTimer.Change(0.Seconds(), TaskEnv.None);
}
//--- Constructors ---
/// <summary>
/// Create a new factory instance.
/// </summary>
/// <param name="timer">Timer factory to provide to the cache instance.</param>
public InMemoryKeyValueCacheFactory(TaskTimerFactory timer)
: this(DEFAULT_MAX_SIZE, timer)
{
}
//--- Constructors ---
private TaskEnv(IDispatchQueue dispatchQueue, TaskTimerFactory taskTimerFactory)
{
_taskTimerFactory = taskTimerFactory;
_dispatchQueue = (dispatchQueue is ImmediateDispatchQueue) ? null : dispatchQueue;
}
/// <summary>
/// Create a new factory instance.
/// </summary>
/// <param name="maxSize">Maximum number of bytes of memory the cache should use for stored items</param>
/// <param name="timer">Timer factory to provide to the cache instance.</param>
public InMemoryKeyValueCacheFactory(int maxSize, TaskTimerFactory timer)
: base(timer, (s, t) => new InMemoryKeyValueCache(s, maxSize, t))
{
}
/// <summary>
/// Clone the current task environment with a task timer factory.
/// </summary>
/// <param name="taskTimerFactory"><see cref="TaskTimer"/> factory to use for the environment.</param>
/// <returns>A new <see cref="TaskEnv"/> instance.</returns>
public static TaskEnv Clone(TaskTimerFactory taskTimerFactory)
{
return(new TaskEnv(_currentEnv, Async.GlobalDispatchQueue, taskTimerFactory));
}
/// <summary>
/// Create a new environment.
/// </summary>
/// <param name="taskTimerFactory"><see cref="TaskTimer"/> factory to use for the environment.</param>
/// <returns>A new <see cref="TaskEnv"/> instance.</returns>
public static TaskEnv New(TaskTimerFactory taskTimerFactory)
{
return(new TaskEnv(Async.GlobalDispatchQueue, taskTimerFactory));
}
//--- Constructor ---
public DekiInstance(DekiWikiService deki, string id, XDoc instanceConfig) {
if(deki == null) {
throw new ArgumentNullException("deki");
}
if(string.IsNullOrEmpty(id)) {
throw new ArgumentNullException("id");
}
this.Id = id;
this.TimerFactory = TaskTimerFactory.Create(this);
this.Cache = new DreamCache(TimerFactory);
var cacheFactory = new InMemoryKeyValueCacheFactory(TimerFactory);
var searchSerializer = new SearchSerializer();
cacheFactory.SetSerializer<SearchResult>(searchSerializer);
cacheFactory.SetSerializer<SearchResultDetail>(searchSerializer);
this.SearchCache = cacheFactory.Create();
this.Config = instanceConfig;
this.Log = LogManager.GetLogger(deki.GetType().Name + "_" + id);
_deki = deki;
_status = DekiInstanceStatus.CREATED;
foreach(XDoc hostDoc in Config["host"]) {
string host = hostDoc.Contents;
if(!StringUtil.EqualsInvariantIgnoreCase(host, "*")) {
string port = hostDoc["@port"].AsText;
string scheme = hostDoc["@https"].AsBool.GetValueOrDefault() ? "https://" : "http://";
string uri = scheme + host + (string.IsNullOrEmpty(port) ? "" : ":" + port);
_canonicalUri = new XUri(uri);
_log.DebugFormat("divined canonical use from hosts as {0}", _canonicalUri);
break;
}
}
if(_canonicalUri == null) {
// Note (arnec): this is a best guess fallback. It will only work in these scenarios:
// a) The host was set up with a uri.public that has ends in @api and with the @api points to the site uri, or
// b) The api lives on the same machine as the site, so that deriving uri.public for the host from the machine
// IP happens to point to the same machine
// Either way it relies on the hard-coded assumption that the api is accessible via {site}/@api
_canonicalUri = DreamContext.Current.ServerUri;
if(_canonicalUri.LastSegment.EqualsInvariantIgnoreCase("@api")) {
_canonicalUri = _canonicalUri.WithoutLastSegment();
}
_log.DebugFormat("using server uri as canonical uri: {0}", _canonicalUri);
} else {
// Note (arnec): Propagating a much hard-coded assumption, i.e. that the Api for any Deki instance can be accessed
// at the instances' canonical uri plus @api
// register the api uri with the dream host so that requests originating from within Dream are guaranteed to be locally routed
_deki.Env.At("status", "aliases").Post(new XDoc("aliases").Elem("uri.alias", _canonicalUri.At("@api")), new Result<DreamMessage>());
}
}
//--- Constructors ---
protected InstanceManager(DekiWikiService dekiService, TaskTimerFactory timerFactory) {
_dekiService = dekiService;
_timerFactory = timerFactory;
}
protected virtual Yield PutConfig(DreamContext context, DreamMessage request, Result<DreamMessage> response)
{
XDoc config = request.ToDocument();
if(config.Name != "config") {
throw new DreamBadRequestException("bad document type");
}
if(IsStarted) {
throw new DreamBadRequestException("service must be stopped first");
}
_timerFactory = TaskTimerFactory.Create(this);
// configure service container
var components = config["components"];
var servicecontainer = _env.CreateServiceContainer(this);
var builder = new ContainerBuilder();
builder.Register(_timerFactory).ExternallyOwned();
if(!components.IsEmpty) {
_log.Debug("registering service level module");
builder.RegisterModule(new XDocAutofacContainerConfigurator(components, DreamContainerScope.Service));
}
builder.Build(servicecontainer);
// call container-less start (which contains shared start logic)
yield return Coroutine.Invoke(Start, request.ToDocument(), new Result());
// call start with container for sub-classes that want to resolve instances at service start
yield return Coroutine.Invoke(Start, config, servicecontainer, new Result());
response.Return(DreamMessage.Ok(new XDoc("service-info")
.Start("private-key")
.Add(DreamCookie.NewSetCookie("service-key", PrivateAccessKey, Self.Uri).AsSetCookieDocument)
.End()
.Start("internal-key")
.Add(DreamCookie.NewSetCookie("service-key", InternalAccessKey, Self.Uri).AsSetCookieDocument)
.End()
));
}
//--- Class Methods ---
/// <summary>
/// Create a new factory.
/// </summary>
/// <param name="owner">
/// The entity that this factory belongs to. Owner is used for usage tracking and is stored as a weak reference, which means
/// that the factory will not prevent the owner from being garbage collected.
/// </param>
/// <returns>New factory instance.</returns>
public static TaskTimerFactory Create(object owner)
{
if(owner == null) {
throw new ArgumentNullException("owner", "Cannot create TaskTimerFactory without an owner");
}
lock(_factories) {
var factory = new TaskTimerFactory(owner);
_factories.Add(factory);
return factory;
}
}
protected virtual Yield PutConfig(DreamContext context, DreamMessage request, Result<DreamMessage> response)
{
XDoc config = request.ToDocument();
if(config.Name != "config") {
throw new DreamBadRequestException("bad document type");
}
if(IsStarted) {
throw new DreamBadRequestException("service must be stopped first");
}
_timerFactory = TaskTimerFactory.Create(this);
// configure service container
var lifetimeScope = _env.CreateServiceLifetimeScope(this, (c, b) => PreInitializeLifetimeScope(c, b, config));
// call container-less start (which contains shared start logic)
yield return Coroutine.Invoke(Start, request.ToDocument(), new Result());
// call start with container for sub-classes that want to resolve instances at service start
yield return Coroutine.Invoke(Start, config, lifetimeScope, new Result());
response.Return(DreamMessage.Ok(new XDoc("service-info")
.Start("private-key")
.Add(DreamCookie.NewSetCookie("service-key", PrivateAccessKey, Self.Uri).AsSetCookieDocument)
.End()
.Start("internal-key")
.Add(DreamCookie.NewSetCookie("service-key", InternalAccessKey, Self.Uri).AsSetCookieDocument)
.End()
));
}
//--- Constructors ---
public SqsPollClient(IAwsSqsClient client, TaskTimerFactory timerFactory)
{
_client = client;
_timerFactory = timerFactory;
}
