/// <summary>
/// Store a file at a path.
/// </summary>
/// <param name="path">Storage path.</param>
/// <param name="fileHandle">File to store.</param>
public void PutFile(string path, AwsS3FileHandle fileHandle)
{
if (IsDirectoryPath(path))
{
throw new InvalidOperationException(string.Format("cannot put a file at a path with a trailing {0}", _config.Delimiter));
}
var p = _rootPlug.AtPath(path);
if (fileHandle.TimeToLive.HasValue)
{
var expiration = fileHandle.Expiration ?? DateTime.UtcNow.Add(fileHandle.TimeToLive.Value);
p = p.WithHeader(EXPIRE, expiration.ToEpoch().ToString())
.WithHeader(TTL, fileHandle.TimeToLive.Value.TotalSeconds.ToString());
_expirationEntries.SetOrUpdate(path, expiration, fileHandle.TimeToLive.Value);
}
if (!string.IsNullOrEmpty(fileHandle.CacheControl))
{
p = p.WithHeader(CACHE_CONTROL, fileHandle.CacheControl);
}
var request = DreamMessage.Ok(fileHandle.MimeType, fileHandle.Size, fileHandle.Stream);
var response = p.Put(request, new Result <DreamMessage>()).Wait();
if (response.IsSuccessful)
{
return;
}
throw new DreamResponseException(response);
}
private ExpiringHashSet <string> .Entry SyncMeta(string filePath)
{
lock (_expirationEntries) {
DateTime?expire = null;
var ttl = TimeSpan.Zero;
var metaPath = GetMetaPath(filePath);
if (File.Exists(metaPath))
{
var meta = XDocFactory.LoadFrom(metaPath, MimeType.XML);
expire = meta["expire.date"].AsDate;
ttl = TimeSpan.FromSeconds(meta["expire.ttl"].AsDouble ?? 0);
}
if (expire.HasValue)
{
// set up expiration
_expirationEntries.SetOrUpdate(filePath, expire.Value, ttl);
}
else
{
// no expiration anymore, so expiration needs to be removed
_expirationEntries.Delete(filePath);
}
return(_expirationEntries[filePath]);
}
}
//--- Methods ---
private IEnumerator <IYield> PollSqs(Result result)
{
_log.DebugFormat("polling SQS queue '{0}'", _queuename);
while (!_isDisposed)
{
Result <IEnumerable <AwsSqsMessage> > messageResult;
yield return(messageResult = _client.ReceiveMax(_queuename, new Result <IEnumerable <AwsSqsMessage> >()).Catch());
if (messageResult.HasException)
{
LogError(messageResult.Exception, "fetching messages");
result.Return();
yield break;
}
var messages = messageResult.Value;
if (!messages.Any())
{
result.Return();
yield break;
}
foreach (var msg in messages.Where(msg => !_cache.SetOrUpdate(msg.MessageId, _cacheTimer)))
{
try {
_log.DebugFormat("dispatching message '{0}' from queue '{1}'", msg.MessageId, _queuename);
_callback(msg);
} catch (Exception e) {
_log.Warn(
string.Format("dispatching message '{0}' from queue '{1}' threw '{2}': {3}",
msg.MessageId,
_queuename,
e,
e.Message
),
e
);
continue;
}
Result <AwsSqsResponse> deleteResult;
yield return(deleteResult = _client.Delete(msg, new Result <AwsSqsResponse>()).Catch());
if (deleteResult.HasException)
{
LogError(deleteResult.Exception, string.Format("deleting message '{0}'", msg.MessageId));
}
else
{
_cache.SetOrUpdate(msg.MessageId, _cacheTimer);
}
}
}
result.Return();
}
public void Clearing_set_releases_all_items()
{
var changed = false;
var ttl = 10.Seconds();
var set = new ExpiringHashSet <string>(TaskTimerFactory.Current);
set.SetOrUpdate("a", ttl);
set.SetOrUpdate("b", ttl);
set.SetOrUpdate("v", ttl);
set.CollectionChanged += (s, e) => changed = true;
Assert.AreEqual(3, set.Count());
Assert.IsFalse(changed);
set.Clear();
Assert.AreEqual(0, set.Count());
Assert.IsTrue(changed);
}
public void Set_will_fire_in_order_of_expirations()
{
var expired = new ManualResetEvent(false);
var entries = new List <int>();
var set = new ExpiringHashSet <int>(TaskTimerFactory.Current);
set.EntryExpired += (s, e) => { entries.Add(e.Entry.Value); expired.Set(); };
set.SetOrUpdate(3, TimeSpan.FromMilliseconds(1600));
set.SetOrUpdate(2, TimeSpan.FromMilliseconds(1500));
set.SetOrUpdate(5, TimeSpan.FromMilliseconds(3000));
set.SetOrUpdate(4, TimeSpan.FromMilliseconds(2000));
set.SetOrUpdate(1, TimeSpan.FromMilliseconds(500));
Assert.IsTrue(expired.WaitOne(1000));
Assert.AreEqual(1, entries.Count);
Assert.IsTrue(Wait.For(() => entries.Count == 5, TimeSpan.FromSeconds(5000)));
Assert.AreEqual(new[] { 1, 2, 3, 4, 5 }, entries.ToArray());
}
public void New_item_with_more_recent_expiration_will_fire_at_expected_time()
{
var expired = new ManualResetEvent(false);
var entries = new List <int>();
var set = new ExpiringHashSet <int>(TaskTimerFactory.Current);
set.EntryExpired += (s, e) => { entries.Add(e.Entry.Value); expired.Set(); };
var n = 1000;
for (var i = 0; i < n; i++)
{
set.SetOrUpdate(i, TimeSpan.FromMinutes(10));
}
set.SetOrUpdate(100000, TimeSpan.FromSeconds(2));
Assert.IsTrue(expired.WaitOne(4000));
Assert.AreEqual(1, entries.Count);
Assert.AreEqual(n, set.Count());
}
public void Can_set_ttl_on_existing_item()
{
var expired = new ManualResetEvent(false);
var changed = new ManualResetEvent(false);
var i = 42;
var ttl = TimeSpan.FromSeconds(10);
ExpiringHashSet <int> .Entry entry = null;
var set = new ExpiringHashSet <int>(TaskTimerFactory.Current);
set.EntryExpired += (s, e) => { entry = e.Entry; expired.Set(); };
set.CollectionChanged += (s, e) => changed.Set();
set.SetOrUpdate(i, ttl);
Assert.IsTrue(changed.WaitOne(2000));
changed.Reset();
Assert.IsFalse(expired.WaitOne(2000));
set.SetOrUpdate(i, TimeSpan.FromSeconds(1));
Assert.IsTrue(changed.WaitOne(5000));
Assert.IsTrue(expired.WaitOne(5000));
}
public void Disposing_set_expires_all_items_before_dispose_returns_but_does_not_trigger_collection_changed()
{
var expired = false;
var changed = false;
var expiredEntries = new List <int>();
var ttl = TimeSpan.FromSeconds(1);
var set = new ExpiringHashSet <int>(TaskTimerFactory.Current);
set.EntryExpired += (s, e) => { expiredEntries.Add(e.Entry.Value); expired = true; };
set.CollectionChanged += (s, e) => { changed = true; };
set.SetOrUpdate(12, ttl);
set.SetOrUpdate(21, ttl);
Assert.IsFalse(expired, "expired was triggered");
Assert.IsTrue(changed, "changed wasn't triggered");
changed = false;
set.Dispose();
Assert.IsFalse(changed, "changed was triggered");
Assert.IsTrue(expired, "expired wasn't triggered");
Assert.AreEqual(new[] { 12, 21 }, expiredEntries.OrderBy(x => x).ToArray());
}
public void Can_retrieve_item_checking_when()
{
var set = new ExpiringHashSet <int>(TaskTimerFactory.Current);
var v = 42;
var when = GlobalClock.UtcNow.AddDays(1);
set.SetOrUpdate(v, when);
var entry = set[v];
Assert.AreEqual(v, entry.Value);
Assert.AreEqual(when, entry.When);
}
public void Access_on_tasktimerfactory_ctor_set_does_not_reset_expiration()
{
var i = 42;
var ttl = TimeSpan.FromSeconds(10);
var set = new ExpiringHashSet<int>(TaskTimerFactory.Current);
var expireTime = GlobalClock.UtcNow.AddSeconds(10);
set.SetOrUpdate(i, expireTime);
Thread.Sleep(TimeSpan.FromSeconds(2));
var entry = set[i];
AssertEx.AreEqual(ttl, entry.TTL);
Assert.AreEqual(expireTime, entry.When);
}
public void Can_retrieve_item_checking_ttl()
{
var set = new ExpiringHashSet <int>(TaskTimerFactory.Current);
var v = 42;
var ttl = TimeSpan.FromSeconds(10);
set.SetOrUpdate(v, ttl);
var entry = set[v];
Assert.AreEqual(v, entry.Value);
Assert.AreEqual(ttl, entry.TTL);
}
public void Access_on_autoRefresh_set_does_reset_expiration()
{
var i = 42;
var ttl = TimeSpan.FromSeconds(10);
var set = new ExpiringHashSet<int>(TaskTimerFactory.Current, true);
var expireTime = GlobalClock.UtcNow.AddSeconds(10);
set.SetOrUpdate(i, expireTime);
Thread.Sleep(TimeSpan.FromSeconds(2));
var entry = set[i];
AssertEx.AreEqual(ttl, entry.TTL);
Assert.AreNotEqual(expireTime, entry.When);
Assert.GreaterOrEqual(entry.When, expireTime);
Assert.LessOrEqual(entry.When, expireTime.AddSeconds(3));
}
public void Access_on_tasktimerfactory_ctor_set_does_not_reset_expiration()
{
var i = 42;
var ttl = TimeSpan.FromSeconds(10);
var set = new ExpiringHashSet <int>(TaskTimerFactory.Current);
var expireTime = GlobalClock.UtcNow.AddSeconds(10);
set.SetOrUpdate(i, expireTime);
Thread.Sleep(TimeSpan.FromSeconds(2));
var entry = set[i];
AssertEx.AreEqual(ttl, entry.TTL);
Assert.AreEqual(expireTime, entry.When);
}
public void Refresh_in_autoRefresh_only_fires_every_half_second()
{
var k = 42;
var ttl = TimeSpan.FromSeconds(10);
var set = new ExpiringHashSet <int>(TaskTimerFactory.Current, true);
var expireTime = GlobalClock.UtcNow.AddSeconds(10);
set.SetOrUpdate(k, expireTime);
var when = set[k].When;
Thread.Sleep(200);
Assert.AreEqual(when, set[k].When);
Thread.Sleep(1000);
Assert.Less(when, set[k].When);
}
public void Access_on_autoRefresh_set_does_reset_expiration()
{
var i = 42;
var ttl = TimeSpan.FromSeconds(10);
var set = new ExpiringHashSet <int>(TaskTimerFactory.Current, true);
var expireTime = GlobalClock.UtcNow.AddSeconds(10);
set.SetOrUpdate(i, expireTime);
Thread.Sleep(TimeSpan.FromSeconds(2));
var entry = set[i];
AssertEx.AreEqual(ttl, entry.TTL);
Assert.AreNotEqual(expireTime, entry.When);
Assert.GreaterOrEqual(entry.When, expireTime);
Assert.LessOrEqual(entry.When, expireTime.AddSeconds(3));
}
public void Can_iterate_over_items()
{
var set = new ExpiringHashSet <int>(TaskTimerFactory.Current);
var n = 10;
for (var i = 1; i <= n; i++)
{
set.SetOrUpdate(i, TimeSpan.FromSeconds(i));
}
var items = from x in set select x;
Assert.AreEqual(n, items.Count());
for (var i = 1; i <= n; i++)
{
var j = i;
Assert.IsTrue((from x in set where x.Value == j && x.TTL == TimeSpan.FromSeconds(j) select x).Any());
}
}
public void Can_set_item_via_when()
{
var expired = new ManualResetEvent(false);
var changed = new ManualResetEvent(false);
var i = 42;
var when = GlobalClock.UtcNow.AddSeconds(1);
ExpiringHashSet <int> .Entry entry = null;
var set = new ExpiringHashSet <int>(TaskTimerFactory.Current);
set.EntryExpired += (s, e) => { entry = e.Entry; expired.Set(); };
set.CollectionChanged += (s, e) => changed.Set();
set.SetOrUpdate(i, when);
Assert.IsTrue(changed.WaitOne(5000));
Assert.IsTrue(expired.WaitOne(5000));
Assert.AreEqual(i, entry.Value);
Assert.AreEqual(when, entry.When);
Assert.IsNull(set[i]);
}
public void Can_set_item_via_ttl()
{
_log.Debug("running test");
var expired = new ManualResetEvent(false);
var changed = new ManualResetEvent(false);
var i = 42;
var ttl = TimeSpan.FromSeconds(1);
ExpiringHashSet <int> .Entry entry = null;
var set = new ExpiringHashSet <int>(TaskTimerFactory.Current);
set.EntryExpired += (s, e) => { entry = e.Entry; expired.Set(); };
set.CollectionChanged += (s, e) => changed.Set();
set.SetOrUpdate(i, ttl);
Assert.IsTrue(changed.WaitOne(5000));
Assert.IsTrue(expired.WaitOne(5000));
Assert.AreEqual(i, entry.Value);
Assert.AreEqual(ttl, entry.TTL);
Assert.IsNull(set[i]);
}
public void Can_access_hashset_during_expiration_event()
{
var expired = new ManualResetEvent(false);
var changed = new ManualResetEvent(false);
var i = 42;
var ttl = TimeSpan.FromSeconds(1);
ExpiringHashSet <int> .Entry entry = null;
var set = new ExpiringHashSet <int>(TaskTimerFactory.Current);
set.EntryExpired += (s, e) => {
entry = e.Entry;
set.Delete(e.Entry.Value);
expired.Set();
};
set.CollectionChanged += (s, e) => changed.Set();
set.SetOrUpdate(i, ttl);
Assert.IsTrue(changed.WaitOne(5000));
Assert.IsTrue(expired.WaitOne(5000));
Assert.AreEqual(i, entry.Value);
Assert.AreEqual(ttl, entry.TTL);
Assert.IsNull(set[i]);
}
public void Can_reset_expiration()
{
var expired = new ManualResetEvent(false);
var changed = new ManualResetEvent(false);
var i = 42;
var ttl = TimeSpan.FromSeconds(2);
ExpiringHashSet <int> .Entry entry = null;
var set = new ExpiringHashSet <int>(TaskTimerFactory.Current);
set.EntryExpired += (s, e) => { entry = e.Entry; expired.Set(); };
set.CollectionChanged += (s, e) => changed.Set();
set.SetOrUpdate(i, ttl);
Assert.IsTrue(changed.WaitOne(500));
changed.Reset();
Thread.Sleep(500);
Assert.IsFalse(expired.WaitOne(500));
var oldExpire = set[i].When;
set.RefreshExpiration(i);
Assert.Greater(set[i].When, oldExpire);
Assert.IsTrue(changed.WaitOne(5000));
Assert.IsTrue(expired.WaitOne(5000));
}
public void Set_will_fire_in_order_of_expirations()
{
var expired = new ManualResetEvent(false);
var entries = new List<int>();
var set = new ExpiringHashSet<int>(TaskTimerFactory.Current);
set.EntryExpired += (s, e) => { entries.Add(e.Entry.Value); expired.Set(); };
set.SetOrUpdate(3, TimeSpan.FromMilliseconds(1600));
set.SetOrUpdate(2, TimeSpan.FromMilliseconds(1500));
set.SetOrUpdate(5, TimeSpan.FromMilliseconds(3000));
set.SetOrUpdate(4, TimeSpan.FromMilliseconds(2000));
set.SetOrUpdate(1, TimeSpan.FromMilliseconds(500));
Assert.IsTrue(expired.WaitOne(1000));
Assert.AreEqual(1, entries.Count);
Assert.IsTrue(Wait.For(() => entries.Count == 5, TimeSpan.FromSeconds(5000)));
Assert.AreEqual(new[] { 1, 2, 3, 4, 5 }, entries.ToArray());
}
public void Can_retrieve_item_checking_when()
{
var set = new ExpiringHashSet<int>(TaskTimerFactory.Current);
var v = 42;
var when = GlobalClock.UtcNow.AddDays(1);
set.SetOrUpdate(v, when);
var entry = set[v];
Assert.AreEqual(v, entry.Value);
Assert.AreEqual(when, entry.When);
}
public void Can_access_hashset_during_expiration_event()
{
var expired = new ManualResetEvent(false);
var changed = new ManualResetEvent(false);
var i = 42;
var ttl = TimeSpan.FromSeconds(1);
ExpiringHashSet<int>.Entry entry = null;
var set = new ExpiringHashSet<int>(TaskTimerFactory.Current);
set.EntryExpired += (s, e) => {
entry = e.Entry;
set.Delete(e.Entry.Value);
expired.Set();
};
set.CollectionChanged += (s, e) => changed.Set();
set.SetOrUpdate(i, ttl);
Assert.IsTrue(changed.WaitOne(5000));
Assert.IsTrue(expired.WaitOne(5000));
Assert.AreEqual(i, entry.Value);
Assert.AreEqual(ttl, entry.TTL);
Assert.IsNull(set[i]);
}
public void Can_set_item_via_ttl()
{
_log.Debug("running test");
var expired = new ManualResetEvent(false);
var changed = new ManualResetEvent(false);
var i = 42;
var ttl = TimeSpan.FromSeconds(1);
ExpiringHashSet<int>.Entry entry = null;
var set = new ExpiringHashSet<int>(TaskTimerFactory.Current);
set.EntryExpired += (s, e) => { entry = e.Entry; expired.Set(); };
set.CollectionChanged += (s, e) => changed.Set();
set.SetOrUpdate(i, ttl);
Assert.IsTrue(changed.WaitOne(5000));
Assert.IsTrue(expired.WaitOne(5000));
Assert.AreEqual(i, entry.Value);
Assert.AreEqual(ttl, entry.TTL);
Assert.IsNull(set[i]);
}
public void Disposing_set_expires_all_items_before_dispose_returns_but_does_not_trigger_collection_changed()
{
var expired = false;
var changed = false;
var expiredEntries = new List<int>();
var ttl = TimeSpan.FromSeconds(1);
var set = new ExpiringHashSet<int>(TaskTimerFactory.Current);
set.EntryExpired += (s, e) => { expiredEntries.Add(e.Entry.Value); expired = true; };
set.CollectionChanged += (s, e) => { changed = true; };
set.SetOrUpdate(12, ttl);
set.SetOrUpdate(21, ttl);
Assert.IsFalse(expired, "expired was triggered");
Assert.IsTrue(changed, "changed wasn't triggered");
changed = false;
set.Dispose();
Assert.IsFalse(changed,"changed was triggered");
Assert.IsTrue(expired, "expired wasn't triggered");
Assert.AreEqual(new[] { 12, 21 },expiredEntries.OrderBy(x =>x).ToArray());
}
public void Can_reset_expiration()
{
var expired = new ManualResetEvent(false);
var changed = new ManualResetEvent(false);
var i = 42;
var ttl = TimeSpan.FromSeconds(2);
ExpiringHashSet<int>.Entry entry = null;
var set = new ExpiringHashSet<int>(TaskTimerFactory.Current);
set.EntryExpired += (s, e) => { entry = e.Entry; expired.Set(); };
set.CollectionChanged += (s, e) => changed.Set();
set.SetOrUpdate(i, ttl);
Assert.IsTrue(changed.WaitOne(500));
changed.Reset();
Thread.Sleep(500);
Assert.IsFalse(expired.WaitOne(500));
var oldExpire = set[i].When;
set.RefreshExpiration(i);
Assert.Greater(set[i].When, oldExpire);
Assert.IsTrue(changed.WaitOne(5000));
Assert.IsTrue(expired.WaitOne(5000));
}
public void Clearing_set_releases_all_items()
{
var changed = false;
var ttl = 10.Seconds();
var set = new ExpiringHashSet<string>(TaskTimerFactory.Current);
set.SetOrUpdate("a", ttl);
set.SetOrUpdate("b", ttl);
set.SetOrUpdate("v", ttl);
set.CollectionChanged += (s, e) => changed = true;
Assert.AreEqual(3, set.Count());
Assert.IsFalse(changed);
set.Clear();
Assert.AreEqual(0, set.Count());
Assert.IsTrue(changed);
}
public void Can_set_item_via_when()
{
var expired = new ManualResetEvent(false);
var changed = new ManualResetEvent(false);
var i = 42;
var when = GlobalClock.UtcNow.AddSeconds(1);
ExpiringHashSet<int>.Entry entry = null;
var set = new ExpiringHashSet<int>(TaskTimerFactory.Current);
set.EntryExpired += (s, e) => { entry = e.Entry; expired.Set(); };
set.CollectionChanged += (s, e) => changed.Set();
set.SetOrUpdate(i, when);
Assert.IsTrue(changed.WaitOne(5000));
Assert.IsTrue(expired.WaitOne(5000));
Assert.AreEqual(i, entry.Value);
Assert.AreEqual(when, entry.When);
Assert.IsNull(set[i]);
}
public void Can_set_ttl_on_existing_item()
{
var expired = new ManualResetEvent(false);
var changed = new ManualResetEvent(false);
var i = 42;
var ttl = TimeSpan.FromSeconds(10);
ExpiringHashSet<int>.Entry entry = null;
var set = new ExpiringHashSet<int>(TaskTimerFactory.Current);
set.EntryExpired += (s, e) => { entry = e.Entry; expired.Set(); };
set.CollectionChanged += (s, e) => changed.Set();
set.SetOrUpdate(i, ttl);
Assert.IsTrue(changed.WaitOne(2000));
changed.Reset();
Assert.IsFalse(expired.WaitOne(2000));
set.SetOrUpdate(i, TimeSpan.FromSeconds(1));
Assert.IsTrue(changed.WaitOne(5000));
Assert.IsTrue(expired.WaitOne(5000));
}
public void Refresh_in_autoRefresh_only_fires_every_half_second()
{
var k = 42;
var ttl = TimeSpan.FromSeconds(10);
var set = new ExpiringHashSet<int>(TaskTimerFactory.Current, true);
var expireTime = GlobalClock.UtcNow.AddSeconds(10);
set.SetOrUpdate(k, expireTime);
var when = set[k].When;
Thread.Sleep(200);
Assert.AreEqual(when, set[k].When);
Thread.Sleep(1000);
Assert.Less(when, set[k].When);
}
public void Can_retrieve_item_checking_ttl()
{
var set = new ExpiringHashSet<int>(TaskTimerFactory.Current);
var v = 42;
var ttl = TimeSpan.FromSeconds(10);
set.SetOrUpdate(v, ttl);
var entry = set[v];
Assert.AreEqual(v, entry.Value);
Assert.AreEqual(ttl, entry.TTL);
}
public void Can_iterate_over_items()
{
var set = new ExpiringHashSet<int>(TaskTimerFactory.Current);
var n = 10;
for(var i = 1; i <= n; i++) {
set.SetOrUpdate(i, TimeSpan.FromSeconds(i));
}
var items = from x in set select x;
Assert.AreEqual(n, items.Count());
for(var i = 1; i <= n; i++) {
var j = i;
Assert.IsTrue((from x in set where x.Value == j && x.TTL == TimeSpan.FromSeconds(j) select x).Any());
}
}
public void New_item_with_more_recent_expiration_will_fire_at_expected_time()
{
var expired = new ManualResetEvent(false);
var entries = new List<int>();
var set = new ExpiringHashSet<int>(TaskTimerFactory.Current);
set.EntryExpired += (s, e) => { entries.Add(e.Entry.Value); expired.Set(); };
var n = 1000;
for(var i = 0; i < n; i++) {
set.SetOrUpdate(i, TimeSpan.FromMinutes(10));
}
set.SetOrUpdate(100000, TimeSpan.FromSeconds(2));
Assert.IsTrue(expired.WaitOne(4000));
Assert.AreEqual(1, entries.Count);
Assert.AreEqual(n, set.Count());
}
