private IAsyncEnumerable <T> ExecuteQuery(string filter, int?take)
{
return(AsyncEnumerableEx.Create <T>(async y =>
{
var query = new CT.TableQuery <FatEntity>();
if (filter != null)
{
query = query.Where(filter);
}
if (take.HasValue)
{
query = query.Take(take);
}
CT.TableQuerySegment <FatEntity> segment = null;
do
{
segment = await _table.ExecuteWrap(t => t.ExecuteQuerySegmentedAsync(query, segment == null ? null : segment.ContinuationToken, null, null, y.CancellationToken)).ConfigureAwait(false);
foreach (var entity in segment)
{
await y.YieldReturn(ConvertFatEntity(entity)).ConfigureAwait(false);
y.ThrowIfCancellationRequested();
}
}while (segment.ContinuationToken != null && !y.CancellationToken.IsCancellationRequested);
}));
}
private IAsyncEnumerable <ICloudBlob> ListBlobs(CloudBlobContainer container, string prefix, BlobListingDetails options)
{
// Clean up the prefix if required
prefix = prefix == null ? null : SafePath.MakeSafeFilePath(prefix);
return(AsyncEnumerableEx.Create <ICloudBlob>(async(y) =>
{
BlobContinuationToken token = new BlobContinuationToken();
try
{
do
{
var segment = await container.ListBlobsSegmentedAsync(prefix, true, options, null, token, null, null, y.CancellationToken).ConfigureAwait(false);
LeoTrace.WriteLine("Listed blob segment for prefix: " + prefix);
foreach (var blob in segment.Results.OfType <ICloudBlob>())
{
await y.YieldReturn(blob).ConfigureAwait(false);
}
token = segment.ContinuationToken;
}while (token != null && !y.CancellationToken.IsCancellationRequested);
}
catch (StorageException e)
{
if (e.RequestInformation.HttpStatusCode != 404)
{
throw e.Wrap(container.Name + "_" + (prefix ?? string.Empty) + "*");
}
}
}));
}
private IAsyncEnumerable <S3ObjectVersion> ListObjects(string prefix)
{
return(AsyncEnumerableEx.Create <S3ObjectVersion>(async y =>
{
var request = new ListVersionsRequest
{
BucketName = _bucket,
Prefix = prefix,
KeyMarker = null,
VersionIdMarker = null
};
while (request != null && !y.CancellationToken.IsCancellationRequested)
{
var resp = await _client.ListVersionsAsync(request, y.CancellationToken).ConfigureAwait(false);
foreach (var v in resp.Versions)
{
await y.YieldReturn(v).ConfigureAwait(false);
}
if (resp.IsTruncated)
{
request.KeyMarker = resp.NextKeyMarker;
request.VersionIdMarker = resp.NextVersionIdMarker;
}
else
{
request = null;
}
}
}));
}
public IAsyncEnumerable <bool> RunEvery(StoreLocation location, TimeSpan interval, Action <Exception> unhandledExceptions = null)
{
return(AsyncEnumerableEx.Create <bool>(async(y) =>
{
var blob = GetBlockBlob(location);
var minimum = TimeSpan.FromSeconds(5); // so we're not polling the leased blob too fast
while (!y.CancellationToken.IsCancellationRequested)
{
var timeLeft = TimeSpan.FromSeconds(0);
// Don't allow you to throw to get out of the loop...
try
{
var lastPerformed = DateTimeOffset.MinValue;
var lease = await LockInternal(blob).ConfigureAwait(false);
if (lease != null)
{
using (var arl = lease.Item1)
{
await blob.ExecuteWrap(b => b.FetchAttributesAsync(null, null, null, y.CancellationToken)).ConfigureAwait(false);
if (blob.Metadata.ContainsKey("lastPerformed"))
{
DateTimeOffset.TryParseExact(blob.Metadata["lastPerformed"], "R", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal, out lastPerformed);
}
if (DateTimeOffset.UtcNow >= lastPerformed + interval)
{
await y.YieldReturn(true).ConfigureAwait(false);
lastPerformed = DateTimeOffset.UtcNow;
blob.Metadata["lastPerformed"] = lastPerformed.ToString("R", CultureInfo.InvariantCulture);
await blob.ExecuteWrap(b => b.SetMetadataAsync(AccessCondition.GenerateLeaseCondition(lease.Item2), null, null, y.CancellationToken)).ConfigureAwait(false);
}
}
}
timeLeft = (lastPerformed + interval) - DateTimeOffset.UtcNow;
}
catch (TaskCanceledException) { throw; }
catch (Exception e)
{
unhandledExceptions?.Invoke(e);
LeoTrace.WriteLine("Error on lock loop: " + e.Message);
}
// Do this outside the exception to prevent it going out of control
await Task.Delay(timeLeft > minimum ? timeLeft : minimum, y.CancellationToken).ConfigureAwait(false);
}
}));
}
private IAsyncEnumerable <T> ExecuteWithEncryptor <T>(Lazy <Task <IEncryptor> > encryptor, Func <IEncryptor, IAsyncEnumerable <T> > factory)
{
encryptor = encryptor ?? new Lazy <Task <IEncryptor> >(() => Task.FromResult((IEncryptor)null));
return(AsyncEnumerableEx.Create((Func <AsyncYielder <T>, Task>)(async y =>
{
var enc = await encryptor.Value.ConfigureAwait(false);
var enumerator = factory(enc).GetEnumerator();
while (!y.CancellationToken.IsCancellationRequested)
{
if (!await enumerator.MoveNext(y.CancellationToken).ConfigureAwait(false))
{
break;
}
await y.YieldReturn(enumerator.Current).ConfigureAwait(false);
}
})));
}
