private Yield Upload_Helper(DreamContext context, DreamMessage request, Result <DreamMessage> response)
{
using (var stream = request.ToStream()) {
var total = 0;
var buffer = new byte[1024 * 1024];
while (total < request.ContentLength)
{
Result <int> read;
yield return(read = stream.Read(buffer, 0, buffer.Length, new Result <int>()));
//int read = stream.Read(buffer, 0, buffer.Length);
if (read.Value == 0)
{
break;
}
total += read.Value;
//fake some latency
yield return(AsyncUtil.Sleep(TimeSpan.FromMilliseconds(1), new Result(TimeSpan.MaxValue)));
}
_log.DebugFormat("read {0}/{1} bytes", total, request.ContentLength);
if (total != request.ContentLength)
{
throw new DreamBadRequestException(string.Format("was supposed to read {0} bytes, only read {1}", request.ContentLength, total));
}
}
response.Return(DreamMessage.Ok());
}
/// <summary>
/// Shutdown the ElasticThreadPool instance. This method blocks until all pending items have finished processing.
/// </summary>
public void Dispose()
{
if (!_disposed)
{
_disposed = true;
_log.DebugFormat("Dispose @{0}", this);
// TODO (steveb): make dispose more reliable
// 1) we can't wait indefinitively!
// 2) we should progressively sleep longer and longer to avoid unnecessary overhead
// 3) this pattern feels useful enough to be captured into a helper method
// wait until all threads have been decommissioned
while (ThreadCount > 0)
{
AsyncUtil.Sleep(100.Milliseconds());
}
// discard all reserved threads
KeyValuePair <DispatchThread, Result <DispatchWorkItem> > reserved;
while (_reservedThreads.TryPop(out reserved))
{
DispatchThreadScheduler.ReleaseThread(reserved.Key, reserved.Value);
}
DispatchThreadScheduler.UnregisterHost(this);
}
}
internal void LongPollSqs(SqsPollingClientSettings settings)
{
var failCounter = 0;
while (true)
{
IEnumerable <SqsMessage> messages;
try {
messages = _container._client.ReceiveMessages(settings.QueueName, settings.LongPollInterval, settings.MaxNumberOfMessages);
failCounter = 0;
} catch (Exception e) {
LogError(e, string.Format("ReceiveMessages (fail count: {0:#,##0})", ++failCounter));
AsyncUtil.Sleep(settings.WaitTimeOnError);
continue;
}
if (_isDisposed)
{
break;
}
if (messages.None())
{
continue;
}
try {
settings.Callback(messages);
} catch (Exception e) {
LogError(e, "callback failed");
}
}
}
private static Yield Ret_Yield_Throw(int p, Result <int> result)
{
yield return(AsyncUtil.Sleep(TimeSpan.FromMilliseconds(10), new Result(TimeSpan.MaxValue)));
result.Return(p);
throw new IntentionalException();
}
private void Receive()
{
if (_stopped)
{
return;
}
var t = Stopwatch.StartNew();
_client.ReceiveMax(_queue, 10.Minutes(), new Result <IEnumerable <AwsSqsMessage> >()).WhenDone(r => {
if (r.HasException)
{
Exception = r.Exception;
return;
}
if (_stopped)
{
return;
}
var received = new List <AwsSqsMessage>(r.Value);
if (!received.Any())
{
_log.DebugFormat("{0}: no messages in queue, sleeping before retry", Id);
AsyncUtil.Sleep(1.Seconds(), new Result(TimeSpan.MaxValue)).WhenDone(r2 => Receive());
return;
}
_log.DebugFormat("{0}: received {1} messages", Id, received.Count);
Delete(received, 0, t);
});
}
/// <summary>
/// Repeatedly test a <b>condition</b> until the call times out or the condition indicates success.
/// </summary>
/// <param name="condition">Func to check whether wait condition has been met.</param>
/// <param name="timeout">The maximum time to keep testing the condition.</param>
/// <returns></returns>
public static bool For(Func <bool> condition, TimeSpan timeout)
{
var timer = Stopwatch.StartNew();
while (timeout > timer.Elapsed)
{
if (condition())
{
return(true);
}
AsyncUtil.Sleep(50.Milliseconds());
}
return(false);
}
/// <summary>
/// Try to open a file for exclusive read/write access
/// </summary>
/// <param name="filename">Path to file</param>
/// <returns>A <see cref="Stream"/> for the opened file, or <see langword="null"/> on failure to open the file.</returns>
public static Stream FileOpenExclusive(string filename)
{
for (int attempts = 0; attempts < 10; ++attempts)
{
try {
return(File.Open(filename, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None));
} catch (IOException e) {
_log.TraceExceptionMethodCall(e, "FileOpenExclusive", filename, attempts);
} catch (UnauthorizedAccessException e) {
_log.TraceExceptionMethodCall(e, "FileOpenExclusive", filename, attempts);
}
AsyncUtil.Sleep(((attempts + 1) * Randomizer.Next(100)).Milliseconds());
}
return(null);
}
/// <summary>
/// Repeatedly test a <b>condition</b> until the call times out or the condition indicates success.
/// </summary>
/// <typeparam name="T">The type of the result <see cref="WaitResult{T}"/> returned on success.</typeparam>
/// <param name="condition">Func to check whether wait condition has been met.</param>
/// <param name="timeout">The maximum time to keep testing the condition.</param>
/// <returns>The value produced by successful execution of the condition.</returns>
public static T For <T>(Func <WaitResult <T> > condition, TimeSpan timeout)
{
var timer = Stopwatch.StartNew();
while (timeout > timer.Elapsed)
{
var waitresult = condition();
if (waitresult.Success)
{
return(waitresult.Value);
}
AsyncUtil.Sleep(50.Milliseconds());
}
throw new TimeoutException("gave up waiting for value");
}
private Yield BubbleCoroutine(int depth, Result result)
{
_log.DebugFormat("{0} levels remaining", depth);
yield return(AsyncUtil.Sleep(TimeSpan.FromMilliseconds(10), new Result(TimeSpan.MaxValue)));
if (depth > 0)
{
_log.Debug("invoking BubbleCoroutine again");
yield return(Coroutine.Invoke(BubbleCoroutine, depth - 1, new Result()));
result.Return();
yield break;
}
_log.Debug("throwing");
throw new BubbledException();
}
/// <summary>
/// Receive zero or more messages from name queue.
/// </summary>
/// <param name="queueName">Queue name.</param>
/// <param name="waitTimeSeconds">Max amount of time to wait until this method returns.</param>
/// <param name="maxNumberOfMessages">Max number of messages to request.</param>
/// <returns>Enumeration of received messages.</returns>
public IEnumerable <SqsMessage> ReceiveMessages(SqsQueueName queueName, TimeSpan waitTimeSeconds, uint maxNumberOfMessages)
{
var start = GlobalClock.UtcNow;
// keep checking for messages until the wait-timeout kicks in
while (true)
{
var msgQueue = GetQueue(queueName);
AssertQueueIsNotNull(queueName, msgQueue);
QueueEntry[] entries;
var now = GlobalClock.UtcNow;
lock (msgQueue) {
var visibilityTimeout = 30.Seconds();
entries = msgQueue
.Where(x => x.VisibleTime <= now)
.OrderBy(x => _random.Next())
.Take(10)
.ToArray();
foreach (var entry in entries)
{
entry.VisibleTime = now + visibilityTimeout;
}
}
if (entries.Any())
{
return(entries.Select(e => new SqsMessage(e.Message.MessageId, e.Message.MessageReceipt, e.Message.Body)).ToArray());
}
if ((now - start) > waitTimeSeconds)
{
return(Enumerable.Empty <SqsMessage>());
}
// no message found; go to sleep for 100ms and try again
AsyncUtil.Sleep(100.Milliseconds());
}
}
private static Yield Yield_Ret(int p, Result <int> result)
{
yield return(AsyncUtil.Sleep(TimeSpan.FromMilliseconds(10), new Result(TimeSpan.MaxValue)));
result.Return(p);
}
private static Yield Yield_Throw(int p, Result <int> result)
{
yield return(AsyncUtil.Sleep(TimeSpan.FromMilliseconds(10)));
throw new IntentionalException();
}
