public void TestWorkflowProgressLoopingSimple()
{
//IWorkContext context = new WorkContext(null, TestEnvironment.Client);
// defines and executes a cyclic (looping) workflow
var w = new LoopingStep();
w.Initialise(_workContext);
var inp = new LoopInput {
begin = 0, end = 2000
};
// loops until condition is true
// start and monitor
_workContext.Logger.LogDebug("Starting...");
_workContext.Logger.LogDebug("[{0}] {1}", "I", w.Progress);
//LoopResult result = w.Execute(inp, TimeSpan.FromSeconds(10));
AsyncResult <WorkflowOutput <LoopResult> > ar = w.BeginExecute(inp, null);
int loop = 0;
while (!ar.IsCompleted)
{
Thread.Sleep(1000);
_workContext.Logger.LogDebug("[{0}] {1}", loop, w.Progress);
loop++;
}
LoopResult result = ar.EndInvoke().Result;
_workContext.Logger.LogDebug("[{0}] {1}", "Z", w.Progress);
Assert.AreEqual(2001000, result.total);
_workContext.Logger.LogDebug("Complete.");
}
public void TestWorkstepLoopingDelegate()
{
// defines and executes a cyclic (looping) workflow
var w = new WorkstepBaseLoop <LoopInput, LoopData, LoopResult>(
// initialiser
delegate(LoopInput input)
{
LoopData loopData;
loopData.index = input.begin;
loopData.end = input.end;
loopData.total = 0;
return(loopData);
},
// condition
loopData => (loopData.index <= loopData.end),
// loop body
delegate(LoopData loopData) { loopData.total += loopData.index; return(loopData); },
// loop iterator
delegate(ref LoopData loopData) { loopData.index++; },
// finaliser
delegate(LoopData loopData) { LoopResult temp; temp.total = loopData.total; return(temp); }
);
w.Initialise(_workContext);
var inp = new LoopInput {
begin = 0, end = 2000
};
// loops until condition is true
LoopResult outp = w.Execute(inp).Result;
Assert.AreEqual(2001000, outp.total);
}
public bool Synchronize()
{
var uploadStatus = new ProgressStatus(alreadyUploadedData, alreadyUploadedData + dataToUpload, new ComputeStats());
using (new ServicePointHandler(blob.Uri, this.maxParallelism))
using (ProgressTracker progressTracker = new ProgressTracker(uploadStatus))
{
Task <LoopResult> task = Task <LoopResult> .Factory.StartNew(() =>
{
return(Threading.Parallel.ForEach(dataWithRanges,
() => new CloudPageBlob(blob.Uri, blob.ServiceClient.Credentials),
(dwr, b) =>
{
using (dwr)
{
var md5HashOfDataChunk = GetBase64EncodedMd5Hash(dwr.Data, (int)dwr.Range.Length);
using (var stream = new MemoryStream(dwr.Data, 0, (int)dwr.Range.Length))
{
b.Properties.ContentMD5 = md5HashOfDataChunk;
b.WritePagesAsync(stream, dwr.Range.StartIndex, contentMD5: null)
.ConfigureAwait(false).GetAwaiter().GetResult();
}
}
uploadStatus.AddToProcessedBytes((int)dwr.Range.Length);
}, this.maxParallelism));
});
while (!task.Wait(TimeSpan.FromSeconds(1)))
{
progressTracker.Update();
}
LoopResult loopResult = task.Result;
if (loopResult.IsExceptional)
{
if (loopResult.Exceptions.Any())
{
Program.SyncOutput.ErrorUploadFailedWithExceptions(loopResult.Exceptions);
throw new AggregateException(loopResult.Exceptions);
}
}
else
{
using (var bdms = new BlobMetaDataScope(new CloudPageBlob(blob.Uri, blob.ServiceClient.Credentials)))
{
bdms.Current.SetBlobMd5Hash(md5Hash);
bdms.Current.CleanUpUploadMetaData();
bdms.Complete();
}
}
}
return(true);
}
public void TestWorkstepLoopingOverride()
{
// defines and executes a cyclic (looping) workflow
var w = new LoopingStep();
w.Initialise(_workContext);
var inp = new LoopInput {
begin = 0, end = 2000
};
// loops until condition is true
LoopResult result = w.Execute(inp).Result;
Assert.AreEqual(2001000, result.total);
}
public bool Synchronize()
{
var uploadStatus = new ProgressStatus(alreadyUploadedData, alreadyUploadedData + dataToUpload, new ComputeStats());
using (new ServicePointHandler(pageBlob.Uri, this.maxParallelism))
using (ProgressTracker progressTracker = new ProgressTracker(uploadStatus))
{
Task <LoopResult> task = Task <LoopResult> .Factory.StartNew(() =>
{
return(Threading.Parallel.ForEach(dataWithRanges,
() => new PSPageBlobClient(pageBlob.Uri),
(dwr, b) =>
{
using (dwr)
{
var md5HashOfDataChunk = GetBase64EncodedMd5Hash(dwr.Data, (int)dwr.Range.Length);
using (var stream = new MemoryStream(dwr.Data, 0, (int)dwr.Range.Length))
{
b.UploadPages(stream, dwr.Range.StartIndex);
}
}
uploadStatus.AddToProcessedBytes((int)dwr.Range.Length);
}, this.maxParallelism));
});
while (!task.Wait(TimeSpan.FromSeconds(1)))
{
progressTracker.Update();
}
LoopResult loopResult = task.Result;
if (loopResult.IsExceptional)
{
if (loopResult.Exceptions.Any())
{
Program.SyncOutput.ErrorUploadFailedWithExceptions(loopResult.Exceptions);
throw new AggregateException(loopResult.Exceptions);
}
}
}
return(true);
}
private void StartInternal()
{
if (_process.IsProcessRunning)
{
throw new InvalidOperationException();
}
//
// We try to start the goddamn host process in a loop until:
// - The start & connection succeeds
// - The IFailureHandler calls quit
// - The number of restarts is 20
//
// If starting failed then we throw the last exception
//
List <Exception> exceptions = null;
const int maxRestarts = 20;
int pid = -1;
ConnectionId connectionId = ConnectionId.None;
for (int currentRestart = 0; currentRestart < maxRestarts; ++currentRestart)
{
LoopResult instruction = StartOnce(currentRestart, out pid, out connectionId, ref exceptions);
if (instruction == LoopResult.Stop)
{
break;
}
}
if (exceptions != null)
{
//
// POINT OF FAILURE
//
_currentPid = -1;
_currentConnection = ConnectionId.None;
throw new AggregateException("Unable to start & establish a connection with the host process",
exceptions);
}
//
// POINT OF NO FAILURE BELOW
//
lock (_syncRoot)
{
_currentPid = pid;
_currentConnection = connectionId;
}
try
{
OnHostStarted?.Invoke();
}
catch (Exception e)
{
Log.WarnFormat("The OnHostStarted event threw an exception, please don't do that: {0}", e);
}
}
public void Download()
{
if (parameters.OverWrite)
{
DeleteTempVhdIfExist(parameters.LocalFilePath);
}
else
{
if (File.Exists(parameters.LocalFilePath))
{
var message = String.Format("File already exists, you can use Overwrite option to delete it:'{0}'", parameters.LocalFilePath);
throw new ArgumentException(message);
}
}
var blobHandle = new BlobHandle(parameters.BlobUri, this.parameters.StorageAccountKey);
if (parameters.ValidateFreeDiskSpace)
{
TryValidateFreeDiskSpace(parameters.LocalFilePath, blobHandle.Length);
}
const int megaByte = 1024 * 1024;
var ranges = blobHandle.GetUploadableRanges();
var bufferManager = BufferManager.CreateBufferManager(Int32.MaxValue, 20 * megaByte);
var downloadStatus = new ProgressStatus(0, ranges.Sum(r => r.Length), new ComputeStats());
Trace.WriteLine(String.Format("Total Data:{0}", ranges.Sum(r => r.Length)));
Program.SyncOutput.WriteVerboseWithTimestamp("Downloading the blob: {0}", parameters.BlobUri.BlobName);
var fileStreamLock = new object();
using (new ServicePointHandler(parameters.BlobUri.Uri, parameters.ConnectionLimit))
{
using (ProgressTracker progressTracker = new ProgressTracker(downloadStatus, parameters.ProgressDownloadStatus, parameters.ProgressDownloadComplete))
{
using (var fileStream = new FileStream(parameters.LocalFilePath, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Write, 8 * megaByte, FileOptions.WriteThrough))
{
fileStream.SetLength(0);
fileStream.SetLength(blobHandle.Length);
Task <LoopResult> task = Task <LoopResult> .Factory.StartNew(() =>
{
return(Threading.Parallel.ForEach <IndexRange, Stream>(ranges,
blobHandle.OpenStream,
(r, b) =>
{
b.Seek(r.StartIndex, SeekOrigin.Begin);
byte[] buffer = this.EnsureReadAsSize(b, (int)r.Length, bufferManager);
lock (fileStreamLock)
{
Trace.WriteLine(String.Format("Range:{0}", r));
fileStream.Seek(r.StartIndex, SeekOrigin.Begin);
fileStream.Write(buffer, 0, (int)r.Length);
fileStream.Flush();
}
downloadStatus.AddToProcessedBytes((int)r.Length);
},
pbwlf =>
{
pbwlf.Dispose();
},
parameters.ConnectionLimit));
});
while (!task.Wait(TimeSpan.FromSeconds(1)))
{
progressTracker.Update();
}
LoopResult lr = task.Result;
if (lr.IsExceptional)
{
throw new AggregateException(lr.Exceptions);
}
}
}
}
Program.SyncOutput.WriteVerboseWithTimestamp("Blob downloaded successfullty: {0}", parameters.BlobUri.BlobName);
}
