public void StartEventProcessing(NetheriteOrchestrationServiceSettings settings, CloudBlockBlob partitionScript)
{
if (!partitionScript.Exists())
{
this.logger.LogInformation("ScriptedEventProcessorHost workerId={workerId} is waiting for script", this.workerId);
while (!partitionScript.Exists())
{
Thread.Sleep(TimeSpan.FromSeconds(1));
}
}
// we use the UTC modification timestamp on the script as the scenario start time
DateTime scenarioStartTimeUtc = partitionScript.Properties.LastModified.Value.UtcDateTime;
// the number of partitions matters only if the script contains wildcards
this.numberOfPartitions = this.parameters.StartPositions.Length;
for (var partitionIndex = 0; partitionIndex < this.numberOfPartitions; partitionIndex++)
{
this.partitionInstances.Add(null);
}
List <PartitionScript.ProcessorHostEvent> timesteps = new List <PartitionScript.ProcessorHostEvent>();;
try
{
using (var memoryStream = new System.IO.MemoryStream())
{
partitionScript.DownloadRangeToStream(memoryStream, null, null);
memoryStream.Seek(0, System.IO.SeekOrigin.Begin);
timesteps.AddRange(PartitionScript.ParseEvents(scenarioStartTimeUtc, settings.WorkerId, this.numberOfPartitions, memoryStream));
}
this.logger.LogInformation("ScriptedEventProcessorHost workerId={workerId} started.", this.workerId);
}
catch (Exception e)
{
this.logger.LogError($"ScriptedEventProcessorHost workerId={this.workerId} failed to parse partitionscript: {e}");
}
int nextTime = 0;
List <PartitionScript.ProcessorHostEvent> nextGroup = new List <PartitionScript.ProcessorHostEvent>();
foreach (var timestep in timesteps)
{
if (nextTime == timestep.TimeSeconds)
{
nextGroup.Add(timestep);
}
else
{
this.Process(nextGroup);
nextGroup.Clear();
nextGroup.Add(timestep);
nextTime = timestep.TimeSeconds;
}
}
this.Process(nextGroup);
}
private static void ParallelDownloadBlob(Stream outPutStream, CloudBlockBlob blob)
{
// Thanks to Bruce Chen for this method (https://stackoverflow.com/a/41819508/)
blob.FetchAttributes();
int bufferLength = 1 * 1024 * 1024; // 1 MB chunk
long blobRemainingLength = blob.Properties.Length;
var queues = new Queue <KeyValuePair <long, long> >();
long offset = 0;
while (blobRemainingLength > 0)
{
var chunkLength = Math.Min(bufferLength, blobRemainingLength);
queues.Enqueue(new KeyValuePair <long, long>(offset, chunkLength));
offset += chunkLength;
blobRemainingLength -= chunkLength;
}
// We only want to show the value of the block with the highest value during downloads
// because it doesn't make sense to see completion percentage value out of sequence.
double maxQueueKey = 0;
using (var progress = new ProgressBar())
{
Parallel.ForEach(queues,
new ParallelOptions
{
//Gets or sets the maximum number of concurrent tasks
MaxDegreeOfParallelism = 10
}, queue =>
{
using (var ms = new MemoryStream())
{
if (queue.Key > maxQueueKey)
{
maxQueueKey = queue.Key;
}
progress.Report(maxQueueKey / blob.Properties.Length);
Thread.Sleep(1);
blob.DownloadRangeToStream(ms, queue.Key, queue.Value);
lock (outPutStream)
{
outPutStream.Position = queue.Key;
var bytes = ms.ToArray();
outPutStream.Write(bytes, 0, bytes.Length);
}
}
});
}
}
public Stream MemoryStreamAzure(string empresa, string azureblob, Guid id, string extension)
{
string myBlobContainerReferenceKey = azureblob + empresa;
Stream outPutStream = new MemoryStream();
var StorageAccount = CloudStorageAccount.Parse(strCloudStorageAccount);
CloudBlobClient BlobClient = StorageAccount.CreateCloudBlobClient();
CloudBlobContainer Content = BlobClient.GetContainerReference(myBlobContainerReferenceKey);
CloudBlockBlob blockBlob = Content.GetBlockBlobReference(string.Format("{0}{1}", id, extension));
blockBlob.FetchAttributes();
int bufferLength = 1 * 1024 * 1024;//1 MB chunk
long blobRemainingLength = blockBlob.Properties.Length;
Queue <KeyValuePair <long, long> > queues = new Queue <KeyValuePair <long, long> >();
long offset = 0;
while (blobRemainingLength > 0)
{
long chunkLength = (long)Math.Min(bufferLength, blobRemainingLength);
queues.Enqueue(new KeyValuePair <long, long>(offset, chunkLength));
offset += chunkLength;
blobRemainingLength -= chunkLength;
}
Parallel.ForEach(queues,
new ParallelOptions()
{
//Gets or sets the maximum number of concurrent tasks
MaxDegreeOfParallelism = 10
}, (queue) =>
{
using (var ms = new MemoryStream())
{
blockBlob.DownloadRangeToStream(ms, queue.Key, queue.Value);
lock (outPutStream)
{
outPutStream.Position = queue.Key;
var bytes = ms.ToArray();
outPutStream.Write(bytes, 0, bytes.Length);
}
}
});
return(outPutStream);
}
private Stream ParallelDownloadBlob(Stream outPutStream, CloudBlockBlob blob)
{
blob.FetchAttributes();
int bufferLength = 1 * 1024 * 1024;//1 MB chunk
long blobRemainingLength = blob.Properties.Length;
Queue <KeyValuePair <long, long> > queues = new Queue <KeyValuePair <long, long> >();
long offset = 0;
while (blobRemainingLength > 0)
{
long chunkLength = (long)Math.Min(bufferLength, blobRemainingLength);
queues.Enqueue(new KeyValuePair <long, long>(offset, chunkLength));
offset += chunkLength;
blobRemainingLength -= chunkLength;
}
Parallel.ForEach(queues,
new ParallelOptions()
{
//Gets or sets the maximum number of concurrent tasks
MaxDegreeOfParallelism = 10
}, (queue) =>
{
using (var ms = new MemoryStream())
{
blob.DownloadRangeToStream(ms, queue.Key, queue.Value);
lock (outPutStream)
{
outPutStream.Position = queue.Key;
var bytes = ms.ToArray();
outPutStream.Write(bytes, 0, bytes.Length);
}
}
});
return(outPutStream);
}
public async Task <string> GetFileFromBlobStorage(BlobRequest blobRequest)
{
await Task.Yield();
if (blobRequest.StorageUri == null || string.IsNullOrEmpty(blobRequest.StorageUri))
{
throw new ArgumentException("Se debe enviar el StorageUri");
}
if (blobRequest.FileName == null || string.IsNullOrEmpty(blobRequest.FileName))
{
throw new ArgumentException("Se debe enviar el FileName");
}
if (blobRequest.Folder == null || string.IsNullOrEmpty(blobRequest.Folder))
{
throw new ArgumentException("Se debe enviar el Folder");
}
var fileFullPath = string.Empty;
Uri uri = new Uri(blobRequest.StorageUri);
CloudBlobClient blobclient = new CloudBlobClient(uri);
CloudBlobContainer blobcontainer = blobclient.GetContainerReference(blobRequest.Folder);
CloudBlockBlob blob = blobcontainer.GetBlockBlobReference(blobRequest.FileName);
blob.FetchAttributes();
var blobSize = blob.Properties.Length;
long blockSize = (10 * 1024 * 1024);
blockSize = Math.Min(blobSize, blockSize);
if (!Directory.Exists("Files"))
{
Directory.CreateDirectory("Files");
}
var filePath = $"Files\\{blobRequest.FileName}";
if (File.Exists(filePath))
{
File.Delete(filePath);
}
using (FileStream fs = new FileStream(filePath, FileMode.Create))
{
fs.SetLength(blobSize);
fileFullPath = fs.Name;
}
var blobRequestOptions = new BlobRequestOptions
{
RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(5), 3),
MaximumExecutionTime = TimeSpan.FromMinutes(60),
ServerTimeout = TimeSpan.FromMinutes(60)
};
long currentPointer = 0;
long bytesRemaining = blobSize;
do
{
try
{
var bytesToFetch = Math.Min(blockSize, bytesRemaining);
using MemoryStream ms = new MemoryStream();
blob.DownloadRangeToStream(ms, currentPointer, bytesToFetch, null, blobRequestOptions);
ms.Position = 0;
var contents = ms.ToArray();
using (var fs = new FileStream(filePath, FileMode.Open))
{
fs.Position = currentPointer;
fs.Write(contents, 0, contents.Length);
}
currentPointer += contents.Length;
bytesRemaining -= contents.Length;
}
catch (Exception ex)
{
throw new Exception($"File has error in {currentPointer} a las {DateTime.Now}", ex);
}
}while (bytesRemaining > 0);
return(fileFullPath);
}
