/// <summary>
/// Process events
/// </summary>
/// <param name="context"></param>
/// <param name="messages"></param>
/// <returns></returns>
public Task ProcessEventsAsync(PartitionContext context, IEnumerable <EventData> messages)
{
long lastSeq = 0;
foreach (var eventData in messages)
{
var message = BinarySerializer.DeserializeStreamEventLong(eventData.Body.ToArray());
lastSeq = eventData.SystemProperties.SequenceNumber;
this.input.OnNext(message);
}
if (this.checkpointStopWatch.Elapsed > TimeSpan.FromSeconds(10))
{
Console.WriteLine("Taking checkpoint");
var storageAccount = CloudStorageAccount.Parse(StorageConnectionString);
var blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference("checkpoints");
var blockBlob = container.GetBlockBlobReference(context.PartitionId + "-" + lastSeq);
CloudBlobStream blobStream = blockBlob.OpenWriteAsync().GetAwaiter().GetResult();
this.queryProcess.Checkpoint(blobStream);
blobStream.Flush();
blobStream.Close();
return(context
.CheckpointAsync()
.ContinueWith(t => DeleteOlderCheckpoints(context.PartitionId + "-" + lastSeq)));
}
return(Task.CompletedTask);
}
public override void Flush()
{
try {
_stream.Flush();
}
catch {
// ignored
}
}
/// <summary>
/// Process events
/// </summary>
/// <param name="context"></param>
/// <param name="messages"></param>
/// <returns></returns>
public Task ProcessEventsAsync(PartitionContext context, IEnumerable <EventData> messages)
{
EventProcessor.__count += messages.Count();
if (DateTime.UtcNow > LastTime.AddSeconds(2))
{
LastTime = DateTime.UtcNow;
Console.Title = $"total: {EventProcessor.__count} offset: {context.ToString()} count: {messages.Count()}";
}
#if true
context.CheckpointAsync();
#else
long lastSeq = 0;
foreach (var eventData in messages)
{
var message = BinarySerializer.DeserializeStreamEventSampleEvent <Measure>(eventData.Body.ToArray());
lastSeq = eventData.SystemProperties.SequenceNumber;
Task.Delay(5).Wait();
this.input.OnNext(message);
//else if (message.Payload.Data is MeasureT3)
//{
// var serializer = Newtonsoft.Json.JsonSerializer.Create();
// var value = JsonConvert.DeserializeObject<StreamEvent<Measure<MeasureT3>>>(JsonConvert.SerializeObject(message));
// this.input.OnNext(value);
//}
//else if (message.Payload.Data is MeasureT4)
// this.input.OnNext((StreamEvent<Measure>)(object)(StreamEvent<Measure<MeasureT4>>)(object)message);
}
if (this.checkpointStopWatch.Elapsed > TimeSpan.FromSeconds(10))
{
Console.WriteLine("Taking checkpoint");
var storageAccount = CloudStorageAccount.Parse(StorageConnectionString);
var blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference("checkpoints");
var blockBlob = container.GetBlockBlobReference(context.PartitionId + "-" + lastSeq);
CloudBlobStream blobStream = blockBlob.OpenWriteAsync().GetAwaiter().GetResult();
this.queryProcess.Checkpoint(blobStream);
blobStream.Flush();
blobStream.Close();
return(context
.CheckpointAsync()
.ContinueWith(t => DeleteOlderCheckpoints(context.PartitionId + "-" + lastSeq)));
}
#endif
return(Task.CompletedTask);
}
/// <summary>
/// Returns true if the file was written.
/// Returns false if the in-process lock failed. Throws an exception if any kind of file or processing exception occurs.
/// </summary>
/// <param name="result"></param>
/// <param name="path"></param>
/// <param name="writeCallback"></param>
/// <param name="timeoutMs"></param>
/// <param name="recheckFS"></param>
/// <returns></returns>
private bool TryWriteFile(CacheResult result, string path, ResizeImageDelegate writeCallback, int timeoutMs, bool recheckFS)
{
bool miss = true;
if (recheckFS)
{
miss = !Index.PathExistInIndex(path);
if (!miss && !Locks.MayBeLocked(path.ToUpperInvariant()))
{
return(true);
}
}
//Lock execution using relativePath as the sync basis. Ignore casing differences. This locking is process-local, but we also have code to handle file locking.
return(Locks.TryExecute(path.ToUpperInvariant(), timeoutMs,
delegate() {
//On the second check, use cached data for speed. The cached data should be updated if another thread updated a file (but not if another process did).
if (!Index.PathExistInIndex(path))
{
//Open stream
//Catch IOException, and if it is a file lock,
// - (and hashmodified is true), then it's another process writing to the file, and we can serve the file afterwards
// - (and hashmodified is false), then it could either be an IIS read lock or another process writing to the file. Correct behavior is to kill the request here, as we can't guarantee accurate image data.
// I.e, hashmodified=true is the only supported setting for multi-process environments.
//TODO: Catch UnathorizedAccessException and log issue about file permissions.
//... If we can wait for a read handle for a specified timeout.
try
{
var blobRef = container.GetBlockBlobReference(path);
// Only write to the blob if it does not exist
if (!blobRef.Exists())
{
CloudBlobStream cloudBlobStream = blobRef.OpenWrite();
bool finished = false;
try
{
using (cloudBlobStream)
{
//Run callback to write the cached data
writeCallback(cloudBlobStream); //Can throw any number of exceptions.
cloudBlobStream.Flush();
finished = true;
}
}
finally
{
//Don't leave half-written files around.
if (!finished)
{
try
{
blobRef.Delete();
}
catch { }
}
}
DateTime createdUtc = DateTime.UtcNow;
}
//Update index
Index.AddCachedFileToIndex(path);
//This was a cache miss
if (result != null)
{
result.Result = CacheQueryResult.Miss;
}
}
catch (IOException ex)
{
//Somehow in between verifying the file didn't exist and trying to create it, the file was created and locked by someone else.
//When hashModifiedDate==true, we don't care what the file contains, we just want it to exist. If the file is available for
//reading within timeoutMs, simply do nothing and let the file be returned as a hit.
Stopwatch waitForFile = new Stopwatch();
bool opened = false;
while (!opened && waitForFile.ElapsedMilliseconds < timeoutMs)
{
waitForFile.Start();
try
{
using (var stream = container.GetBlockBlobReference(path).OpenRead())
opened = true;
}
catch (IOException iex)
{
Thread.Sleep((int)Math.Min(30, Math.Round((float)timeoutMs / 3.0)));
}
waitForFile.Stop();
}
if (!opened)
{
throw; //By not throwing an exception, it is considered a hit by the rest of the code.
}
}
}
}));
}
public override void Flush()
{
_inner.Flush();
}
private void ProcessQueueMessage(object state)
{
CloudQueueMessage msg = state as CloudQueueMessage;
try
{
// Log and delete if this is a "poison" queue message (repeatedly processed
// and always causes an error that prevents processing from completing).
// Production applications should move the "poison" message to a "dead message"
// queue for analysis rather than deleting the message.
if (msg.DequeueCount > 5)
{
Trace.TraceError("Deleting poison message: message {0} Role Instance {1}.",
msg.ToString(), GetRoleInstance());
DataModel.StorageFactory.Instance.IpcAzureAppWorkerJobQueue.DeleteMessage(msg);
return;
}
RmsCommand rmsCommand = new RmsCommand(msg.AsString);
switch (rmsCommand.RmsOperationCommand)
{
case RmsCommand.Command.GetTemplate:
{
ServicePrincipalModel sp = ServicePrincipalModel.GetFromStorage(rmsCommand.Parameters.First <object>().ToString());
RMS.RmsContentPublisher rmsPublisher = RMS.RmsContentPublisher.Create(sp.TenantId, sp.AppId, sp.Key);
var templates = rmsPublisher.GetRmsTemplates();
List <TemplateModel> templateEntityList = new List <TemplateModel>();
foreach (var temp in templates)
{
TemplateModel templateEntity = new TemplateModel();
templateEntity.TenantId = sp.TenantId;
templateEntity.TemplateId = temp.TemplateId;
templateEntity.TemplateName = temp.Name;
templateEntity.TemplateDescription = temp.Description;
templateEntityList.Add(templateEntity);
}
TemplateModel.SaveToStorage(templateEntityList);
break;
}
case RmsCommand.Command.PublishTemplate:
{
PublishModel publishJob = PublishModel.GetFromStorage(rmsCommand.Parameters[0].ToString(), rmsCommand.Parameters[1].ToString());
ServicePrincipalModel sp = ServicePrincipalModel.GetFromStorage(rmsCommand.Parameters[0].ToString());
CloudBlockBlob originalFileBlob = DataModel.StorageFactory.Instance.IpcAzureAppFileBlobContainer.GetBlockBlobReference(publishJob.OriginalFileBlobRef);
Stream sinkStream = null;
string tempFilePath = null;
try
{
//if file length is less than 100,000 bytes, keep it in memory
if (publishJob.OriginalFileSizeInBytes < 100000)
{
sinkStream = new MemoryStream();
}
else
{
tempFilePath = Path.GetRandomFileName();
sinkStream = new FileStream(tempFilePath, FileMode.Create);
}
using (Stream sourceStream = originalFileBlob.OpenRead())
using (sinkStream)
{
RMS.RmsContent rmsContent = new RMS.RmsContent(sourceStream, sinkStream);
rmsContent.RmsTemplateId = publishJob.TemplateId;
rmsContent.OriginalFileNameWithExtension = publishJob.OriginalFileName;
RMS.RmsContentPublisher rmsContentPublisher = RMS.RmsContentPublisher.Create(sp.TenantId, sp.AppId, sp.Key);
rmsContentPublisher.PublishContent(rmsContent);
publishJob.PublishedFileName = rmsContent.PublishedFileNameWithExtension;
sinkStream.Flush();
sinkStream.Seek(0, SeekOrigin.Begin);
//published file is uploaded to blob storage.
//Note: This sample code doesn't manage lifetime of this original and published file blob
//Actual code must manage the lifetime as appropriate
CloudBlockBlob destFileBlob = DataModel.StorageFactory.Instance.IpcAzureAppFileBlobContainer.GetBlockBlobReference(publishJob.PublishedFileBlobRef);
using (CloudBlobStream blobStream = destFileBlob.OpenWrite())
{
int tempSize = 1024;
byte[] tempBuffer = new byte[tempSize];
while (true)
{
int readSize = sinkStream.Read(tempBuffer, 0, tempSize);
if (readSize <= 0)
{
break;
}
blobStream.Write(tempBuffer, 0, readSize);
}
blobStream.Flush();
}
}
publishJob.JState = PublishModel.JobState.Completed.ToString();
publishJob.SaveToStorage();
break;
}
finally
{
if (!string.IsNullOrWhiteSpace(tempFilePath) && File.Exists(tempFilePath))
{
File.Delete(tempFilePath);
}
}
}
}
//delete the message from the queue
DataModel.StorageFactory.Instance.IpcAzureAppWorkerJobQueue.DeleteMessage(msg);
}
catch (Exception ex)
{
Process p = Process.GetCurrentProcess();
string a = p.ProcessName;
string b = p.MainModule.FileName;
string err = ex.Message;
if (ex.InnerException != null)
{
err += " Inner Exception: " + ex.InnerException.Message;
}
if (msg != null)
{
err += " Last queue message retrieved: " + msg.AsString;
}
Trace.TraceError(err);
}
}