/// <summary>
/// Process events
/// </summary>
/// <param name="context"></param>
/// <param name="messages"></param>
/// <returns></returns>
public Task ProcessEventsAsync(PartitionContext context, IEnumerable <EventData> messages)
{
long lastSeq = 0;
foreach (var eventData in messages)
{
var message = BinarySerializer.DeserializeStreamEventLong(eventData.Body.ToArray());
lastSeq = eventData.SystemProperties.SequenceNumber;
this.input.OnNext(message);
}
if (this.checkpointStopWatch.Elapsed > TimeSpan.FromSeconds(10))
{
Console.WriteLine("Taking checkpoint");
var storageAccount = CloudStorageAccount.Parse(StorageConnectionString);
var blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference("checkpoints");
var blockBlob = container.GetBlockBlobReference(context.PartitionId + "-" + lastSeq);
CloudBlobStream blobStream = blockBlob.OpenWriteAsync().GetAwaiter().GetResult();
this.queryProcess.Checkpoint(blobStream);
blobStream.Flush();
blobStream.Close();
return(context
.CheckpointAsync()
.ContinueWith(t => DeleteOlderCheckpoints(context.PartitionId + "-" + lastSeq)));
}
return(Task.CompletedTask);
}
public string CreateBlob(List <Tone> tones, string fileId, int baseSamplesPerPixel = 88200, bool synth = true)
{
UseSynth = synth;
var samples = GetTotalSamples(baseSamplesPerPixel, tones);
string blobReference;
CloudBlobStream stream = _repo.GetStream(fileId, out blobReference);
WriteWaves(tones, baseSamplesPerPixel, stream, samples);
stream.Close();
return(blobReference);
}
/// <summary>
/// Process events
/// </summary>
/// <param name="context"></param>
/// <param name="messages"></param>
/// <returns></returns>
public Task ProcessEventsAsync(PartitionContext context, IEnumerable <EventData> messages)
{
EventProcessor.__count += messages.Count();
if (DateTime.UtcNow > LastTime.AddSeconds(2))
{
LastTime = DateTime.UtcNow;
Console.Title = $"total: {EventProcessor.__count} offset: {context.ToString()} count: {messages.Count()}";
}
#if true
context.CheckpointAsync();
#else
long lastSeq = 0;
foreach (var eventData in messages)
{
var message = BinarySerializer.DeserializeStreamEventSampleEvent <Measure>(eventData.Body.ToArray());
lastSeq = eventData.SystemProperties.SequenceNumber;
Task.Delay(5).Wait();
this.input.OnNext(message);
//else if (message.Payload.Data is MeasureT3)
//{
// var serializer = Newtonsoft.Json.JsonSerializer.Create();
// var value = JsonConvert.DeserializeObject<StreamEvent<Measure<MeasureT3>>>(JsonConvert.SerializeObject(message));
// this.input.OnNext(value);
//}
//else if (message.Payload.Data is MeasureT4)
// this.input.OnNext((StreamEvent<Measure>)(object)(StreamEvent<Measure<MeasureT4>>)(object)message);
}
if (this.checkpointStopWatch.Elapsed > TimeSpan.FromSeconds(10))
{
Console.WriteLine("Taking checkpoint");
var storageAccount = CloudStorageAccount.Parse(StorageConnectionString);
var blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference("checkpoints");
var blockBlob = container.GetBlockBlobReference(context.PartitionId + "-" + lastSeq);
CloudBlobStream blobStream = blockBlob.OpenWriteAsync().GetAwaiter().GetResult();
this.queryProcess.Checkpoint(blobStream);
blobStream.Flush();
blobStream.Close();
return(context
.CheckpointAsync()
.ContinueWith(t => DeleteOlderCheckpoints(context.PartitionId + "-" + lastSeq)));
}
#endif
return(Task.CompletedTask);
}
private void WriteLocalDataToCloudIfNeeded()
{
lock (this.syncRoot)
{
byte[] localContentHash;
if (!Serialization.ModifiedSince(
this.localObject,
this.lastKnownBlobContentsHash,
out localContentHash))
{
return;
}
AccessCondition writeAccessConditionAtStartOfUpload = new AccessCondition();
lock (this.writeAccessCondition)
{
writeAccessConditionAtStartOfUpload.LeaseId = this.writeAccessCondition.LeaseId;
}
StorageOperation.Try(
() =>
{
OperationContext context = new OperationContext();
CloudBlobStream uploadStream = this.backingBlob.OpenWrite(
accessCondition: writeAccessConditionAtStartOfUpload,
operationContext: context);
Serialization.SerializeIntoStream(this.localObject, uploadStream);
uploadStream.Close();
this.readAccessCondition.IfNoneMatchETag = context.LastResult.Etag;
this.lastKnownBlobContentsHash = localContentHash;
},
catchHttp400: e =>
{
// Can happen when two clients race to perform an upload to the same blob; one
// client will get a HTTP 400 when committing their list of uploaded blocks.
OnWriteToCloudFailure(e, writeAccessConditionAtStartOfUpload);
},
catchHttp412: e =>
{
// There is a lease in place that prevents this write.
OnWriteToCloudFailure(e, writeAccessConditionAtStartOfUpload);
});
}
}
/// <summary>
/// Define a sharded vertex type and register with CRA.
/// </summary>
/// <param name="vertexDefinition">Name of the vertex type</param>
/// <param name="creator">Lambda that describes how to instantiate the vertex, taking in an object as parameter</param>
public CRAErrorCode DefineVertex(string vertexDefinition, Expression <Func <IShardedVertex> > creator)
{
CloudBlobContainer container = _blobClient.GetContainerReference("cra");
container.CreateIfNotExistsAsync().Wait();
var blockBlob = container.GetBlockBlobReference(vertexDefinition + "/binaries");
CloudBlobStream blobStream = blockBlob.OpenWriteAsync().GetAwaiter().GetResult();
AssemblyUtils.WriteAssembliesToStream(blobStream);
blobStream.Close();
// Add metadata
var newRow = new VertexTable("", vertexDefinition, vertexDefinition, "", 0, creator, null, true);
TableOperation insertOperation = TableOperation.InsertOrReplace(newRow);
_vertexTable.ExecuteAsync(insertOperation).Wait();
return(CRAErrorCode.Success);
}
public static void Run([EventGridTrigger] EventGridEvent eventGridEvent,
[Blob("{data.fileUrl}", FileAccess.Read, Connection = "feedstore_STORAGE")] CloudBlockBlob inputBlob,
[Blob("parquet/{rand-guid}.parquet", FileAccess.Write, Connection = "feedstore_STORAGE")] CloudBlockBlob outputBlob,
TraceWriter log)
{
log.Info(eventGridEvent.ToString());
JObject dataObject = eventGridEvent.Data as JObject;
var eventData = dataObject.ToObject <EventHubCaptureFileCreatedEventData>();
log.Info(eventData.EventCount.ToString() + " events in record");
if (eventData.EventCount > 0)
{
log.Info("Got a real file");
Stream avroStream = inputBlob.OpenReadAsync().Result;
CloudBlobStream parquetStream = outputBlob.OpenWriteAsync().Result;
CreateParquetFile(avroStream, parquetStream);
parquetStream.Close();
}
}
internal CRAErrorCode InstantiateVertex(string instanceName, string vertexName, string vertexDefinition, object vertexParameter, bool sharded)
{
var procDefRow = VertexTable.GetRowForVertexDefinition(_vertexTable, vertexDefinition);
string blobName = vertexName + "-" + instanceName;
// Serialize and write the vertex parameters to a blob
CloudBlobContainer container = _blobClient.GetContainerReference("cra");
container.CreateIfNotExistsAsync().Wait();
var blockBlob = container.GetBlockBlobReference(vertexDefinition + "/" + blobName);
CloudBlobStream blobStream = blockBlob.OpenWriteAsync().GetAwaiter().GetResult();
byte[] parameterBytes = Encoding.UTF8.GetBytes(
SerializationHelper.SerializeObject(vertexParameter));
blobStream.WriteByteArray(parameterBytes);
blobStream.Close();
// Add metadata
var newRow = new VertexTable(instanceName, vertexName, vertexDefinition, "", 0,
procDefRow.VertexCreateAction,
blobName,
false, sharded);
TableOperation insertOperation = TableOperation.InsertOrReplace(newRow);
_vertexTable.ExecuteAsync(insertOperation).Wait();
CRAErrorCode result = CRAErrorCode.Success;
// Send request to CRA instance
VertexTable instanceRow;
try
{
instanceRow = VertexTable.GetRowForInstance(_vertexTable, instanceName);
// Get a stream connection from the pool if available
Stream stream;
if (!TryGetSenderStreamFromPool(instanceRow.Address, instanceRow.Port.ToString(), out stream))
{
TcpClient client = new TcpClient(instanceRow.Address, instanceRow.Port);
client.NoDelay = true;
stream = client.GetStream();
if (SecureStreamConnectionDescriptor != null)
{
stream = SecureStreamConnectionDescriptor.CreateSecureClient(stream, instanceName);
}
}
stream.WriteInt32((int)CRATaskMessageType.LOAD_VERTEX);
stream.WriteByteArray(Encoding.UTF8.GetBytes(vertexName));
stream.WriteByteArray(Encoding.UTF8.GetBytes(vertexDefinition));
stream.WriteByteArray(Encoding.UTF8.GetBytes(newRow.VertexParameter));
result = (CRAErrorCode)stream.ReadInt32();
if (result != 0)
{
Console.WriteLine("Vertex was logically loaded. However, we received an error code from the hosting CRA instance: " + result);
}
// Add/Return stream connection to the pool
TryAddSenderStreamToPool(instanceRow.Address, instanceRow.Port.ToString(), stream);
}
catch
{
Console.WriteLine("The CRA instance appears to be down. Restart it and this vertex will be instantiated automatically");
}
return(result);
}
public override void Close()
{
_inner.Close();
}
/// <summary>
/// Execute method is the only method of IDotNetActivity interface you must implement.
/// In this sample, the method invokes the Calculate method to perform the core logic.
/// </summary>
public IDictionary <string, string> Execute(
IEnumerable <LinkedService> linkedServices,
IEnumerable <Dataset> datasets,
Activity activity,
IActivityLogger logger)
{
/////////////////
// Log input parameters
// to get extended properties (for example: SliceStart)
foreach (LinkedService ls in linkedServices)
{
logger.Write("linkedServices: {0}, {1}, {2}, {3}", ls.Name, ls.Properties.Type,
ls.Properties.Description, ls.Properties.ErrorMessage);
}
var sliceYear = ((DotNetActivity)activity.TypeProperties).ExtendedProperties["Year"];
var sliceMonth = ((DotNetActivity)activity.TypeProperties).ExtendedProperties["Month"];
var sliceDay = ((DotNetActivity)activity.TypeProperties).ExtendedProperties["Day"];
logger.Write("dataSlice: {0}-{1}-{2}", sliceYear, sliceMonth, sliceDay);
/////////////////
// Open up input Blob
var inputDataset = datasets.Single(dataset => dataset.Name == activity.Inputs.Single().Name);
var inputLinkedService = linkedServices.Single(
linkedService =>
linkedService.Name ==
inputDataset.Properties.LinkedServiceName);
var inputLocation = new BlobLocation(inputLinkedService, inputDataset, sliceYear, sliceMonth, sliceDay);
var inputContainer = new CloudBlobContainer(inputLocation.ConnectionSasUri);
var sourceBlob = inputContainer.GetBlobReferenceFromServer(inputLocation.BlobFullPath);
////////////////
// Get output location
var outputDataset = datasets.Single(dataset => dataset.Name == activity.Outputs.Single().Name);
var outputLinkedService = linkedServices.Single(
linkedService =>
linkedService.Name ==
outputDataset.Properties.LinkedServiceName);
var outputLocation = new BlobLocation(outputLinkedService, outputDataset, sliceYear, sliceMonth, sliceDay);
CloudStorageAccount outputStorageAccount = CloudStorageAccount.Parse(outputLocation.ConnectionString);
CloudBlobClient outputClient = outputStorageAccount.CreateCloudBlobClient();
var outContainer = outputClient.GetContainerReference(outputLocation.ContainerName);
outContainer.CreateIfNotExists();
//format output path string
var outputFilenameFormatString = outputLocation.BlobFullPath;
using (var sourceBlobStream = sourceBlob.OpenRead())
using (var unZipStream = new System.IO.Compression.GZipStream(sourceBlobStream, System.IO.Compression.CompressionMode.Decompress))
using (var tarStream = new TarStream(unZipStream))
{
logger.Write("BlobRead: {0}/{1}", inputLocation.ContainerName, inputLocation.BlobFullPath);
while (tarStream.NextFile())
{
var tableName = Path.GetFileNameWithoutExtension(tarStream.CurrentFilename);
var taredFileExtention = Path.GetExtension(tarStream.CurrentFilename);
if (taredFileExtention == ".bson")
{
int fileNumber = 0;
var outputBlob = outContainer.GetBlockBlobReference(outputFilenameFormatString.Replace("{EventName}", tableName).Replace("{Number}", fileNumber.ToString("D4")));
CloudBlobStream outBlobStream = null;
GZipStream gzipOut = null;
StreamWriter outText = null;
try
{
outBlobStream = outputBlob.OpenWrite();
gzipOut = new GZipStream(outBlobStream, System.IO.Compression.CompressionLevel.Optimal);
outText = new StreamWriter(gzipOut, Encoding.UTF8);
using (var reader = new BsonReader(tarStream))
{
logger.Write("BlobWrite: {0}/{1}", outputLocation.ContainerName, outputBlob.Name);
reader.CloseInput = false;
var jsonSerializer = new JsonSerializer();
reader.ReadRootValueAsArray = false;
reader.SupportMultipleContent = true;
while (reader.Read())
{
var row = (JObject)jsonSerializer.Deserialize(reader);
var outString = row.ToString(Formatting.None);
outText.WriteLine(outString);
if (outBlobStream.Position > 1024 * 1024 * 1024)
{
outText.Close();
gzipOut.Close();
outBlobStream.Close();
fileNumber++;
outputBlob = outContainer.GetBlockBlobReference(outputFilenameFormatString.Replace("{EventName}", tableName).Replace("{Number}", fileNumber.ToString("D4")));
logger.Write("BlobWrite: {0}/{1}", outputLocation.ContainerName, outputBlob.Name);
outBlobStream = outputBlob.OpenWrite();
gzipOut = new GZipStream(outBlobStream, System.IO.Compression.CompressionLevel.Optimal);
outText = new StreamWriter(gzipOut, Encoding.UTF8);
}
}
}
} finally
{
if (outText != null)
{
outText.Close();
}
if (gzipOut != null)
{
gzipOut.Close();
}
if (outBlobStream != null)
{
outBlobStream.Close();
}
}
}
}
;
}
// return a new Dictionary object
return(new Dictionary <string, string>());
}