public static async Task Run([BlobTrigger("databus/{name}", Connection = "DataBusStorageAccount")] CloudBlockBlob myBlob, [OrchestrationClient] DurableOrchestrationClient starter, TraceWriter log) { log.Info($"Blob created at {myBlob.Uri}"); var instanceId = myBlob.Name; var existingInstance = await starter.GetStatusAsync(instanceId); if (existingInstance != null) { log.Info($"{nameof(DataBusCleanupOrchestrator)} has already been started for blob {myBlob.Uri}."); return; } var validUntilUtc = DataBusBlobTimeoutCalculator.GetValidUntil(myBlob); if (validUntilUtc == DateTime.MaxValue) { log.Error($"Could not parse the 'ValidUntil' value `{myBlob.Metadata["ValidUntil"]}` for blob {myBlob.Uri}. Cleanup will not happen on this blob. You may consider manually removing this entry if non-expiry is incorrect."); return; } await starter.StartNewAsync(nameof(DataBusCleanupOrchestrator), instanceId, new DataBusBlobData { Path = myBlob.Uri.ToString(), ValidUntilUtc = DataBusBlobTimeoutCalculator.ToWireFormattedString(validUntilUtc) }); }
public static async Task Run( [OrchestrationClient] DurableOrchestrationClient starter, TraceWriter log) { var storageConnectionString = Environment.GetEnvironmentVariable("DataBusStorageAccount"); var cloudStorageAccount = CloudStorageAccount.Parse(storageConnectionString); var cloudBlobClient = cloudStorageAccount.CreateCloudBlobClient(); var container = cloudBlobClient.GetContainerReference("databus"); BlobContinuationToken token = null; do { var segment = await container.ListBlobsSegmentedAsync(token).ConfigureAwait(false); token = segment.ContinuationToken; foreach (var blockBlob in segment.Results.Where(blob => blob is CloudBlockBlob).Cast <CloudBlockBlob>()) { var instanceId = blockBlob.Name; var existingInstance = await starter.GetStatusAsync(instanceId); if (existingInstance != null) { log.Info($"{nameof(DataBusCleanupOrchestrator)} has already been started for blob {blockBlob.Uri}."); continue; } var validUntilUtc = DataBusBlobTimeoutCalculator.GetValidUntil(blockBlob); if (validUntilUtc == DateTime.MaxValue) { log.Error($"Could not parse the 'ValidUntil' value `{blockBlob.Metadata["ValidUntil"]}` for blob {blockBlob.Uri}. Cleanup will not happen on this blob. You may consider manually removing this entry if non-expiry is incorrect."); continue; } await starter.StartNewAsync(nameof(DataBusCleanupOrchestrator), instanceId, new DataBusBlobData { Path = blockBlob.Uri.ToString(), ValidUntilUtc = DataBusBlobTimeoutCalculator.ToWireFormattedString(validUntilUtc) }); } } while (token != null); }
public static async Task RunOrchestrator([OrchestrationTrigger] DurableOrchestrationContext context, TraceWriter log) { var blobData = context.GetInput <DataBusBlobData>(); log.Info($"Orchestrating deletion for blob at {blobData.Path} with ValidUntilUtc of {blobData.ValidUntilUtc}"); var validUntilUtc = DataBusBlobTimeoutCalculator.ToUtcDateTime(blobData.ValidUntilUtc); DateTime timeoutUntil; //Timeouts currently have a 7 day limit, use 6 day loops until the wait is less than 6 days do { timeoutUntil = validUntilUtc > context.CurrentUtcDateTime.AddDays(2) ? context.CurrentUtcDateTime.AddDays(2) : validUntilUtc; log.Info($"Waiting until {timeoutUntil}/{validUntilUtc} for blob at {blobData.Path}. Currently {context.CurrentUtcDateTime}."); await context.CreateTimer(DataBusBlobTimeoutCalculator.ToUtcDateTime(blobData.ValidUntilUtc), CancellationToken.None); } while (validUntilUtc > timeoutUntil); var blob = await cloudBlobClient.GetBlobReferenceFromServerAsync(new Uri(blobData.Path)); log.Info($"Deleting blob at {blobData.Path}"); await blob.DeleteIfExistsAsync(); }
public DataBusCleanupOrchestrator(DataBusBlobTimeoutCalculator calculator) { this.calculator = calculator; }
public DataBusBlobCreated(DataBusBlobTimeoutCalculator calculator) { this.calculator = calculator; }
public DataBusOrchestrateExistingBlobs(CloudBlobContainer container, DataBusBlobTimeoutCalculator calculator) { this.container = container; this.calculator = calculator; }