public Task UploadFromStreamAsync(Stream stream, CancellationToken cancellationToken)
{
return(this.baseBlockBlob.UploadFromStreamAsync(
stream,
AccessCondition.GenerateIfExistsCondition(),
null, null, cancellationToken));
}
public void GenerateIfExistsConditionReturnsIfMatchAccessCondition()
{
AccessCondition result = AccessCondition.GenerateIfExistsCondition();
Assert.Equal("*", result.IfMatch);
Assert.Null(result.IfNoneMatch);
}
/// <summary>
/// Process blob
/// </summary>
/// <param name="blobUri"></param>
/// <param name="properties"></param>
/// <param name="ct"></param>
/// <returns></returns>
private async Task ProcessBlobAsync(string blobUri,
IDictionary <string, string> properties, CancellationToken ct)
{
try {
var blob = new CloudBlockBlob(new Uri(blobUri), _client);
while (true)
{
var context = new OperationContext();
var stream = await blob.OpenReadAsync(
AccessCondition.GenerateIfExistsCondition(), _options, context, ct);
properties.AddOrUpdate("RequestId", context.ClientRequestID);
var disposition = await _processor.ProcessAsync(stream, properties, ct);
if (disposition == BlobDisposition.Retry)
{
continue;
}
if (disposition != BlobDisposition.Delete)
{
break;
}
await blob.DeleteAsync(DeleteSnapshotsOption.IncludeSnapshots,
AccessCondition.GenerateIfExistsCondition(), _options, context, ct);
break;
}
}
catch (StorageException ex) {
_logger.Error(ex, "Failed to process blob stream due to storage exception");
}
}
private void AssertAccessCondition(Uri resourceUri, AccessCondition accessCondition)
{
Content.TryGetValue(resourceUri, out var existingContent);
if (IsAccessCondition(AccessCondition.GenerateEmptyCondition(), accessCondition))
{
return;
}
if (IsAccessCondition(AccessCondition.GenerateIfNotExistsCondition(), accessCondition))
{
Assert.Null(existingContent);
return;
}
if (IsAccessCondition(AccessCondition.GenerateIfExistsCondition(), accessCondition))
{
Assert.NotNull(existingContent);
return;
}
if (existingContent is StringStorageContentWithETag eTagContent)
{
var eTag = eTagContent.ETag;
if (IsAccessCondition(AccessCondition.GenerateIfMatchCondition(eTag), accessCondition))
{
return;
}
}
throw new InvalidOperationException("Could not validate access condition!");
}
private async Task <CloudAppendBlob> GetAppendBlobReferenceWithAttributes(string fileId,
CancellationToken cancellationToken)
{
var file = GetAppendBlobReference(fileId);
await file.FetchAttributesAsync(AccessCondition.GenerateIfExistsCondition(), new BlobRequestOptions(),
new OperationContext(), cancellationToken);
return(file);
}
protected override async Task DeleteAsync(SnapshotMetadata metadata)
{
var blob = Container.GetBlockBlobReference(metadata.ToSnapshotBlobId());
using (var cts = new CancellationTokenSource(_settings.RequestTimeout))
{
await blob.DeleteIfExistsAsync(DeleteSnapshotsOption.None, AccessCondition.GenerateIfExistsCondition(),
GenerateOptions(), new OperationContext(),
cts.Token);
}
}
protected override async Task <SelectedSnapshot> LoadAsync(string persistenceId,
SnapshotSelectionCriteria criteria)
{
var requestOptions = GenerateOptions();
BlobResultSegment results = null;
using (var cts = new CancellationTokenSource(_settings.RequestTimeout))
{
results = await Container.ListBlobsSegmentedAsync(SeqNoHelper.ToSnapshotSearchQuery(persistenceId),
true,
BlobListingDetails.Metadata, null, null, requestOptions, new OperationContext(), cts.Token);
}
// if we made it down here, the initial request succeeded.
async Task <SelectedSnapshot> FilterAndFetch(BlobResultSegment segment)
{
// apply filter criteria
var filtered = segment.Results
.Where(x => x is CloudBlockBlob)
.Cast <CloudBlockBlob>()
.Where(x => FilterBlobSeqNo(criteria, x))
.Where(x => FilterBlobTimestamp(criteria, x))
.OrderByDescending(x => FetchBlobSeqNo(x)) // ordering matters - get highest seqNo item
.ThenByDescending(x =>
FetchBlobTimestamp(
x)) // if there are multiple snapshots taken at same SeqNo, need latest timestamp
.FirstOrDefault();
// couldn't find what we were looking for. Onto the next part of the query
// or return null to sender possibly.
if (filtered == null)
{
return(null);
}
using (var cts = new CancellationTokenSource(_settings.RequestTimeout))
using (var memoryStream = new MemoryStream())
{
await filtered.DownloadToStreamAsync(memoryStream, AccessCondition.GenerateIfExistsCondition(),
GenerateOptions(), new OperationContext(), cts.Token);
var snapshot = _serialization.SnapshotFromBytes(memoryStream.ToArray());
return(new SelectedSnapshot(new SnapshotMetadata(persistenceId, FetchBlobSeqNo(filtered)),
snapshot.Data));
}
}
// TODO: see if there's ever a scenario where the most recent snapshots aren't in the beginning of the pagination list.
var result = await FilterAndFetch(results);
return(result);
}
internal static void DeleteIfExistsWorksOnlyWhenResourceExists <T>(
Action <string, SearchRequestOptions, AccessCondition> deleteAction,
Func <T> createResource,
string resourceName)
where T : IResourceWithETag
{
Action <string, AccessCondition> delete = (a, b) => deleteAction(a, null, b);
createResource();
delete(resourceName, AccessCondition.GenerateIfExistsCondition());
SearchAssert.ThrowsCloudException(
() => delete(resourceName, AccessCondition.GenerateIfExistsCondition()),
e => e.IsAccessConditionFailed());
}
internal static void UpdateIfExistsFailsOnNoResource <T>(
Func <T, SearchRequestOptions, AccessCondition, T> createOrUpdateFunc,
Func <T> newResourceDefinition)
where T : IResourceWithETag
{
Func <T, AccessCondition, T> createOrUpdate = (a, b) => createOrUpdateFunc(a, null, b);
var resource = newResourceDefinition();
SearchAssert.ThrowsCloudException(
() => createOrUpdate(resource, AccessCondition.GenerateIfExistsCondition()),
e => e.IsAccessConditionFailed());
// The resource should never have been created on the server, and thus it should not have an ETag
Assert.Null(resource.ETag);
}
internal static void UpdateIfExistsSucceedsOnExistingResource <T>(
Func <T, SearchRequestOptions, AccessCondition, T> createOrUpdateFunc,
Func <T> newResourceDefinition,
Func <T, T> mutateResourceDefinition)
where T : IResourceWithETag
{
Func <T, AccessCondition, T> createOrUpdate = (a, b) => createOrUpdateFunc(a, null, b);
var createdResource = createOrUpdate(newResourceDefinition(), AccessCondition.GenerateEmptyCondition());
var mutatedResource = mutateResourceDefinition(createdResource);
var updatedResource = createOrUpdate(mutatedResource, AccessCondition.GenerateIfExistsCondition());
Assert.NotEmpty(updatedResource.ETag);
Assert.NotEqual(createdResource.ETag, updatedResource.ETag);
}
/// <summary>
/// Upload file in blob storage. If it exists will override it
/// </summary>
/// <param name="fileName">File in this location will be stored in cloud</param>
/// <param name="metadata"></param>
public string UploadFile(string ContainerName, string fileName, string TempFolder, Dictionary <string, string> metadata = null)
{
string tmpfileName = Path.Combine(TempFolder, Path.GetFileName(fileName));
try
{
string ErrorMsg = null;
var cloudBlobContainer = _cloudBlobClient.GetContainerReference(ContainerName.ToLower());
cloudBlobContainer.CreateIfNotExists();
//cloudBlobContainer.GetSharedAccessSignature()
CloudBlockBlob blob = cloudBlobContainer.GetBlockBlobReference(Path.GetFileName(tmpfileName));
blob.StreamWriteSizeInBytes = 256 * 1024;
blob.UploadFromFile(tmpfileName, AccessCondition.GenerateIfExistsCondition(), new BlobRequestOptions(), new OperationContext());
metadata.Add("FileName", tmpfileName);
metadata.Add("RunDate", DateTime.Now.Date.Ticks.ToString());
if (metadata != null)
{
metadata.Keys.ToList().ForEach(key =>
{
var value = metadata[key];
value = Regex.Replace(value,
@"[,\(\)" + Regex.Escape(new string(Path.GetInvalidFileNameChars())) + "]", "_");
if (!blob.Metadata.ContainsKey(key))
{
blob.Metadata.Add(key, value);
}
else
{
blob.Metadata[key] = value;
}
});
}
blob.SetMetadata();
return(blob.Uri.ToString());
}
catch (Exception ex)
{
Console.WriteLine("Uploading fail: " + ex);
return(null);
}
}
private async Task DownloadBlob()
{
Console.WriteLine("Downloading...");
var storageAccount = CloudStorageAccount.Parse(storageConnectionString);
var blobClient = storageAccount.CreateCloudBlobClient();
var containerReference = blobClient.GetContainerReference("consoleappcontainer");
var blobReference = containerReference.GetBlockBlobReference("myblob.txt");
var blobContent = await blobReference.DownloadTextAsync(
Encoding.UTF8,
AccessCondition.GenerateIfExistsCondition(),
new BlobRequestOptions()
{
EncryptionPolicy = new BlobEncryptionPolicy(null, keyResolver)
},
null);
Console.WriteLine(blobContent);
}
/// <summary>
/// NB Not handling case where source immutable blob is deleted at same time as copy.
/// NB Not using GetBlobReferenceFromServer as this requires a lot of complex exception handling.
/// </summary>
/// <param name="relativePath"></param>
public async Task Execute(string relativePath)
{
try
{
var source = _SourceStorageAccountConfig.GetBlobDirect(relativePath, AccessCondition.GenerateIfExistsCondition());
var destination = _DestinationStorageAccountConfig.GetBlobIndirect(relativePath);
if (destination.Exists())
{
return;
}
destination.StartCopy(source);
}
catch (Exception e)
{
throw;
}
}
public async Task MoveAsync(string blobName, string destinationContainerName, CancellationToken token)
{
if (string.IsNullOrEmpty(blobName))
{
throw new ArgumentException("message", nameof(blobName));
}
var destinationDirectory = _blobDirectory.GetDirectoryReference(destinationContainerName);
var sourceBlob = GetBlob(blobName);
var destinationBlob = destinationDirectory.GetBlockBlobReference(blobName);
await destinationBlob.StartCopyAsync(sourceBlob, AccessCondition.GenerateIfExistsCondition(), AccessCondition.GenerateEmptyCondition(), new BlobRequestOptions()
{
StoreBlobContentMD5 = true
}, null, token);
while (destinationBlob.CopyState.Status != CopyStatus.Success)
{
await Task.Delay(TimeSpan.FromSeconds(0.2), token);
await destinationBlob.ExistsAsync();
}
await sourceBlob.DeleteAsync();
}
private void ForwardLogAttachments(DeviceLog logSchema)
{
// Export attachments
if (logSchema.MessageType == MessageTypes.ErrorLog ||
logSchema.MessageType == MessageTypes.ErrorAttachmentLog ||
logSchema.MessageType == MessageTypes.HandledErrorLog)
{
var sharedAccessBlobPolicy = new SharedAccessBlobPolicy
{
Permissions = SharedAccessBlobPermissions.Read,
SharedAccessExpiryTime = DateTimeOffset.UtcNow.AddHours(1),
SharedAccessStartTime = DateTimeOffset.UtcNow.AddHours(-1),
};
var attachmentBlobName = logSchema.Properties;
var attachmentBlobSas = InputBlobContainer.GetSharedAccessSignature(sharedAccessBlobPolicy);
var outputBlobReference = OutputBlobContainer.GetBlockBlobReference(attachmentBlobName);
outputBlobReference.StartCopyAsync(new Uri(attachmentBlobSas),
AccessCondition.GenerateIfExistsCondition(),
AccessCondition.GenerateEmptyCondition(),
RequestOptions,
new OperationContext()).GetAwaiter().GetResult();
}
}
public override async Task DeleteTemporaryFilesAsync(int hours)
{
var files = _container.ListBlobs(prefix: TEMP_PREFIX).OfType <CloudBlob>()
.Where(b => b.Name.EndsWith(".TMP") && b.Properties.LastModified.HasValue && b.Properties.LastModified.Value < DateTimeOffset.Now.AddHours(hours * -1));
foreach (var file in files)
{
if (await file.DeleteIfExistsAsync(DeleteSnapshotsOption.IncludeSnapshots, AccessCondition.GenerateIfExistsCondition(), null, null))
{
_logger.LogInformation($"Temporary file { file.Name } deleted from Azure Blob Storage as part of cleanup due to being older than { hours } hours.");
}
}
}
// This sample shows how ETags work by performing conditional updates and deletes
// on an Azure Search index.
static void Main(string[] args)
{
IConfigurationBuilder builder = new ConfigurationBuilder().AddJsonFile("appsettings.json");
IConfigurationRoot configuration = builder.Build();
SearchServiceClient serviceClient = CreateSearchServiceClient(configuration);
Console.WriteLine("Deleting index...\n");
DeleteTestIndexIfExists(serviceClient);
// Every top-level resource in Azure Search has an associated ETag that keeps track of which version
// of the resource you're working on. When you first create a resource such as an index, its ETag is
// empty.
Index index = DefineTestIndex();
Console.WriteLine(
$"Test index hasn't been created yet, so its ETag should be blank. ETag: '{index.ETag}'");
// Once the resource exists in Azure Search, its ETag will be populated. Make sure to use the object
// returned by the SearchServiceClient! Otherwise, you will still have the old object with the
// blank ETag.
Console.WriteLine("Creating index...\n");
index = serviceClient.Indexes.Create(index);
Console.WriteLine($"Test index created; Its ETag should be populated. ETag: '{index.ETag}'");
// ETags let you do some useful things you couldn't do otherwise. For example, by using an If-Match
// condition, we can update an index using CreateOrUpdate and be guaranteed that the update will only
// succeed if the index already exists.
index.Fields.Add(new Field("name", AnalyzerName.EnMicrosoft));
index =
serviceClient.Indexes.CreateOrUpdate(
index,
accessCondition: AccessCondition.GenerateIfExistsCondition());
Console.WriteLine(
$"Test index updated; Its ETag should have changed since it was created. ETag: '{index.ETag}'");
// More importantly, ETags protect you from concurrent updates to the same resource. If another
// client tries to update the resource, it will fail as long as all clients are using the right
// access conditions.
Index indexForClient1 = index;
Index indexForClient2 = serviceClient.Indexes.Get("test");
Console.WriteLine("Simulating concurrent update. To start, both clients see the same ETag.");
Console.WriteLine($"Client 1 ETag: '{indexForClient1.ETag}' Client 2 ETag: '{indexForClient2.ETag}'");
// Client 1 successfully updates the index.
indexForClient1.Fields.Add(new Field("a", DataType.Int32));
indexForClient1 =
serviceClient.Indexes.CreateOrUpdate(
indexForClient1,
accessCondition: AccessCondition.IfNotChanged(indexForClient1));
Console.WriteLine($"Test index updated by client 1; ETag: '{indexForClient1.ETag}'");
// Client 2 tries to update the index, but fails, thanks to the ETag check.
try
{
indexForClient2.Fields.Add(new Field("b", DataType.Boolean));
serviceClient.Indexes.CreateOrUpdate(
indexForClient2,
accessCondition: AccessCondition.IfNotChanged(indexForClient2));
Console.WriteLine("Whoops; This shouldn't happen");
Environment.Exit(1);
}
catch (CloudException e) when(e.IsAccessConditionFailed())
{
Console.WriteLine("Client 2 failed to update the index, as expected.");
}
// You can also use access conditions with Delete operations. For example, you can implement an
// atomic version of the DeleteTestIndexIfExists method from this sample like this:
Console.WriteLine("Deleting index...\n");
serviceClient.Indexes.Delete("test", accessCondition: AccessCondition.GenerateIfExistsCondition());
// This is slightly better than using the Exists method since it makes only one round trip to
// Azure Search instead of potentially two. It also avoids an extra Delete request in cases where
// the resource is deleted concurrently, but this doesn't matter much since resource deletion in
// Azure Search is idempotent.
// And we're done! Bye!
Console.WriteLine("Complete. Press any key to end application...\n");
Console.ReadKey();
}
protected override async Task DeleteAsync(string persistenceId, SnapshotSelectionCriteria criteria)
{
var requestOptions = GenerateOptions();
BlobResultSegment results = null;
using (var cts = new CancellationTokenSource(_settings.RequestTimeout))
{
/*
* Query only the metadata - don't need to stream the entire blob back to us
* in order to delete it from storage in the next request.
*/
results = await Container.ListBlobsSegmentedAsync(SeqNoHelper.ToSnapshotSearchQuery(persistenceId),
true,
BlobListingDetails.Metadata, null, null, requestOptions, new OperationContext(), cts.Token);
}
// if we made it down here, the initial request succeeded.
async Task FilterAndDelete(BlobResultSegment segment)
{
// apply filter criteria
var filtered = segment.Results.Where(x => x is CloudBlockBlob)
.Cast <CloudBlockBlob>()
.Where(x => FilterBlobSeqNo(criteria, x))
.Where(x => FilterBlobTimestamp(criteria, x));
var deleteTasks = new List <Task>();
using (var cts = new CancellationTokenSource(_settings.RequestTimeout))
{
foreach (var blob in filtered)
{
deleteTasks.Add(blob.DeleteIfExistsAsync(DeleteSnapshotsOption.None,
AccessCondition.GenerateIfExistsCondition(),
GenerateOptions(), new OperationContext(), cts.Token));
}
await Task.WhenAll(deleteTasks);
}
}
var continuationToken = results.ContinuationToken;
var deleteTask = FilterAndDelete(results);
while (continuationToken != null)
{
// get the next round of results in parallel with the deletion of the previous
var nextResults = await Container.ListBlobsSegmentedAsync(continuationToken);
// finish our previous delete tasks
await deleteTask;
// start next round of deletes
deleteTask = FilterAndDelete(nextResults);
// move the loop forward if there are more results to be processed still
continuationToken = nextResults.ContinuationToken;
}
// wait for the final delete operation to complete
await deleteTask;
}
public async Task DeleteIconAsync(
IStorage destinationStorage,
string destinationStoragePath,
CancellationToken cancellationToken,
string packageId,
string normalizedPackageVersion)
{
_logger.LogInformation("Deleting icon blob {IconPath}", destinationStoragePath);
var iconUri = new Uri(destinationStorage.BaseAddress, destinationStoragePath);
try
{
await destinationStorage.DeleteAsync(iconUri, cancellationToken, new DeleteRequestOptionsWithAccessCondition(AccessCondition.GenerateIfExistsCondition()));
}
catch
{
_telemetryService.TrackIconDeletionFailure(packageId, normalizedPackageVersion);
throw;
}
_telemetryService.TrackIconDeletionSuccess(packageId, normalizedPackageVersion);
}
public static async Task Run([TimerTrigger("0/10 * * * * *")] TimerInfo myTimer, ILogger log)
{
var connectionString = Environment.GetEnvironmentVariable("AzureWebJobsStorage");
var storageAccount = CloudStorageAccount.Parse(connectionString);
var cloudBlobClient = storageAccount.CreateCloudBlobClient();
var containerName = Environment.GetEnvironmentVariable("ContainerName") ?? "supercoolcontainer";
var container = cloudBlobClient.GetContainerReference(containerName);
await container.CreateIfNotExistsAsync();
var blobs = container.ListBlobs();
using (var memoryStream = new MemoryStream())
{
using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Create, true))
{
var blobsToPack = blobs.Where(b =>
b.GetType() == typeof(CloudBlockBlob))
.Cast <CloudBlockBlob>()
.Where(a => !a.Name.EndsWith(".zip"));
foreach (var blob in blobsToPack)
{
var entry = archive.CreateEntry(blob.Name);
using (var entryStream = entry.Open())
{
await blob.DownloadToStreamAsync(entryStream);
log.LogInformation($"added {blob.Name} to the zip");
}
await blob.DeleteIfExistsAsync(DeleteSnapshotsOption.IncludeSnapshots, AccessCondition.GenerateIfExistsCondition(), null, null);
log.LogInformation($"remove {blob.Name} from the container");
}
}
var filename = DateTime.Now.ToString("yyyyMMdd");
var reference = container.GetBlockBlobReference(filename + ".zip");
memoryStream.Position = 0;
await reference.UploadFromStreamAsync(memoryStream);
}
}
public Task <Stream> DownloadStreamAsync(CancellationToken cancellationToken)
{
return(this.baseBlockBlob.OpenReadAsync(
AccessCondition.GenerateIfExistsCondition(),
null, null, cancellationToken));
}
/// <summary>
/// Append the event to the end of the event stream
/// </summary>
/// <param name="eventInstance">
/// The event to append to the end of the event stream
/// </param>
/// <param name="expectedTopSequenceNumber">
/// if this is set to > 0 and the event stream is further on then a consistency issue has arisen and the
/// event should not be written but rather throw an error
/// </param>
/// <param name="eventVersionNumber">
/// The version number to add to the event wrapper
/// </param>
/// <param name="streamConstraint">
/// An additional constrain that must be satisfied by the event stream in order to persist the event
/// </param>
/// <returns></returns>
public async Task <IAppendResult> AppendEvent(IEvent eventInstance,
int expectedTopSequenceNumber = 0,
int eventVersionNumber = 1,
EventStreamExistenceConstraint streamConstraint = EventStreamExistenceConstraint.Loose)
{
if (base.EventStreamBlob != null)
{
// acquire a lease for the blob..
string writeStreamLeaseId = null;
if (await Exists())
{
writeStreamLeaseId = await base.EventStreamBlob.AcquireLeaseAsync(TimeSpan.FromSeconds(15));
}
int nextSequence = await base.GetSequenceNumber() + 1;
if (expectedTopSequenceNumber > 0)
{
// check against actual top sequence number
if ((expectedTopSequenceNumber + 1) < nextSequence)
{
throw new EventStreamWriteException(this,
(nextSequence - 1),
message: $"Out of sequence write - expected seqeunce number {expectedTopSequenceNumber }",
source: "Blob Event Stream Writer");
}
}
string eventName = "";
if (null != eventInstance)
{
eventName = EventNameAttribute.GetEventName(eventInstance.GetType());
}
// create an access condition
AccessCondition condition = AccessCondition.GenerateEmptyCondition();
if (streamConstraint == EventStreamExistenceConstraint.MustBeNew)
{
condition = AccessCondition.GenerateIfNotExistsCondition();
}
if (streamConstraint == EventStreamExistenceConstraint.MustExist)
{
condition = AccessCondition.GenerateIfExistsCondition();
}
if (!string.IsNullOrWhiteSpace(writeStreamLeaseId))
{
condition.LeaseId = writeStreamLeaseId;
}
// default the writer context if it is not already set
if (null == _writerContext)
{
_writerContext = WriteContext.DefaultWriterContext();
}
BlobBlockJsonWrappedEvent evtToWrite = BlobBlockJsonWrappedEvent.Create(eventName,
nextSequence,
eventVersionNumber,
null,
eventInstance,
_writerContext);
try
{
// Create it if it doesn't exist and initialsie the metadata
await base.Refresh();
Microsoft.Azure.Storage.OperationContext context = new Microsoft.Azure.Storage.OperationContext()
{
};
await EventStreamBlob.AppendBlockAsync(new System.IO.MemoryStream(Encoding.UTF8.GetBytes(evtToWrite.ToJSonText())),
"",
condition,
null, // use the default blob request options
context
);
}
catch (Microsoft.Azure.Storage.StorageException exBlob)
{
throw new EventStreamWriteException(this,
(nextSequence - 1),
message: "Failed to save an event to the event stream",
source: "Blob Event Stream Writer",
innerException: exBlob);
}
await IncrementSequence(writeStreamLeaseId);
if (!string.IsNullOrWhiteSpace(writeStreamLeaseId))
{
// and release the lease
await base.EventStreamBlob.ReleaseLeaseAsync(condition);
}
int sequence = await base.GetSequenceNumber();
return(new AppendResult((sequence == 0), sequence));
}
else
{
return(null);
}
}
// This shows how to delete all of the devices for the IoT Hub.
// First, export the list to devices.txt (ExportDevices).
// Next, read in that file. Each row is a serialized object;
// read them into the generic list serializedDevices.
// Delete the devices.txt in blob storage, because you're going to recreate it.
// For each serializedDevice, deserialize it, set ImportMode to Delete,
// reserialize it, and write it to a StringBuilder. The ImportMode field is what
// tells the job framework to delete each one.
// Write the new StringBuilder to the block blob.
// This essentially replaces the list with a list of devices that have ImportJob = Delete.
// Call ImportDevicesAsync, which will read in the list in devices.txt, then delete each one.
public static async Task DeleteAllDevicesFromHub(string hubConnectionString,
CloudBlobContainer cloudBlobContainer, string containerURI, string deviceListFile)
{
// Read the devices from the hub and write them to devices.txt in blob storage.
await ExportDevices(containerURI, hubConnectionString);
// Read devices.txt which contains serialized objects.
// Write each line to the serializedDevices list. (List<string>).
CloudBlockBlob blockBlob = cloudBlobContainer.GetBlockBlobReference(deviceListFile);
// Get the URI for the blob.
string blobURI = blockBlob.Uri.ToString();
// Instantiate the generic list.
var serializedDevices = new List <string>();
// Read the blob file of devices, import each row into serializedDevices.
using (var streamReader =
new StreamReader(await blockBlob.OpenReadAsync(AccessCondition.GenerateIfExistsCondition(),
null, null), Encoding.UTF8))
{
while (streamReader.Peek() != -1)
{
string line = await streamReader.ReadLineAsync();
serializedDevices.Add(line);
}
}
// Delete the blob containing the list of devices,
// because you're going to recreate it.
CloudBlockBlob blobToDelete = cloudBlobContainer.GetBlockBlobReference("devices.txt");
// Step 1: Update each device's ImportMode to be Delete
StringBuilder sb = new StringBuilder();
serializedDevices.ForEach(serializedDevice =>
{
// Deserialize back to an ExportImportDevice.
var device = JsonConvert.DeserializeObject <ExportImportDevice>(serializedDevice);
// Update the property.
device.ImportMode = ImportMode.Delete;
// Re-serialize the object now that you're updated the property.
sb.AppendLine(JsonConvert.SerializeObject(device));
});
// Step 2: Delete the blob if it already exists, then write the list in memory to the blob.
await blobToDelete.DeleteIfExistsAsync();
using (CloudBlobStream stream = await blobToDelete.OpenWriteAsync())
{
byte[] bytes = Encoding.UTF8.GetBytes(sb.ToString());
for (var i = 0; i < bytes.Length; i += 500)
{
int length = Math.Min(bytes.Length - i, 500);
await stream.WriteAsync(bytes, i, length);
}
}
// Step 3: Call import using the same blob to delete all devices.
// Loads devices.txt and applies that change.
RegistryManager registryManager =
RegistryManager.CreateFromConnectionString(hubConnectionString);
JobProperties importJob =
await registryManager.ImportDevicesAsync(containerURI, containerURI);
// Wait until job is finished
while (true)
{
importJob = await registryManager.GetJobAsync(importJob.JobId);
if (importJob.Status == JobStatus.Completed ||
importJob.Status == JobStatus.Failed ||
importJob.Status == JobStatus.Cancelled)
{
// Job has finished executing
break;
}
await Task.Delay(TimeSpan.FromSeconds(5));
}
}
public override async Task DeleteTempFileChunkAsync(string identifier)
{
var chunks = GetDocuments(TEMP_PREFIX + identifier).ToArray();
foreach (var chunk in chunks)
{
if (await chunk.DeleteIfExistsAsync(DeleteSnapshotsOption.IncludeSnapshots, AccessCondition.GenerateIfExistsCondition(), null, null))
{
_logger.LogDebug("Successfully deleted the file chunk {0} from Azure Blob Storage.", chunk.Name);
}
}
}
// This shows how to copy devices from one IoT Hub to another.
// First, export the list from the Source hut to devices.txt (ExportDevices).
// Next, read in that file. Each row is a serialized object;
// read them into the generic list serializedDevices.
// Delete the devices.txt in blob storage, because you're going to recreate it.
// For each serializedDevice, deserialize it, set ImportMode to CREATE,
// reserialize it, and write it to a StringBuilder. The ImportMode field is what
// tells the job framework to add each device.
// Write the new StringBuilder to the block blob.
// This essentially replaces the list with a list of devices that have ImportJob = Delete.
// Call ImportDevicesAsync, which will read in the list in devices.txt, then add each one
// because it doesn't already exist. If it already exists, it will write an entry to
// the import error log and not add the new one.
private async Task CopyAllDevicesToNewHub(string sourceHubConnectionString, string destHubConnectionString, string containerUri, string deviceListFile)
{
Console.WriteLine("Exporting devices on current hub");
// Read the devices from the hub and write them to devices.txt in blob storage.
await ExportDevices(containerUri, sourceHubConnectionString).ConfigureAwait(false);
// Read devices.txt which contains serialized objects.
// Write each line to the serializedDevices list. (List<string>).
CloudBlockBlob blockBlob = _cloudBlobContainer.GetBlockBlobReference(deviceListFile);
// Get the URI for the blob.
string blobUri = blockBlob.Uri.ToString();
// Instantiate the generic list.
var serializedDevices = new List <string>();
Console.WriteLine("Read in list of devices from blob storage.");
// Read the blob file of devices, import each row into serializedDevices.
using Stream blobStream = await blockBlob.OpenReadAsync(AccessCondition.GenerateIfExistsCondition(), null, null).ConfigureAwait(false);
using var streamReader = new StreamReader(blobStream, Encoding.UTF8);
while (streamReader.Peek() != -1)
{
string line = await streamReader.ReadLineAsync().ConfigureAwait(false);
serializedDevices.Add(line);
}
// Delete the blob containing the list of devices, because you're going to recreate it.
CloudBlockBlob blobToDelete = _cloudBlobContainer.GetBlockBlobReference("devices.txt");
Console.WriteLine("Update ImportMode to be Create.");
// Step 1: Update each device's ImportMode to be Create
var sb = new StringBuilder();
serializedDevices.ForEach(serializedDevice =>
{
// Deserialize back to an ExportImportDevice.
var device = JsonConvert.DeserializeObject <ExportImportDevice>(serializedDevice);
// Update the property.
device.ImportMode = ImportMode.Create;
// Re-serialize the object now that you're updated the property.
sb.AppendLine(JsonConvert.SerializeObject(device));
});
// Step 2: Delete the blob if it already exists, then write the list in memory to the blob.
await blobToDelete.DeleteIfExistsAsync().ConfigureAwait(false);
using CloudBlobStream stream = await blobToDelete.OpenWriteAsync().ConfigureAwait(false);
byte[] bytes = Encoding.UTF8.GetBytes(sb.ToString());
for (var i = 0; i < bytes.Length; i += 500)
{
int length = Math.Min(bytes.Length - i, 500);
await stream.WriteAsync(bytes, i, length).ConfigureAwait(false);
}
Console.WriteLine("Creating and running registry manager job to import the entries from the text file to the new hub");
// Step 3: Call import using the same blob to create all devices.
// Loads devices.txt and adds the devices to the destination hub.
using RegistryManager registryManager = RegistryManager.CreateFromConnectionString(destHubConnectionString);
JobProperties importJob = await registryManager.ImportDevicesAsync(containerUri, containerUri).ConfigureAwait(false);
// Wait until job is finished
while (true)
{
importJob = await registryManager.GetJobAsync(importJob.JobId).ConfigureAwait(false);
Console.WriteLine($"Import job status is {importJob.Status}");
if (importJob.Status == JobStatus.Completed ||
importJob.Status == JobStatus.Failed ||
importJob.Status == JobStatus.Cancelled)
{
// Job has finished executing
break;
}
await Task.Delay(TimeSpan.FromSeconds(5)).ConfigureAwait(false);
}
}
