} // End of the UploadFromStream method
public async Task <Stream> GetWriteStream(string container_name, string blob_name)
{
// Get a container reference
CloudBlobContainer container = GetContainerReference(container_name);
// Get a blob object
CloudBlockBlob blob = container.GetBlockBlobReference(blob_name);
blob.StreamWriteSizeInBytes = 500 * 1024;
//blockBlob.Properties.ContentType = contentType;
// Create a stream
CloudBlobStream stream = null;
try
{
stream = await blob.OpenWriteAsync();
}
catch (Exception ex)
{
// Log the exception
logger.LogError(ex, $"GetWriteStream: {blob_name}", null);
}
// Return a stream
return(stream);
} // End of the GetWriteStream method
/// <summary>
/// Process events
/// </summary>
/// <param name="context"></param>
/// <param name="messages"></param>
/// <returns></returns>
public Task ProcessEventsAsync(PartitionContext context, IEnumerable <EventData> messages)
{
long lastSeq = 0;
foreach (var eventData in messages)
{
var message = BinarySerializer.DeserializeStreamEventLong(eventData.Body.ToArray());
lastSeq = eventData.SystemProperties.SequenceNumber;
this.input.OnNext(message);
}
if (this.checkpointStopWatch.Elapsed > TimeSpan.FromSeconds(10))
{
Console.WriteLine("Taking checkpoint");
var storageAccount = CloudStorageAccount.Parse(StorageConnectionString);
var blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference("checkpoints");
var blockBlob = container.GetBlockBlobReference(context.PartitionId + "-" + lastSeq);
CloudBlobStream blobStream = blockBlob.OpenWriteAsync().GetAwaiter().GetResult();
this.queryProcess.Checkpoint(blobStream);
blobStream.Flush();
blobStream.Close();
return(context
.CheckpointAsync()
.ContinueWith(t => DeleteOlderCheckpoints(context.PartitionId + "-" + lastSeq)));
}
return(Task.CompletedTask);
}
public static async Task Run(
[BlobTrigger("summariseddocuments/{name}.{ext}")] Stream myBlob,
[Blob("summariseddocuments/{name}.{ext}.summary.txt", FileAccess.Write)] CloudBlobStream summaryBlob,
string name,
string ext,
TraceWriter log
)
{
// Because suffix filters don't work yet - this should take non-pdfs off the todo list
if (ext.ToLower() != "pdf")
{
return;
}
log.Info($"Text Processing beginning for {name} ({myBlob.Length} Bytes)");
log.Info($"Extracting text from the PDF");
var pages = iTextPDFHelper.GetPDFPages(myBlob, log, ocrImages: true);
log.Info($"Calling Text Analytics to determine key phrases");
Dictionary <string, int> keyPhrases = await TextAnalyticsHelper.GetKeyPhrases(pages, log);
var topPhrases = keyPhrases.OrderByDescending(pair => pair.Value).Take(20).ToList();
log.Info($"Building summary");
string summary = TextAnalyticsHelper.BuildSummary(pages, topPhrases);
log.Info($"Saving summary to new blob");
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(summary), false))
{
stream.CopyTo(summaryBlob);
}
}
public override Task <CloudBlobStream> OpenWriteAsync(long?size, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext, CancellationToken cancellationToken)
{
CloudBlobStream stream = _store.OpenWritePage(_containerName, _blobName, size, _metadata);
return(Task.FromResult(stream));
// return base.OpenWriteAsync(size, accessCondition, options, operationContext, cancellationToken);
}
static void Main(string[] args)
{
string match_val = null;
using (FileStream fileStream = File.Open(@"D:\User\Pictures\test.png", FileMode.Open))
using (MemoryStream memoryStream = new MemoryStream()){
fileStream.CopyTo(memoryStream);
match_val = Convert.ToBase64String(memoryStream.ToArray());
}
StorageCredentials storageCredentials = new StorageCredentials(storageAccountName, storageAccountKey);
CloudStorageAccount cloudStorageAccount = new CloudStorageAccount(storageCredentials, true);
CloudBlobClient cloudBlobClient = cloudStorageAccount.CreateCloudBlobClient();
CloudBlobContainer cloudBlobContainer = cloudBlobClient.GetContainerReference("pub"); // For convenience, it is a public container, so I do not need to set access permission
cloudBlobContainer.CreateIfNotExists();
CloudBlockBlob cloudBlockBlob = cloudBlobContainer.GetBlockBlobReference("test2.png");
using (MemoryStream memoryStream = new MemoryStream(Convert.FromBase64String(match_val)))
using (Image image = Image.FromStream(memoryStream, true, true))
using (CloudBlobStream cloudBlobStream = cloudBlockBlob.OpenWrite()){
image.Save(cloudBlobStream, System.Drawing.Imaging.ImageFormat.Png);
}
Console.WriteLine(cloudBlockBlob.Uri.AbsoluteUri);
Console.ReadLine();
}
public CloudBlobStream GetStream(string fileId, out string blobReference)
{
blobReference = fileId + ".wav";
var blob = Container.GetBlockBlobReference(blobReference);
CloudBlobStream stream = blob.OpenWrite();
return(stream);
}
public async Task <string> UploadOpportunityImageAsync(IFormFile imageFile, string imageName)
{
CloudStorageAccount storageAccount = new CloudStorageAccount(
new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials(
StorageName, StorageApiKey), true);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer cloudBlobContainer = blobClient.GetContainerReference(ImageContainer);
CloudBlockBlob blob = cloudBlobContainer.GetBlockBlobReference(imageName.Replace(" ", String.Empty));
await blob.DeleteIfExistsAsync();
using (MemoryStream memoryStream = new MemoryStream())
{
using (var image = Image.FromStream(imageFile.OpenReadStream(), true, true))
{
var imageWidth = ImageFixedWidth;
var imageHeight = ImageFixedHeight;
var widthOffset = 0;
var heightOffset = 0;
if ((double)image.Width / image.Height > ImageAspectRatio)
{
imageHeight = (int)Math.Round(image.Height * (double)ImageFixedWidth / image.Width);
heightOffset = (int)Math.Abs((ImageFixedHeight - imageHeight) / 2);
}
else
{
imageWidth = (int)Math.Round(image.Width * (double)ImageFixedHeight / image.Height);
widthOffset = (int)Math.Abs((ImageFixedWidth - imageWidth) / 2);
}
using var newImage = new Bitmap(ImageFixedWidth, ImageFixedHeight);
using (var graphics = Graphics.FromImage(newImage))
{
graphics.DrawImage(
image,
widthOffset, heightOffset,
imageWidth, imageHeight
);
}
newImage.Save(memoryStream, System.Drawing.Imaging.ImageFormat.Png);
}
CloudBlobStream blobStream = blob.OpenWriteAsync().Result;
memoryStream.Position = 0;
await memoryStream.CopyToAsync(blobStream);
await blobStream.CommitAsync();
}
return(blob.Uri.ToString());
}
public override Task UploadTextAsync(string content, Encoding encoding, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext, CancellationToken cancellationToken)
{
using (CloudBlobStream stream = _store.OpenWriteBlock(_containerName, _blobName, _metadata))
{
byte[] buffer = Encoding.UTF8.GetBytes(content);
stream.Write(buffer, 0, buffer.Length);
stream.CommitAsync().Wait();
}
return(Task.FromResult(0));
}
public Task UploadTextAsync(string content, Encoding encoding = null, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken))
{
using (CloudBlobStream stream = _store.OpenWriteAppend(_containerName, _blobName, _metadata))
{
byte[] buffer = Encoding.UTF8.GetBytes(content);
stream.Write(buffer, 0, buffer.Length);
stream.Commit();
}
return(Task.FromResult(0));
}
public static async Task UploadEmptyPageAsync(this CloudPageBlob blob)
{
if (blob == null)
{
throw new ArgumentNullException("blob");
}
using (CloudBlobStream stream = await blob.OpenWriteAsync(512))
{
await stream.CommitAsync();
}
}
public static void UploadEmptyPage(this IStoragePageBlob blob)
{
if (blob == null)
{
throw new ArgumentNullException("blob");
}
using (CloudBlobStream stream = blob.OpenWriteAsync(512, CancellationToken.None).GetAwaiter().GetResult())
{
stream.CommitAsync().Wait();
}
}
public string CreateBlob(List <Tone> tones, string fileId, int baseSamplesPerPixel = 88200, bool synth = true)
{
UseSynth = synth;
var samples = GetTotalSamples(baseSamplesPerPixel, tones);
string blobReference;
CloudBlobStream stream = _repo.GetStream(fileId, out blobReference);
WriteWaves(tones, baseSamplesPerPixel, stream, samples);
stream.Close();
return(blobReference);
}
public CloudBlobStream GetWriteBlobStream(string path)
{
CloudBlockBlob blob = GetCloudBlob(path);
if (blob == null)
{
return(null);
}
CloudBlobStream stream = blob.OpenWrite();
return(stream);
}
/// <summary>
/// Process events
/// </summary>
/// <param name="context"></param>
/// <param name="messages"></param>
/// <returns></returns>
public Task ProcessEventsAsync(PartitionContext context, IEnumerable <EventData> messages)
{
EventProcessor.__count += messages.Count();
if (DateTime.UtcNow > LastTime.AddSeconds(2))
{
LastTime = DateTime.UtcNow;
Console.Title = $"total: {EventProcessor.__count} offset: {context.ToString()} count: {messages.Count()}";
}
#if true
context.CheckpointAsync();
#else
long lastSeq = 0;
foreach (var eventData in messages)
{
var message = BinarySerializer.DeserializeStreamEventSampleEvent <Measure>(eventData.Body.ToArray());
lastSeq = eventData.SystemProperties.SequenceNumber;
Task.Delay(5).Wait();
this.input.OnNext(message);
//else if (message.Payload.Data is MeasureT3)
//{
// var serializer = Newtonsoft.Json.JsonSerializer.Create();
// var value = JsonConvert.DeserializeObject<StreamEvent<Measure<MeasureT3>>>(JsonConvert.SerializeObject(message));
// this.input.OnNext(value);
//}
//else if (message.Payload.Data is MeasureT4)
// this.input.OnNext((StreamEvent<Measure>)(object)(StreamEvent<Measure<MeasureT4>>)(object)message);
}
if (this.checkpointStopWatch.Elapsed > TimeSpan.FromSeconds(10))
{
Console.WriteLine("Taking checkpoint");
var storageAccount = CloudStorageAccount.Parse(StorageConnectionString);
var blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference("checkpoints");
var blockBlob = container.GetBlockBlobReference(context.PartitionId + "-" + lastSeq);
CloudBlobStream blobStream = blockBlob.OpenWriteAsync().GetAwaiter().GetResult();
this.queryProcess.Checkpoint(blobStream);
blobStream.Flush();
blobStream.Close();
return(context
.CheckpointAsync()
.ContinueWith(t => DeleteOlderCheckpoints(context.PartitionId + "-" + lastSeq)));
}
#endif
return(Task.CompletedTask);
}
private static void WriteLog(string manifestName, string logData, ILogger logger, bool createNew)
{
string containerName = Constant.Storage.BlobContainer.MediaServices;
string logName = manifestName.Replace(Constant.Media.IngestManifest.FileExtension, Constant.Media.IngestManifest.FileExtensionLog);
CloudAppendBlob logBlob = _blobClient.GetAppendBlob(containerName, logName);
CloudBlobStream logStream = logBlob.OpenWriteAsync(createNew).Result;
using (StreamWriter logWriter = new StreamWriter(logStream))
{
logWriter.WriteLine(logData);
}
if (logger != null)
{
logger.LogInformation(logData);
}
}
private void WriteLocalDataToCloudIfNeeded()
{
lock (this.syncRoot)
{
byte[] localContentHash;
if (!Serialization.ModifiedSince(
this.localObject,
this.lastKnownBlobContentsHash,
out localContentHash))
{
return;
}
AccessCondition writeAccessConditionAtStartOfUpload = new AccessCondition();
lock (this.writeAccessCondition)
{
writeAccessConditionAtStartOfUpload.LeaseId = this.writeAccessCondition.LeaseId;
}
StorageOperation.Try(
() =>
{
OperationContext context = new OperationContext();
CloudBlobStream uploadStream = this.backingBlob.OpenWrite(
accessCondition: writeAccessConditionAtStartOfUpload,
operationContext: context);
Serialization.SerializeIntoStream(this.localObject, uploadStream);
uploadStream.Close();
this.readAccessCondition.IfNoneMatchETag = context.LastResult.Etag;
this.lastKnownBlobContentsHash = localContentHash;
},
catchHttp400: e =>
{
// Can happen when two clients race to perform an upload to the same blob; one
// client will get a HTTP 400 when committing their list of uploaded blocks.
OnWriteToCloudFailure(e, writeAccessConditionAtStartOfUpload);
},
catchHttp412: e =>
{
// There is a lease in place that prevents this write.
OnWriteToCloudFailure(e, writeAccessConditionAtStartOfUpload);
});
}
}
public static async Task <WatchableCloudBlobStream> BindStreamAsync(ICloudBlob blob,
ValueBindingContext context, IBlobWrittenWatcher blobWrittenWatcher)
{
var blockBlob = blob as CloudBlockBlob;
if (blockBlob == null)
{
throw new InvalidOperationException("Cannot bind to page or append blobs using 'out string', 'TextWriter', or writable 'Stream' parameters.");
}
BlobCausalityManager.SetWriter(blob.Metadata, context.FunctionInstanceId);
CloudBlobStream rawStream = await blockBlob.OpenWriteAsync(context.CancellationToken);
IBlobCommitedAction committedAction = new BlobCommittedAction(blob, blobWrittenWatcher);
return(new WatchableCloudBlobStream(rawStream, committedAction));
}
public async Task TimeSeriesDailyAdjustedLoad(TimeSeriesDailyAdjustedResponse timeSeriesDaily)
{
// get a reference to the blob
string blobName = timeSeriesDaily.Ticker;
CloudBlockBlob cloudBlob = blobContainer.GetBlockBlobReference(blobName);
// generate the blob
CloudBlobStream stream = await cloudBlob.OpenWriteAsync();
// serialize the quotes
byte[] vs = timeSeriesDaily.Serialize();
// write the Quotes into a blob
await stream.WriteAsync(vs, 0, vs.Length);
// commit the changes to azure storage
await stream.CommitAsync();
}
public static async Task <WatchableCloudBlobStream> BindStreamAsync(IStorageBlob blob,
ValueBindingContext context, IBlobWrittenWatcher blobWrittenWatcher)
{
IStorageBlockBlob blockBlob = blob as IStorageBlockBlob;
if (blockBlob == null)
{
throw new InvalidOperationException("Cannot bind a page blob using an out string.");
}
BlobCausalityManager.SetWriter(blob.Metadata, context.FunctionInstanceId);
CloudBlobStream rawStream = await blockBlob.OpenWriteAsync(context.CancellationToken);
IBlobCommitedAction committedAction = new BlobCommittedAction(blob, blobWrittenWatcher);
return(new WatchableCloudBlobStream(rawStream, committedAction));
}
private static async Task uploadBackup(CloudBlockBlob blob)
{
string graphDBBackupUrl = $"{dataAPI}/repositories/Master/statements?infer=false&context=null";
using (HttpClient client = new HttpClient())
{
client.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", subscriptionKey);
client.DefaultRequestHeaders.Add("Api-Version", apiVersion);
client.DefaultRequestHeaders.Accept.Add(new System.Net.Http.Headers.MediaTypeWithQualityHeaderValue("application/x-trig"));
using (HttpResponseMessage response = await client.GetAsync(graphDBBackupUrl, HttpCompletionOption.ResponseHeadersRead))
{
using (CloudBlobStream backupFileStream = blob.OpenWrite())
{
await response.Content.CopyToAsync(backupFileStream);
}
}
}
}
/// <summary>
/// Define a sharded vertex type and register with CRA.
/// </summary>
/// <param name="vertexDefinition">Name of the vertex type</param>
/// <param name="creator">Lambda that describes how to instantiate the vertex, taking in an object as parameter</param>
public CRAErrorCode DefineVertex(string vertexDefinition, Expression <Func <IShardedVertex> > creator)
{
CloudBlobContainer container = _blobClient.GetContainerReference("cra");
container.CreateIfNotExistsAsync().Wait();
var blockBlob = container.GetBlockBlobReference(vertexDefinition + "/binaries");
CloudBlobStream blobStream = blockBlob.OpenWriteAsync().GetAwaiter().GetResult();
AssemblyUtils.WriteAssembliesToStream(blobStream);
blobStream.Close();
// Add metadata
var newRow = new VertexTable("", vertexDefinition, vertexDefinition, "", 0, creator, null, true);
TableOperation insertOperation = TableOperation.InsertOrReplace(newRow);
_vertexTable.ExecuteAsync(insertOperation).Wait();
return(CRAErrorCode.Success);
}
public static void Run([EventGridTrigger] EventGridEvent eventGridEvent,
[Blob("{data.fileUrl}", FileAccess.Read, Connection = "feedstore_STORAGE")] CloudBlockBlob inputBlob,
[Blob("parquet/{rand-guid}.parquet", FileAccess.Write, Connection = "feedstore_STORAGE")] CloudBlockBlob outputBlob,
TraceWriter log)
{
log.Info(eventGridEvent.ToString());
JObject dataObject = eventGridEvent.Data as JObject;
var eventData = dataObject.ToObject <EventHubCaptureFileCreatedEventData>();
log.Info(eventData.EventCount.ToString() + " events in record");
if (eventData.EventCount > 0)
{
log.Info("Got a real file");
Stream avroStream = inputBlob.OpenReadAsync().Result;
CloudBlobStream parquetStream = outputBlob.OpenWriteAsync().Result;
CreateParquetFile(avroStream, parquetStream);
parquetStream.Close();
}
}
public override Stream GetStream(HttpContent parent, HttpContentHeaders headers)
{
if (parent == null)
{
throw new ArgumentNullException(nameof(parent));
}
if (headers == null)
{
throw new ArgumentNullException(nameof(headers));
}
if (!_supportedMimeTypes.Contains(headers.ContentType.ToString().ToLower()))
{
throw new NotSupportedException("Only jpeg and png are supported");
}
var fileName = UnquoteString(headers.ContentDisposition.FileName);
_pngExtension = ImageFormat.Png;
var fileNameExtension = "." + _pngExtension;
if (!IsNullOrWhiteSpace(fileName))
{
fileNameExtension = Path.GetExtension(fileName);
}
if (IsNullOrWhiteSpace(_filename))
{
throw new FileNotFoundException("This request is not properly formatted.");
}
_filename = _filename + fileNameExtension;
var storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting(_connectionStringName));
var imagesBlobClient = storageAccount.CreateCloudBlobClient();
_container = imagesBlobClient.GetContainerReference(_containerName);
_blockBlob = _container.GetBlockBlobReference(_filename);
_stream = _blockBlob.OpenWriteAsync().Result;
return(_stream);
}
public async Task <string> UploadImageAsync(IFormFile image, string imageName)
{
CloudStorageAccount storageAccount = new CloudStorageAccount(
new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials(
StorageName, StorageApiKey), true);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer cloudBlobContainer = blobClient.GetContainerReference(ImageContainer);
CloudBlockBlob blob = cloudBlobContainer.GetBlockBlobReference(imageName.Replace(" ", String.Empty));
await blob.DeleteIfExistsAsync();
CloudBlobStream blobStream = blob.OpenWriteAsync().Result;
await image.CopyToAsync(blobStream);
await blobStream.CommitAsync();
return(blob.Uri.ToString());
}
/// <summary>
/// Returns true if the file was written.
/// Returns false if the in-process lock failed. Throws an exception if any kind of file or processing exception occurs.
/// </summary>
/// <param name="result"></param>
/// <param name="path"></param>
/// <param name="writeCallback"></param>
/// <param name="timeoutMs"></param>
/// <param name="recheckFS"></param>
/// <returns></returns>
private bool TryWriteFile(CacheResult result, string path, ResizeImageDelegate writeCallback, int timeoutMs, bool recheckFS)
{
bool miss = true;
if (recheckFS)
{
miss = !Index.PathExistInIndex(path);
if (!miss && !Locks.MayBeLocked(path.ToUpperInvariant()))
{
return(true);
}
}
//Lock execution using relativePath as the sync basis. Ignore casing differences. This locking is process-local, but we also have code to handle file locking.
return(Locks.TryExecute(path.ToUpperInvariant(), timeoutMs,
delegate() {
//On the second check, use cached data for speed. The cached data should be updated if another thread updated a file (but not if another process did).
if (!Index.PathExistInIndex(path))
{
//Open stream
//Catch IOException, and if it is a file lock,
// - (and hashmodified is true), then it's another process writing to the file, and we can serve the file afterwards
// - (and hashmodified is false), then it could either be an IIS read lock or another process writing to the file. Correct behavior is to kill the request here, as we can't guarantee accurate image data.
// I.e, hashmodified=true is the only supported setting for multi-process environments.
//TODO: Catch UnathorizedAccessException and log issue about file permissions.
//... If we can wait for a read handle for a specified timeout.
try
{
var blobRef = container.GetBlockBlobReference(path);
// Only write to the blob if it does not exist
if (!blobRef.Exists())
{
CloudBlobStream cloudBlobStream = blobRef.OpenWrite();
bool finished = false;
try
{
using (cloudBlobStream)
{
//Run callback to write the cached data
writeCallback(cloudBlobStream); //Can throw any number of exceptions.
cloudBlobStream.Flush();
finished = true;
}
}
finally
{
//Don't leave half-written files around.
if (!finished)
{
try
{
blobRef.Delete();
}
catch { }
}
}
DateTime createdUtc = DateTime.UtcNow;
}
//Update index
Index.AddCachedFileToIndex(path);
//This was a cache miss
if (result != null)
{
result.Result = CacheQueryResult.Miss;
}
}
catch (IOException ex)
{
//Somehow in between verifying the file didn't exist and trying to create it, the file was created and locked by someone else.
//When hashModifiedDate==true, we don't care what the file contains, we just want it to exist. If the file is available for
//reading within timeoutMs, simply do nothing and let the file be returned as a hit.
Stopwatch waitForFile = new Stopwatch();
bool opened = false;
while (!opened && waitForFile.ElapsedMilliseconds < timeoutMs)
{
waitForFile.Start();
try
{
using (var stream = container.GetBlockBlobReference(path).OpenRead())
opened = true;
}
catch (IOException iex)
{
Thread.Sleep((int)Math.Min(30, Math.Round((float)timeoutMs / 3.0)));
}
waitForFile.Stop();
}
if (!opened)
{
throw; //By not throwing an exception, it is considered a hit by the rest of the code.
}
}
}
}));
}
public Task <CloudBlobStream> OpenWriteAsync(CancellationToken cancellationToken)
{
CloudBlobStream stream = _store.OpenWriteBlock(_containerName, _blobName, _metadata);
return(Task.FromResult(stream));
}
public WatchableCloudBlobStream(CloudBlobStream inner, IBlobCommitedAction committedAction)
: base(inner)
{
_committedAction = committedAction;
}
public AzureIndexOutput(CloudBlockBlob blob)
{
_stream = blob.OpenWrite();
_crc = new CRC32();
}
// This shows how to copy devices from one IoT Hub to another.
// First, export the list from the Source hut to devices.txt (ExportDevices).
// Next, read in that file. Each row is a serialized object;
// read them into the generic list serializedDevices.
// Delete the devices.txt in blob storage, because you're going to recreate it.
// For each serializedDevice, deserialize it, set ImportMode to CREATE,
// reserialize it, and write it to a StringBuilder. The ImportMode field is what
// tells the job framework to add each device.
// Write the new StringBuilder to the block blob.
// This essentially replaces the list with a list of devices that have ImportJob = Delete.
// Call ImportDevicesAsync, which will read in the list in devices.txt, then add each one
// because it doesn't already exist. If it already exists, it will write an entry to
// the import error log and not add the new one.
private async Task CopyAllDevicesToNewHub(string sourceHubConnectionString, string destHubConnectionString, string containerUri, string deviceListFile)
{
Console.WriteLine("Exporting devices on current hub");
// Read the devices from the hub and write them to devices.txt in blob storage.
await ExportDevices(containerUri, sourceHubConnectionString).ConfigureAwait(false);
// Read devices.txt which contains serialized objects.
// Write each line to the serializedDevices list. (List<string>).
CloudBlockBlob blockBlob = _cloudBlobContainer.GetBlockBlobReference(deviceListFile);
// Get the URI for the blob.
string blobUri = blockBlob.Uri.ToString();
// Instantiate the generic list.
var serializedDevices = new List <string>();
Console.WriteLine("Read in list of devices from blob storage.");
// Read the blob file of devices, import each row into serializedDevices.
using Stream blobStream = await blockBlob.OpenReadAsync(AccessCondition.GenerateIfExistsCondition(), null, null).ConfigureAwait(false);
using var streamReader = new StreamReader(blobStream, Encoding.UTF8);
while (streamReader.Peek() != -1)
{
string line = await streamReader.ReadLineAsync().ConfigureAwait(false);
serializedDevices.Add(line);
}
// Delete the blob containing the list of devices, because you're going to recreate it.
CloudBlockBlob blobToDelete = _cloudBlobContainer.GetBlockBlobReference("devices.txt");
Console.WriteLine("Update ImportMode to be Create.");
// Step 1: Update each device's ImportMode to be Create
var sb = new StringBuilder();
serializedDevices.ForEach(serializedDevice =>
{
// Deserialize back to an ExportImportDevice.
var device = JsonConvert.DeserializeObject <ExportImportDevice>(serializedDevice);
// Update the property.
device.ImportMode = ImportMode.Create;
// Re-serialize the object now that you're updated the property.
sb.AppendLine(JsonConvert.SerializeObject(device));
});
// Step 2: Delete the blob if it already exists, then write the list in memory to the blob.
await blobToDelete.DeleteIfExistsAsync().ConfigureAwait(false);
using CloudBlobStream stream = await blobToDelete.OpenWriteAsync().ConfigureAwait(false);
byte[] bytes = Encoding.UTF8.GetBytes(sb.ToString());
for (var i = 0; i < bytes.Length; i += 500)
{
int length = Math.Min(bytes.Length - i, 500);
await stream.WriteAsync(bytes, i, length).ConfigureAwait(false);
}
Console.WriteLine("Creating and running registry manager job to import the entries from the text file to the new hub");
// Step 3: Call import using the same blob to create all devices.
// Loads devices.txt and adds the devices to the destination hub.
using RegistryManager registryManager = RegistryManager.CreateFromConnectionString(destHubConnectionString);
JobProperties importJob = await registryManager.ImportDevicesAsync(containerUri, containerUri).ConfigureAwait(false);
// Wait until job is finished
while (true)
{
importJob = await registryManager.GetJobAsync(importJob.JobId).ConfigureAwait(false);
Console.WriteLine($"Import job status is {importJob.Status}");
if (importJob.Status == JobStatus.Completed ||
importJob.Status == JobStatus.Failed ||
importJob.Status == JobStatus.Cancelled)
{
// Job has finished executing
break;
}
await Task.Delay(TimeSpan.FromSeconds(5)).ConfigureAwait(false);
}
}
/// <summary>c
/// Generate NumToAdd devices and add them to the hub.
/// To do this, generate each identity.
/// Include authentication keys.
/// Write the device info to a block blob.
/// Import the devices into the identity registry by calling the import job.
/// </summary>
private async Task GenerateAndAddDevices(string hubConnectionString, string containerURI, int NumToAdd, string devicesToAdd)
{
int interimProgressCount = 0;
int displayProgressCount = 1000;
int totalProgressCount = 0;
//generate reference for list of new devices you're going to add, will write list to this blob
CloudBlockBlob generatedListBlob = _cloudBlobContainer.GetBlockBlobReference(devicesToAdd);
// define serializedDevices as a generic list<string>
List <string> serializedDevices = new List <string>();
for (var i = 1; i <= NumToAdd; i++)
{
// Create device name with this format: Hub_00000000 + a new guid.
// This should be large enough to display the largest number (1 million).
//string deviceName = "Hub_" + i.ToString("D8") + "-" + Guid.NewGuid().ToString();
string deviceName = $"Hub_{i.ToString("D8")}-{Guid.NewGuid().ToString()}";
Debug.Print($"device = '{deviceName}'\n");
// Create a new ExportImportDevice.
// CryptoKeyGenerator is in the Microsoft.Azure.Devices.Common namespace.
var deviceToAdd = new ExportImportDevice()
{
Id = deviceName,
Status = DeviceStatus.Enabled,
Authentication = new AuthenticationMechanism
{
SymmetricKey = new SymmetricKey
{
PrimaryKey = CryptoKeyGenerator.GenerateKey(32),
SecondaryKey = CryptoKeyGenerator.GenerateKey(32)
}
},
// This indicates that the entry should be added as a new device.
ImportMode = ImportMode.Create
};
// Add device to the list as a serialized object.
serializedDevices.Add(JsonConvert.SerializeObject(deviceToAdd));
// Not real progress as you write the new devices, but will at least show *some* progress.
interimProgressCount++;
totalProgressCount++;
if (interimProgressCount >= displayProgressCount)
{
Console.WriteLine("Added {0} messages.", totalProgressCount);
interimProgressCount = 0;
}
}
// Now have a list of devices to be added, each one has been serialized.
// Write the list to the blob.
var sb = new StringBuilder();
serializedDevices.ForEach(serializedDevice => sb.AppendLine(serializedDevice));
// Before writing the new file, make sure there's not already one there.
await generatedListBlob.DeleteIfExistsAsync().ConfigureAwait(false);
// Write list of serialized objects to the blob.
using (CloudBlobStream stream = await generatedListBlob.OpenWriteAsync().ConfigureAwait(false))
{
byte[] bytes = Encoding.UTF8.GetBytes(sb.ToString());
for (var i = 0; i < bytes.Length; i += 500)
{
int length = Math.Min(bytes.Length - i, 500);
await stream.WriteAsync(bytes, i, length).ConfigureAwait(false);
}
}
Console.WriteLine("Creating and running registry manager job to write the new devices.");
// Should now have a file with all the new devices in it as serialized objects in blob storage.
// generatedListBlob has the list of devices to be added as serialized objects.
// Call import using the blob to add the new devices.
// Log information related to the job is written to the same container.
// This normally takes 1 minute per 100 devices (according to the docs).
// First, initiate an import job.
// This reads in the rows from the text file and writes them to IoT Devices.
// If you want to add devices from a file, you can create a file and use this to import it.
// They have to be in the exact right format.
JobProperties importJob = new JobProperties();
RegistryManager registryManager = RegistryManager.CreateFromConnectionString(hubConnectionString);
try
{
// First URL is the container to import from; the file must be called devices.txt
// Second URL points to the container to write errors to as a block blob.
// This lets you import the devices from any file name. Since we wrote the new
// devices to [devicesToAdd], need to read the list from there as well.
importJob = await registryManager.ImportDevicesAsync(containerURI, containerURI, devicesToAdd).ConfigureAwait(false);
// This will catch any errors if something bad happens to interrupt the job.
while (true)
{
importJob = await registryManager.GetJobAsync(importJob.JobId).ConfigureAwait(false);
if (importJob.Status == JobStatus.Completed ||
importJob.Status == JobStatus.Failed ||
importJob.Status == JobStatus.Cancelled)
{
// Job has finished executing
break;
}
await Task.Delay(TimeSpan.FromSeconds(5)).ConfigureAwait(false);
}
}
catch (Exception ex)
{
Debug.Print("exception message {0}", ex.Message);
}
}
