// This method gets called by the runtime. Use this method to add services to the container.
public void ConfigureServices(IServiceCollection services)
{
services.AddControllers(options =>
{
options.SuppressAsyncSuffixInActionNames = false;
options.Filters.Add <ExceptionFilter>();
})
.AddFluentValidation(opt =>
{
opt.RegisterValidatorsFromAssemblyContaining(typeof(Startup));
})
.AddJsonOptions(opts =>
{
opts.JsonSerializerOptions.Converters.Add(new JsonStringEnumConverter());
});
services.AddAuthentication(ApiKeyDefaults.AuthenticationScheme)
.AddApiKeyInHeaderOrQueryParams <ApiKeyProvider>(options =>
{
options.Realm = "Bunnings API";
options.KeyName = "X-API-KEY";
});
services.AddAuthorization(options =>
{
options.FallbackPolicy = new AuthorizationPolicyBuilder()
.RequireAuthenticatedUser()
.Build();
});
services.AddSingleton(provider =>
{
var blobContainerClient =
new BlobContainerClient(Configuration["ConnectionStrings:PeopleStore"], "people");
blobContainerClient.CreateIfNotExists();
return(blobContainerClient);
});
services.AddSingleton(provider =>
{
try
{
var storageAccount = CloudStorageAccount.Parse(Configuration["ConnectionStrings:PeopleStore"]);
var tableClient = storageAccount.CreateCloudTableClient();
Console.WriteLine("Create a Table for the demo");
// Create a table client for interacting with the table service
var table = tableClient.GetTableReference("people");
table.CreateIfNotExists();
return(tableClient);
}
catch (FormatException)
{
Console.WriteLine(
"Invalid storage account information provided. Please confirm the AccountName and AccountKey are valid in the app.config file - then restart the application.");
throw;
}
catch (ArgumentException)
{
Console.WriteLine(
"Invalid storage account information provided. Please confirm the AccountName and AccountKey are valid in the app.config file - then restart the sample.");
Console.ReadLine();
throw;
}
});
services.AddSingleton <IPeopleService, PeopleService>();
services.AddSwaggerGen(c =>
{
c.SwaggerDoc("v1", new OpenApiInfo
{
Version = "v1",
Title = "Bunnings API",
Description = "Bunnings API",
TermsOfService = new Uri("https://example.com/terms"),
Contact = new OpenApiContact
{
Name = "Dinuka Jayamuni",
Email = string.Empty,
Url = new Uri("https://www.linkedin.com/in/dinuka-jayamuni-82b8a13/"),
},
License = new OpenApiLicense
{
Name = "Use under LICX",
Url = new Uri("https://example.com/license"),
}
});
c.ExampleFilters();
c.AddFluentValidationRules();
var xmlFile = $"{Assembly.GetExecutingAssembly().GetName().Name}.xml";
var xmlPath = Path.Combine(AppContext.BaseDirectory, xmlFile);
c.IncludeXmlComments(xmlPath);
c.AddSecurityDefinition("ApiKey", new OpenApiSecurityScheme
{
Description = "Enter your Api Key below:",
Name = "X-API-KEY",
In = ParameterLocation.Header,
Type = SecuritySchemeType.ApiKey
});
c.AddSecurityRequirement(new OpenApiSecurityRequirement()
{
{
new OpenApiSecurityScheme
{
Reference = new OpenApiReference
{
Type = ReferenceType.SecurityScheme,
Id = "ApiKey"
},
},
new List <string>()
}
});
});
services.AddSwaggerExamplesFromAssemblies(Assembly.GetEntryAssembly());
}
/// <summary> /// Create a new <see cref="BlobBatchClient"/> object for the /// container associated with the <see cref="BlobContainerClient"/>. The new /// <see cref="BlobBatchClient"/> uses the same request policy pipeline /// as the <see cref="BlobContainerClient"/>. /// </summary> /// <param name="client">The <see cref="BlobContainerClient"/>.</param> /// <returns>A new <see cref="BlobBatchClient"/> instance.</returns> public static BlobBatchClient GetBlobBatchClient(this BlobContainerClient client) => new BlobBatchClient(client);
/// <summary>
/// Runs the sample using the specified Event Hubs and Azure storage connection information.
/// </summary>
///
/// <param name="eventHubsConnectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that the sample should run against.</param>
/// <param name="blobStorageConnectionString">The connection string for the storage account where checkpoints and state should be persisted.</param>
/// <param name="blobContainerName">The name of the blob storage container where checkpoints and state should be persisted.</param>
///
public async Task RunAsync(string eventHubsConnectionString,
string eventHubName,
string blobStorageConnectionString,
string blobContainerName)
{
// When the Event Processor client begins processing, it will take ownership over a set of Event Hub partitions to process. The ownership that a
// processor has of a partition is transient; ownership may be relinquished by the processor in several scenarios:
//
// - Because processors work collaboratively within the context of a consumer group, they will share responsibility for partitions of the Event Hub,
// with ownership of a partition potentially moving from one processor to another.
//
// - If a processor encounters an issue and believes that it cannot safely recover processing for a partition that it has ownership of, it will
// relinquish ownership and allow the partition to be claimed by another processor or may reclaim it with fresh state itself. This scenario will
// also result in errors being sent for processing by the handler.
//
// - Should there be a request for a processor to stop processing, it will relinquish ownership of its partitions to allow for other processors to
// potentially claim them so that processing continues as event processor instances are scaled for the consumer group.
//
// In any of these cases, the "PartitionClosing" event will be triggered on the processor. While there are no actions that you may take to influence
// the closing of a partition, this event serves as the logical partner of the "PartitionInitializing" event, allowing you to track the partitions owned
// by the processor.
string consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
BlobContainerClient storageClient = new BlobContainerClient(blobStorageConnectionString, blobContainerName);
EventProcessorClient processor = new EventProcessorClient(storageClient, consumerGroup, eventHubsConnectionString, eventHubName);
// For this example, we'll keep a list of partitions that are owned by our processor instance and keep it up to
// date with the last event processed for that partition. When our processor closes that partition, we'll ensure that we
// have an up-to-date checkpoint based on the last event.
int eventsProcessed = 0;
ConcurrentDictionary <string, ProcessEventArgs> ownedPartitions = new ConcurrentDictionary <string, ProcessEventArgs>();
// The handler for partition initialization is responsible for beginning to track the partition.
Task partitionInitializingHandler(PartitionInitializingEventArgs eventArgs)
{
if (eventArgs.CancellationToken.IsCancellationRequested)
{
return(Task.CompletedTask);
}
try
{
ownedPartitions[eventArgs.PartitionId] = default(ProcessEventArgs);
Console.WriteLine($"Initialized partition: { eventArgs.PartitionId }");
}
catch (Exception ex)
{
// For real-world scenarios, you should take action appropriate to your application. For our example, we'll just log
// the exception to the console.
Console.WriteLine();
Console.WriteLine($"An error was observed while initializing partition: { eventArgs.PartitionId }. Message: { ex.Message }");
Console.WriteLine();
}
return(Task.CompletedTask);
}
// The handler for partition close will stop tracking the partition and checkpoint if an event was processed for it.
async Task partitionClosingHandler(PartitionClosingEventArgs eventArgs)
{
if (eventArgs.CancellationToken.IsCancellationRequested)
{
return;
}
try
{
if (ownedPartitions.TryRemove(eventArgs.PartitionId, out ProcessEventArgs lastProcessEventArgs))
{
await lastProcessEventArgs.UpdateCheckpointAsync();
}
Console.WriteLine($"Closing partition: { eventArgs.PartitionId }");
}
catch (Exception ex)
{
// For real-world scenarios, you should take action appropriate to your application. For our example, we'll just log
// the exception to the console.
Console.WriteLine();
Console.WriteLine($"An error was observed while closing partition: { eventArgs.PartitionId }. Message: { ex.Message }");
Console.WriteLine();
}
}
// When an event is received, update the partition if tracked. In the case that the value changes in the
// time that it was checked, consider the other event fresher and do not force an update.
Task processEventHandler(ProcessEventArgs eventArgs)
{
if (eventArgs.CancellationToken.IsCancellationRequested)
{
return(Task.CompletedTask);
}
try
{
ownedPartitions[eventArgs.Partition.PartitionId] = eventArgs;
++eventsProcessed;
Console.WriteLine($"Event Received: { Encoding.UTF8.GetString(eventArgs.Data.Body.ToArray()) }");
}
catch (Exception ex)
{
// For real-world scenarios, you should take action appropriate to your application. For our example, we'll just log
// the exception to the console.
Console.WriteLine();
Console.WriteLine($"An error was observed while processing events. Message: { ex.Message }");
Console.WriteLine();
}
return(Task.CompletedTask);
};
// For this example, exceptions will just be logged to the console.
Task processErrorHandler(ProcessErrorEventArgs eventArgs)
{
if (eventArgs.CancellationToken.IsCancellationRequested)
{
return(Task.CompletedTask);
}
Console.WriteLine("===============================");
Console.WriteLine($"The error handler was invoked during the operation: { eventArgs.Operation ?? "Unknown" }, for Exception: { eventArgs.Exception.Message }");
Console.WriteLine("===============================");
Console.WriteLine();
return(Task.CompletedTask);
}
processor.PartitionInitializingAsync += partitionInitializingHandler;
processor.PartitionClosingAsync += partitionClosingHandler;
processor.ProcessEventAsync += processEventHandler;
processor.ProcessErrorAsync += processErrorHandler;
try
{
// To begin, we'll publish a batch of events for our processor to receive. Because we are not specifying any routing hints,
// the Event Hubs service will automatically route these to partitions. We'll split the events into a couple of batches to
// increase the chance they'll be spread around.
var expectedEvents = new List <EventData>()
{
new EventData(Encoding.UTF8.GetBytes("First Event, First Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, First Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, First Batch")),
new EventData(Encoding.UTF8.GetBytes("First Event, Second Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, Second Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, Second Batch")),
new EventData(Encoding.UTF8.GetBytes("First Event, Third Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, Third Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, Third Batch")),
};
int sentIndex = 0;
int numberOfBatches = 3;
int eventsPerBatch = (expectedEvents.Count / numberOfBatches);
await using (var producer = new EventHubProducerClient(eventHubsConnectionString, eventHubName))
{
while (sentIndex < expectedEvents.Count)
{
using EventDataBatch eventBatch = await producer.CreateBatchAsync();
for (int index = 0; index < eventsPerBatch; ++index)
{
eventBatch.TryAdd(expectedEvents[sentIndex]);
++sentIndex;
}
await producer.SendAsync(eventBatch);
}
}
// In order to begin processing, an explicit call must be made to the processor. This will instruct the processor to begin
// processing in the background, invoking your handlers when they are needed.
eventsProcessed = 0;
await processor.StartProcessingAsync();
// Because processing takes place in the background, we'll continue to wait until all of our events were
// read and handled before stopping. To ensure that we don't wait indefinitely should an unrecoverable
// error be encountered, we'll also add a timed cancellation.
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(60));
while ((!cancellationSource.IsCancellationRequested) && (eventsProcessed <= expectedEvents.Count))
{
await Task.Delay(TimeSpan.FromMilliseconds(250));
}
// Once we arrive at this point, either cancellation was requested or we have processed all of our events. In
// both cases, we'll want to shut down the processor.
await processor.StopProcessingAsync();
}
finally
{
// It is encouraged that you unregister your handlers when you have finished
// using the Event Processor to ensure proper cleanup. This is especially
// important when using lambda expressions or handlers in any form that may
// contain closure scopes or hold other references.
processor.PartitionInitializingAsync -= partitionInitializingHandler;
processor.PartitionClosingAsync -= partitionClosingHandler;
processor.ProcessEventAsync -= processEventHandler;
processor.ProcessErrorAsync -= processErrorHandler;
}
// The Event Processor client has been stopped and is not explicitly disposable; there
// is nothing further that we need to do for cleanup.
Console.WriteLine();
}
private static async Task RunAsyncProgram()
{
// Settings for Msal
var tenantId = "************************************";
var clientId = "************************************";
var clientSecret = "************************************";
// Setting for getting blob credential from Key Vault
var keyVault = "https://myvault.vault.azure.net/";
var secretForAccountKey = "************************************";
var secretForSas = "************************************";
// Setting for Blob
string blobEndpoint = "https://mystorage.blob.core.windows.net/";
string containerName = "************************************";
// Acuire Token
/* Use Managed ID
* Console.ForegroundColor = ConsoleColor.Yellow;
* Console.WriteLine("=== Getting Access Token using Managed Id ===");
* Console.ResetColor();
*
* HttpWebRequest httpWebRequest = (HttpWebRequest)WebRequest.Create("http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https://vault.azure.net");
* httpWebRequest.Headers["Metadata"] = "true";
* httpWebRequest.Method = "GET";
* HttpWebResponse httpWebResponse = (HttpWebResponse)httpWebRequest.GetResponse();
*
* StreamReader streamReader = new StreamReader(httpWebResponse.GetResponseStream());
* string stringResponse = streamReader.ReadToEnd();
* JObject jobjResponse = JObject.Parse(stringResponse);
* string accessToken = (string)jobjResponse["access_token"];
* Console.WriteLine(stringResponse);
*/
/* Use MSAL with client credential
*/
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("=== Getting Access Token using Msal ===");
Console.ResetColor();
IConfidentialClientApplication app;
app = ConfidentialClientApplicationBuilder.Create(clientId)
.WithClientSecret(clientSecret)
.WithAuthority("https://login.microsoftonline.com/" + tenantId)
.Build();
string[] scopes = new string[] { "https://vault.azure.net/.default" };
AuthenticationResult authenticationResult = await app.AcquireTokenForClient(scopes).ExecuteAsync();
var accessToken = authenticationResult.AccessToken;
Console.WriteLine(authenticationResult.AccessToken);
// Call KeyVault to get connection string
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("=== Get Access Key from KeyVault ===");
Console.ResetColor();
var httpClient = new HttpClient();
httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", accessToken);
var keyVaultUriForAccessKey = keyVault + "secrets/" + secretForAccountKey + "?api-version=7.0";
HttpResponseMessage httpResponseForAccountKey = await httpClient.GetAsync(keyVaultUriForAccessKey);
string jsonForAccountKey = await httpResponseForAccountKey.Content.ReadAsStringAsync();
Console.WriteLine(jsonForAccountKey);
// Upload file to Blob using BlobClient
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("=== Upload file to Blob using Access Key ===");
Console.ResetColor();
JObject jObjectForAccontKey = JObject.Parse(jsonForAccountKey);
string connectionString = (string)jObjectForAccontKey["value"];
BlobServiceClient blobServiceClient = new BlobServiceClient(connectionString);
BlobContainerClient blobContainerClient = blobServiceClient.GetBlobContainerClient(containerName);
string localPath = "./";
string fileName = "quickStart" + Guid.NewGuid().ToString() + ".txt";
string localFilePath = Path.Combine(localPath, fileName);
await File.WriteAllTextAsync(localFilePath, "Hello world");
var blobClient = blobContainerClient.GetBlobClient(fileName);
FileStream fileStream = File.OpenRead(localFilePath);
await blobClient.UploadAsync(fileStream);
fileStream.Close();
Console.WriteLine("Uploaded file to Blob storage as blob:\n\t {0}\n", blobClient.Uri);
// Call KeyVault to get Sas Token
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("=== Get Sas Token from KeyVault ===");
Console.ResetColor();
var keyVaultUriForSas = keyVault + "secrets/" + secretForSas + "?api-version=7.0";
HttpResponseMessage httpResponseForSas = await httpClient.GetAsync(keyVaultUriForSas);
string jsonForSas = await httpResponseForSas.Content.ReadAsStringAsync();
Console.WriteLine(jsonForSas);
// List files to Blob using SasClient
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("=== List blob files using Sas ===");
Console.ResetColor();
JObject jObjectForSas = JObject.Parse(jsonForSas);
string sasToken = (string)jObjectForSas["value"];
var accountSasCredential = new StorageCredentials(sasToken);
var accountWithSas = new CloudStorageAccount(accountSasCredential, new Uri(blobEndpoint), null, null, null);
var blobClientWithSas = accountWithSas.CreateCloudBlobClient();
var cloudBlobContainer = blobClientWithSas.GetContainerReference(containerName);
var listBlobItems = cloudBlobContainer.ListBlobs();
Console.WriteLine("Listing blobs.....");
foreach (var blobItem in listBlobItems)
{
Console.WriteLine("\t" + blobItem.Uri);
}
}
public DisposingContainer(BlobContainerClient client)
{
Container = client;
}
/// <summary>
/// Runs the sample using the specified Event Hubs and Azure storage connection information.
/// </summary>
///
/// <param name="eventHubsConnectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that the sample should run against.</param>
/// <param name="blobStorageConnectionString">The connection string for the storage account where checkpoints and state should be persisted.</param>
/// <param name="blobContainerName">The name of the blob storage container where checkpoints and state should be persisted.</param>
///
public async Task RunAsync(string eventHubsConnectionString,
string eventHubName,
string blobStorageConnectionString,
string blobContainerName)
{
// A checkpoint is the term used to describe a snapshot of the state of processing for a partition which has been persisted
// to durable storage and which allows an Event Processor client to resume processing at a specific location in the partition's
// event stream. When a processor starts, it will seek out an existing checkpoint and, if found, use that as the place where it
// begins reading events. If no checkpoint is found, a default location is used.
//
// Checkpoints are intended as a means to allow an Event Processor client or cluster of processors for a consumer group to coordinate
// on which events were processed, so that processors can dynamically start, stop, join the cluster, and leave the cluster without the
// need to start processing at the beginning of a partition and revisit all of its events.
//
// A checkpoint is based on an event and represents the last event that should be considered as processed for the partition; should a processor
// start with that checkpoint, the next available event would be used as the starting point.
//
// The creation of checkpoints comes at a cost, both in terms of processing performance/throughput and a potential monetary cost associated with
// the underlying storage resource. While it may seem desirable to create checkpoints for each event that is processed, that is typically considered
// an anti-pattern for most scenarios.
//
// When deciding how frequently to checkpoint, you'll need to consider the trade-off between the costs of creating the checkpoint against the costs of
// processing events. For scenarios where processing events is very cheap, it is often a better approach to checkpoint once per some number of events or
// once per time interval. For scenarios where processing events is more expensive, it may be a better approach to checkpoint more frequently.
//
// In either case, it is important to understand that your processing must be tolerant of receiving the same event to be processed more than once; the
// Event Hubs service, like most messaging platforms, guarantees at-least-once delivery. Even were you to create a checkpoint for each event that you
// process, it is entirely possible that you would receive that same event again from the service.
var expectedEvents = new List <EventData>()
{
new EventData(Encoding.UTF8.GetBytes("First Event, First Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, First Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, First Batch")),
new EventData(Encoding.UTF8.GetBytes("First Event, Second Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, Second Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, Second Batch")),
new EventData(Encoding.UTF8.GetBytes("First Event, Third Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, Third Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, Third Batch")),
};
// To begin, we'll publish a batch of events for our processor to receive. Because we are not specifying any routing hints,
// the Event Hubs service will automatically route these to partitions. We'll split the events into a couple of batches to
// increase the chance they'll be spread around.
int sentIndex = 0;
int numberOfBatches = 3;
int eventsPerBatch = (expectedEvents.Count / numberOfBatches);
await using (var producer = new EventHubProducerClient(eventHubsConnectionString, eventHubName))
{
while (sentIndex < expectedEvents.Count)
{
using EventDataBatch eventBatch = await producer.CreateBatchAsync();
for (int index = 0; index < eventsPerBatch; ++index)
{
eventBatch.TryAdd(expectedEvents[sentIndex]);
++sentIndex;
}
await producer.SendAsync(eventBatch);
}
}
// With our events having been published, we'll create an Event Hub Processor to read them.
string consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
BlobContainerClient storageClient = new BlobContainerClient(blobStorageConnectionString, blobContainerName);
EventProcessorClient processor = new EventProcessorClient(storageClient, consumerGroup, eventHubsConnectionString, eventHubName);
// When your handler for processing events is invoked, the set of event arguments that it are passed allow the associated event to be
// saved as a checkpoint for the processor. For this example, our handler will create a checkpoint once per batch of events that we
// sent.
int eventIndex = 0;
int eventsSinceLastCheckpoint = 0;
async Task processEventHandler(ProcessEventArgs eventArgs)
{
if (eventArgs.CancellationToken.IsCancellationRequested)
{
return;
}
try
{
++eventIndex;
++eventsSinceLastCheckpoint;
if (eventsSinceLastCheckpoint >= eventsPerBatch)
{
// Updating the checkpoint will interact with the Azure Storage. As a service call,
// this is done asynchronously and may be long-running. You may want to influence its behavior,
// such as limiting the time that it may execute in order to ensure throughput for
// processing events.
//
// In our case, we'll limit the checkpoint operation to a second and request cancellation
// if it runs longer.
using CancellationTokenSource cancellationSource = new CancellationTokenSource(TimeSpan.FromSeconds(1));
try
{
await eventArgs.UpdateCheckpointAsync(cancellationSource.Token);
eventsSinceLastCheckpoint = 0;
Console.WriteLine("Created checkpoint");
}
catch (TaskCanceledException)
{
Console.WriteLine("Checkpoint creation took too long and was canceled.");
}
Console.WriteLine();
}
Console.WriteLine($"Event Received: { Encoding.UTF8.GetString(eventArgs.Data.EventBody.ToBytes().ToArray()) }");
}
catch (Exception ex)
{
// For real-world scenarios, you should take action appropriate to your application. For our example, we'll just log
// the exception to the console.
Console.WriteLine();
Console.WriteLine($"An error was observed while processing events. Message: { ex.Message }");
Console.WriteLine();
}
};
// For this example, exceptions will just be logged to the console.
Task processErrorHandler(ProcessErrorEventArgs eventArgs)
{
if (eventArgs.CancellationToken.IsCancellationRequested)
{
return(Task.CompletedTask);
}
Console.WriteLine("===============================");
Console.WriteLine($"The error handler was invoked during the operation: { eventArgs.Operation ?? "Unknown" }, for Exception: { eventArgs.Exception.Message }");
Console.WriteLine("===============================");
Console.WriteLine();
return(Task.CompletedTask);
}
processor.ProcessEventAsync += processEventHandler;
processor.ProcessErrorAsync += processErrorHandler;
try
{
// In order to begin processing, an explicit call must be made to the processor. This will instruct the processor to begin
// processing in the background, invoking your handlers when they are needed.
eventIndex = 0;
await processor.StartProcessingAsync();
// Because processing takes place in the background, we'll continue to wait until all of our events were
// read and handled before stopping. To ensure that we don't wait indefinitely should an unrecoverable
// error be encountered, we'll also add a timed cancellation.
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(60));
while ((!cancellationSource.IsCancellationRequested) && (eventIndex <= expectedEvents.Count))
{
await Task.Delay(TimeSpan.FromMilliseconds(250));
}
// Once we arrive at this point, either cancellation was requested or we have processed all of our events. In
// both cases, we'll want to shut down the processor.
await processor.StopProcessingAsync();
}
finally
{
// It is encouraged that you unregister your handlers when you have finished
// using the Event Processor to ensure proper cleanup. This is especially
// important when using lambda expressions or handlers in any form that may
// contain closure scopes or hold other references.
processor.ProcessEventAsync -= processEventHandler;
processor.ProcessErrorAsync -= processErrorHandler;
}
// The Event Processor client has been stopped and is not explicitly disposable; there
// is nothing further that we need to do for cleanup.
Console.WriteLine();
}
public AzureContainerBrowser(BlobContainerClient client, bool prependContainerName, int maxTasks)
{
_client = client ?? throw new ArgumentNullException(nameof(client));
_prependContainerName = prependContainerName;
_asyncLimiter = new AsyncLimiter(maxTasks);
}
/// <summary>
/// Initializes a new instance of the <see cref="LeaseClient"/> class.
/// </summary>
/// <param name="client">
/// A <see cref="BlobContainerClient"/> representing the container
/// being leased.
/// </param>
/// <param name="leaseId">
/// An optional lease ID. If no lease ID is provided, a random lease
/// ID will be created.
/// </param>
public LeaseClient(BlobContainerClient client, string leaseId = null)
{
this._blob = null;
this._container = client ?? throw new ArgumentNullException(nameof(client));
this.LeaseId = leaseId ?? CreateUniqueLeaseId();
}
public static void Main(string[] args)
{
#region Snippet:Azure_Quantum_Jobs_CreateClient
// Create a QuantumJobClient
var subscriptionId = "your_subscription_id";
var resourceGroupName = "your_resource_group_name";
var workspaceName = "your_quantum_workspace_name";
var location = "your_location";
var storageContainerName = "your_container_name";
var credential = new DefaultAzureCredential(true);
var quantumJobClient =
new QuantumJobClient(
subscriptionId,
resourceGroupName,
workspaceName,
location,
credential);
#endregion
Console.WriteLine($@"Created QuantumJobClient for:
SubscriptionId: {subscriptionId}
ResourceGroup: {resourceGroupName}
workspaceName: {workspaceName}
location: {location}
");
Console.WriteLine($@"Getting Container Uri with SAS key...");
#region Snippet:Azure_Quantum_Jobs_GetContainerSasUri
// Get container Uri with SAS key
var containerUri = (quantumJobClient.GetStorageSasUri(
new BlobDetails(storageContainerName))).Value.SasUri;
#endregion
Console.WriteLine($@"Container Uri with SAS key:
{containerUri}
");
Console.WriteLine($@"Creating Container if not exist...");
// Create container if not exists
var containerClient = new BlobContainerClient(new Uri(containerUri));
containerClient.CreateIfNotExists();
Console.WriteLine($@"Uploading data into a blob...");
#region Snippet:Azure_Quantum_Jobs_UploadInputData
string problemFilePath = "./problem.json";
// Get input data blob Uri with SAS key
string blobName = Path.GetFileName(problemFilePath);
var inputDataUri = (quantumJobClient.GetStorageSasUri(
new BlobDetails(storageContainerName)
{
BlobName = blobName,
})).Value.SasUri;
using (var problemStreamToUpload = new MemoryStream())
{
using (FileStream problemFileStream = File.OpenRead(problemFilePath))
{
// Check if problem file is a gzip file.
// If it is, just read its contents.
// If not, read and compress the content.
var fileExtension = Path.GetExtension(problemFilePath).ToLower();
if (fileExtension == ".gz" ||
fileExtension == ".gzip")
{
problemFileStream.CopyTo(problemStreamToUpload);
}
else
{
using (var gzip = new GZipStream(problemStreamToUpload, CompressionMode.Compress, leaveOpen: true))
{
byte[] buffer = new byte[8192];
int count;
while ((count = problemFileStream.Read(buffer, 0, buffer.Length)) > 0)
{
gzip.Write(buffer, 0, count);
}
}
}
}
problemStreamToUpload.Position = 0;
// Upload input data to blob
var blobClient = new BlobClient(new Uri(inputDataUri));
var blobHeaders = new BlobHttpHeaders
{
ContentType = "application/json",
ContentEncoding = "gzip"
};
var blobUploadOptions = new BlobUploadOptions {
HttpHeaders = blobHeaders
};
blobClient.Upload(problemStreamToUpload, options: blobUploadOptions);
}
#endregion
Console.WriteLine($@"Input data Uri with SAS key:
{inputDataUri}
");
Console.WriteLine($@"Creating Quantum job...");
#region Snippet:Azure_Quantum_Jobs_CreateJob
// Submit job
var jobId = $"job-{Guid.NewGuid():N}";
var jobName = $"jobName-{Guid.NewGuid():N}";
var inputDataFormat = "microsoft.qio.v2";
var outputDataFormat = "microsoft.qio-results.v2";
var providerId = "microsoft";
var target = "microsoft.paralleltempering-parameterfree.cpu";
var inputParams = new Dictionary <string, object>()
{
{ "params", new Dictionary <string, object>() }
};
var createJobDetails = new JobDetails(containerUri, inputDataFormat, providerId, target)
{
Id = jobId,
InputDataUri = inputDataUri,
Name = jobName,
InputParams = inputParams,
OutputDataFormat = outputDataFormat
};
JobDetails myJob = (quantumJobClient.CreateJob(jobId, createJobDetails)).Value;
#endregion
Console.WriteLine($@"Job created:
Id: {myJob.Id}
Name: {myJob.Name}
CreationTime: {myJob.CreationTime}
Status: {myJob.Status}
");
Console.WriteLine($@"Awaiting job to complete...");
while (myJob.Status == JobStatus.Waiting ||
myJob.Status == JobStatus.Executing)
{
Thread.Sleep(TimeSpan.FromSeconds(5));
myJob = (quantumJobClient.GetJob(jobId)).Value;
Console.WriteLine($@"Job status: {myJob.Status}");
}
if (myJob.Status == JobStatus.Failed)
{
Console.WriteLine($@"Job has failed with error: {myJob.ErrorData.Message}");
}
Console.WriteLine($@"Getting Quantum job...");
#region Snippet:Azure_Quantum_Jobs_GetJob
// Get the job that we've just created based on its jobId
myJob = (quantumJobClient.GetJob(jobId)).Value;
#endregion
Console.WriteLine($@"Job obtained:
Id: {myJob.Id}
Name: {myJob.Name}
CreationTime: {myJob.CreationTime}
Status: {myJob.Status}
BeginExecutionTime: {myJob.BeginExecutionTime}
EndExecutionTime: {myJob.EndExecutionTime}
CancellationTime: {myJob.CancellationTime}
OutputDataFormat: {myJob.OutputDataFormat}
OutputDataUri: {myJob.OutputDataUri}
");
Console.WriteLine($@"Getting list of Quantum jobs...");
#region Snippet:Azure_Quantum_Jobs_GetJobs
foreach (JobDetails job in quantumJobClient.GetJobs())
{
Console.WriteLine($"{job.Name}");
}
#endregion
Console.WriteLine("Press [Enter] to exit...");
Console.ReadLine();
}
/// <summary>
/// Initializes a new instance of the <see cref="LeaseClient"/> class.
/// </summary>
/// <param name="client">
/// A <see cref="BlobContainerClient"/> representing the container
/// being leased.
/// </param>
/// <param name="leaseId">
/// An optional lease ID. If no lease ID is provided, a random lease
/// ID will be created.
/// </param>
public static LeaseClient GetLeaseClient(
this BlobContainerClient client,
string leaseId = null) =>
new LeaseClient(client, leaseId);
/// <summary>
/// Initializes a new instance of the <see cref="LeaseClient"/> class
/// for mocking.
/// </summary>
protected LeaseClient()
{
this._blob = null;
this._container = null;
}
public EventHubProcessorContext(IHostConfiguration hostConfiguration, ReceiveSettings receiveSettings, BlobContainerClient blobContainerClient,
EventProcessorClient client, Func <PartitionInitializingEventArgs, Task> partitionInitializingHandler,
Func <PartitionClosingEventArgs, Task> partitionClosingHandler, CancellationToken cancellationToken)
: base(cancellationToken)
{
_blobContainerClient = blobContainerClient;
_client = client;
var lockContext = new ProcessorLockContext(hostConfiguration, receiveSettings);
_client.PartitionInitializingAsync += async args =>
{
await lockContext.OnPartitionInitializing(args).ConfigureAwait(false);
if (partitionInitializingHandler != null)
{
await partitionInitializingHandler(args).ConfigureAwait(false);
}
};
_client.PartitionClosingAsync += async args =>
{
if (partitionClosingHandler != null)
{
await partitionClosingHandler(args).ConfigureAwait(false);
}
await lockContext.OnPartitionClosing(args).ConfigureAwait(false);
};
_client.ProcessErrorAsync += OnError;
_client.ProcessEventAsync += OnMessage;
ReceiveSettings = receiveSettings;
_lockContext = lockContext;
}
private static async Task GetFiles(BlobContainerClient client, string path, List <BlobFilePath> files)
{
var pages = client.GetBlobsByHierarchyAsync(prefix: path, delimiter: "/").AsPages(default);
/// <summary>
/// Runs the sample using the specified Event Hubs and Azure storage connection information.
/// </summary>
///
/// <param name="eventHubsConnectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that the sample should run against.</param>
/// <param name="blobStorageConnectionString">The connection string for the storage account where checkpoints and state should be persisted.</param>
/// <param name="blobContainerName">The name of the blob storage container where checkpoints and state should be persisted.</param>
///
public async Task RunAsync(string eventHubsConnectionString,
string eventHubName,
string blobStorageConnectionString,
string blobContainerName)
{
// To begin, we'll publish a batch of events for our processor to receive. Because we are not specifying any routing hints,
// the Event Hubs service will automatically route these to partitions. We'll split the events into a couple of batches to
// increase the chance they'll be spread around.
var expectedEvents = new List <EventData>()
{
new EventData(Encoding.UTF8.GetBytes("First Event, First Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, First Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, First Batch")),
new EventData(Encoding.UTF8.GetBytes("First Event, Second Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, Second Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, Second Batch")),
new EventData(Encoding.UTF8.GetBytes("First Event, Third Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, Third Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, Third Batch"))
};
int sentIndex = 0;
int numberOfBatches = 3;
int eventsPerBatch = (expectedEvents.Count / numberOfBatches);
await using (var producer = new EventHubProducerClient(eventHubsConnectionString, eventHubName))
{
while (sentIndex < expectedEvents.Count)
{
using EventDataBatch eventBatch = await producer.CreateBatchAsync();
for (int index = 0; index < eventsPerBatch; ++index)
{
eventBatch.TryAdd(expectedEvents[sentIndex]);
++sentIndex;
}
await producer.SendAsync(eventBatch);
}
}
// With our events having been published, we'll create an Event Hub Processor to read them.
string consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
BlobContainerClient storageClient = new BlobContainerClient(blobStorageConnectionString, blobContainerName);
EventProcessorClient processor = new EventProcessorClient(storageClient, consumerGroup, eventHubsConnectionString, eventHubName);
// Create a simple handler for event processing.
int eventIndex = 0;
Task processEventHandler(ProcessEventArgs eventArgs)
{
if (eventArgs.CancellationToken.IsCancellationRequested)
{
return(Task.CompletedTask);
}
try
{
++eventIndex;
Console.WriteLine($"Event Received: { Encoding.UTF8.GetString(eventArgs.Data.Body.ToArray()) }");
}
catch (Exception ex)
{
Console.WriteLine();
Console.WriteLine($"An error was observed while processing events. Message: { ex.Message }");
Console.WriteLine();
}
// Because our example handler is running synchronously, we'll manually return a completed
// task.
return(Task.CompletedTask);
};
// The error handler is invoked when there is an exception observed within the Event Processor client; it is not invoked for
// exceptions in your handler code. The Event Processor client will make every effort to recover from exceptions and continue
// processing. Should an exception that cannot be recovered from is encountered, the processor will forfeit ownership of all partitions
// that it was processing so that work may redistributed.
//
// For this example, arbitrarily choose to restart processing when an Event Hubs service exception is encountered and simply
// log other exceptions to the console.
//
// It is important to note that this selection is for demonstration purposes only; it does not constitute the recommended course
// of action for service errors. Because the right approach for handling errors can vary greatly between different types of
// application, you will need to determine the error handling strategy that best suits your scenario.
async Task processErrorHandler(ProcessErrorEventArgs eventArgs)
{
// As with the process event handler, the event arguments contain a cancellation token used by the Event Processor client to signal
// that the operation should be canceled. The handler should respect cancellation as it is able in order to ensure that the Event
// Processor client is able to perform its operations efficiently.
//
// The process error handler is not awaited by the Event Processor client and is, instead, executed in a fire-and-forget manner. This
// means that you may safely interact with the Event Processor client, such as requesting that it stop processing.
if (eventArgs.CancellationToken.IsCancellationRequested)
{
return;
}
// Because there is no long-running or I/O operation, inspecting the cancellation token again does not make sense in this scenario.
// However, in real-world processing, it is recommended that you do so as you are able without compromising your ability to capture
// and troubleshooting information.
//
// It is also recommended that the cancellation token be passed to any asynchronous operations that are awaited in this handler.
Console.WriteLine();
Console.WriteLine("===============================");
Console.WriteLine($"The error handler was invoked during the operation: { eventArgs.Operation ?? "Unknown" }, for Exception: { eventArgs.Exception.Message }");
Console.WriteLine("===============================");
Console.WriteLine();
// We will not pass the cancellation token from the event arguments here, as it will be
// signaled when we request that the processor stop.
if (eventArgs.Exception is EventHubsException)
{
Console.WriteLine("Detected an service error. Restarting the processor...");
await processor.StopProcessingAsync();
await processor.StartProcessingAsync();
Console.WriteLine("Processor has been restarted....");
Console.WriteLine();
}
}
processor.ProcessEventAsync += processEventHandler;
processor.ProcessErrorAsync += processErrorHandler;
try
{
Console.WriteLine("Starting the Event Processor client...");
eventIndex = 0;
await processor.StartProcessingAsync();
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
// Unfortunately, because the handler is invoked when exceptions are encountered in the Event Processor client and not for the code
// developers write in the event handlers, there is no reliable way to force an exception. As a result, it is unlikely that you will
// be able to observe the error handler in action by just running the Event Processor.
//
// Instead, we will wait a short bit to allow processing to take place and then artificially trigger the event handler for illustration
// purposes.
await Task.Delay(TimeSpan.FromMilliseconds(250));
Console.WriteLine("Triggering the error handler...");
ProcessErrorEventArgs eventArgs = new ProcessErrorEventArgs("fake", "artificial invoke", new EventHubsException(true, eventHubName), cancellationSource.Token);
await processErrorHandler(eventArgs);
while ((!cancellationSource.IsCancellationRequested) && (eventIndex <= expectedEvents.Count))
{
await Task.Delay(TimeSpan.FromMilliseconds(250));
}
// Once we arrive at this point, either cancellation was requested or we have processed all of our events. In
// both cases, we'll want to shut down the processor.
Console.WriteLine();
Console.WriteLine("Stopping the processor...");
await processor.StopProcessingAsync();
}
finally
{
// It is encouraged that you unregister your handlers when you have finished
// using the Event Processor to ensure proper cleanup. This is especially
// important when using lambda expressions or handlers in any form that may
// contain closure scopes or hold other references.
processor.ProcessEventAsync -= processEventHandler;
processor.ProcessErrorAsync -= processErrorHandler;
}
// The Event Processor client has been stopped and is not explicitly disposable; there
// is nothing further that we need to do for cleanup.
Console.WriteLine();
}
private static async Task <bool> ContainerExistsAsync(BlobContainerClient blobContainerClient)
{
return((await blobContainerClient.ExistsAsync()).Value);
}
public async Task ProcessEvents()
{
await using var eventHubScope = await EventHubScope.CreateAsync(1);
await using var storageScope = await StorageScope.CreateAsync();
#region Snippet:EventHubs_Processor_Sample06_ChooseStorageVersion
var storageConnectionString = "<< CONNECTION STRING FOR THE STORAGE ACCOUNT >>";
var blobContainerName = "<< NAME OF THE BLOB CONTAINER >>";
/*@@*/
/*@@*/ storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
/*@@*/ blobContainerName = storageScope.ContainerName;
var eventHubsConnectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = "<< NAME OF THE EVENT HUB CONSUMER GROUP >>";
/*@@*/
/*@@*/ eventHubsConnectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
/*@@*/ eventHubName = eventHubScope.EventHubName;
/*@@*/ consumerGroup = eventHubScope.ConsumerGroups.First();
var storageClientOptions = new BlobClientOptions();
storageClientOptions.AddPolicy(
new StorageApiVersionPolicy(),
HttpPipelinePosition.PerCall);
var storageClient = new BlobContainerClient(
storageConnectionString,
blobContainerName,
storageClientOptions);
var processor = new EventProcessorClient(
storageClient,
consumerGroup,
eventHubsConnectionString,
eventHubName);
try
{
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
// The event handlers are not relevant for this sample; for
// illustration, they're delegating the implementation to the
// host application.
processor.ProcessEventAsync += Application.ProcessorEventHandler;
processor.ProcessErrorAsync += Application.ProcessorErrorHandler;
try
{
await processor.StartProcessingAsync(cancellationSource.Token);
await Task.Delay(Timeout.Infinite, cancellationSource.Token);
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
// This may take up to the length of time defined
// as part of the configured TryTimeout of the processor;
// by default, this is 60 seconds.
await processor.StopProcessingAsync();
}
}
catch
{
// The processor will automatically attempt to recover from any
// failures, either transient or fatal, and continue processing.
// Errors in the processor's operation will be surfaced through
// its error handler.
//
// If this block is invoked, then something external to the
// processor was the source of the exception.
}
finally
{
// It is encouraged that you unregister your handlers when you have
// finished using the Event Processor to ensure proper cleanup. This
// is especially important when using lambda expressions or handlers
// in any form that may contain closure scopes or hold other references.
processor.ProcessEventAsync -= Application.ProcessorEventHandler;
processor.ProcessErrorAsync -= Application.ProcessorErrorHandler;
}
#endregion
}
static async Task Main(string[] args)
{
var connectionString = Environment.GetEnvironmentVariable("STORAGE_CONNECTION_STRING");
if (string.IsNullOrEmpty(connectionString))
{
throw new InvalidOperationException("Environment variable STORAGE_CONNECTION_STRING is not set");
}
Log($"ITERATIONS: {_iterations}");
Log($"UPLOAD_SIZE: {_uploadSize}");
Log($"BUFFER_SIZE: {_bufferSize}");
Log($"MAX_CONCURRENCY: {_maxConcurrency}");
// Enable SDK logging (with timestamps)
using var azureListener = new AzureEventSourceListener(
(eventData, text) => Log(String.Format("[{1}] {0}: {2}", eventData.EventSource.Name, eventData.Level, text)),
EventLevel.Verbose);
// Enable System.Net logging
using var httpListener = new LogEventListener("Microsoft-System-Net-Http");
using var socketsListener = new LogEventListener("Microsoft-System-Net-Sockets");
var containerName = $"container{DateTime.Now.Ticks}";
// Test custom transport with shorter timeout
var containerClient = new BlobContainerClient(connectionString, containerName, new BlobClientOptions()
{
Transport = new HttpClientTransport(new HttpClient()
{
Timeout = _httpClientTimeout
})
});
Log($"Creating container {containerName}");
await containerClient.CreateAsync();
Log($"Created container {containerName}");
var randomBuffer = new byte[_uploadSize];
new Random(0).NextBytes(randomBuffer);
var randomStream = new NonSeekableMemoryStream(randomBuffer);
for (var i = 0; i < _iterations; i++)
{
try
{
Log($"Iteration {i}");
var blobName = $"blob{DateTime.Now.Ticks}";
var blobClient = containerClient.GetBlobClient(blobName);
randomStream.Seek(0, SeekOrigin.Begin);
Log($"Uploading blob {blobName}");
await blobClient.UploadAsync(randomStream, transferOptions : new StorageTransferOptions()
{
MaximumConcurrency = _maxConcurrency,
MaximumTransferLength = _bufferSize
});
Log($"Uploaded blob {blobName}");
Log($"Deleting blob {blobName}");
await blobClient.DeleteAsync();
Log($"Deleted blob {blobName}");
}
catch (Exception e)
{
Log(e);
}
}
Log($"Deleting container {containerName}");
await containerClient.DeleteAsync();
Log($"Deleted container {containerName}");
}
private static async Task LoadFiles(string backupPath)
{
try
{
string connectionString = Environment.GetEnvironmentVariable("AZURE_STORAGE_CONNECTION_STRING");
//Creating Container
string containerName = $"mysqlbackup-{DateTime.UtcNow.ToString("ddMyyyy-HHmmss")}";
BlobServiceClient blobServiceClient = new BlobServiceClient(connectionString);
Console.WriteLine("containerName: " + containerName);
BlobContainerClient containerClient = await blobServiceClient.CreateBlobContainerAsync(containerName);
Console.WriteLine($"containerClient {containerClient.Name}");
// Create a local file in the ./data/ directory for uploading and downloading
/* string localPath = "./data/";
* string fileName = $"quickstart{Guid.NewGuid().ToString()}.txt";
* string localFilePath = Path.Combine(localPath, fileName);
*
* await File.WriteAllTextAsync(localFilePath, "Hello, World!"); */
string fileName = "all_databases.sql";
string localFilePath = Path.Combine(backupPath, fileName);
// Ger a reference to a blob
BlobClient blobClient = containerClient.GetBlobClient(fileName);
Console.WriteLine("Uploading to Blob storage as blob:\n\t {0}\n", blobClient.Uri);
// Open the file and upload its data
using (FileStream uploadFileStream = File.OpenRead(localFilePath))
{
await blobClient.UploadAsync(uploadFileStream, true);
}
Console.WriteLine("Listing blobs...");
await foreach (BlobItem blobItem in containerClient.GetBlobsAsync())
{
Console.WriteLine("\t" + blobItem.Name);
}
// Download the blob to a local file
// Append the string "DOWNLOAD" before the .txt extension
// so you can compare the files in the data directory
/* string downloadFilePath = localFilePath.Replace(".txt", "_download.txt");
* Console.WriteLine($"\nDownloading blob to \n\t{downloadFilePath}");
*
* // Download the blob's contents and save it to a file
* BlobDownloadInfo download = await blobClient.DownloadAsync();
*
* using (FileStream downloadFileStream = File.OpenWrite(downloadFilePath))
* {
* await download.Content.CopyToAsync(downloadFileStream);
* downloadFileStream.Close();
* } */
// Clean up
// await containerClient.DeleteAsync();
// File.Delete(localFilePath);
Console.WriteLine("Backup finished");
}
catch (System.Exception ex)
{
Console.WriteLine("Error " + ex.Message);
throw;
}
}
public PortfolioController(UrbanSisterContext context, IMapper mapper, BlobServiceClient blobServiceClient)
{
this._context = context;
this._mapper = mapper;
this._blobContainerClient = blobServiceClient.GetBlobContainerClient("portfoliopicture");
}
static void Main(string[] args)
{
Console.WriteLine("*** Welcome to the Blob Container Copy Utility ***");
Console.Write("Enter the connection string of the source storage account: ");
string sourceStorageConnectionString = Console.ReadLine();
Console.Write("Enter the name of the source container: ");
string sourceContainer = Console.ReadLine();
Console.Write("Enter the connection string of the destination storage account: ");
string destStorageConnectionString = Console.ReadLine();
Console.Write("Enter the name of the destination container: ");
string destContainer = Console.ReadLine();
if (string.IsNullOrEmpty(sourceStorageConnectionString) ||
string.IsNullOrEmpty(sourceContainer) ||
string.IsNullOrEmpty(destStorageConnectionString) ||
string.IsNullOrEmpty(destContainer))
{
Console.WriteLine("You must provide values for the source connection string, source container, destination connection string and destination container");
}
else
{
BlobServiceClient sourceBlobServiceClient = new BlobServiceClient(sourceStorageConnectionString);
BlobContainerClient sourceContainerClient = sourceBlobServiceClient.GetBlobContainerClient(sourceContainer);
BlobServiceClient destBlobServiceClient = new BlobServiceClient(destStorageConnectionString);
BlobContainerClient destContainerClient = destBlobServiceClient.GetBlobContainerClient(destContainer);
if (sourceContainerClient.Exists())
{
if (!destContainerClient.Exists())
{
destContainerClient.Create(PublicAccessType.None);
}
List <BlobItem> blobs = sourceContainerClient.GetBlobs().ToList();
Console.WriteLine($"Copying {blobs.Count} files from source to destination.");
foreach (BlobItem blob in blobs)
{
BlobClient sourceBlob = sourceContainerClient.GetBlobClient(blob.Name);
if (sourceBlob.Exists())
{
Console.WriteLine($"{blob.Name} copy started...");
BlobDownloadInfo download = sourceBlob.Download();
BlobClient destBlob = destContainerClient.GetBlobClient(blob.Name);
destBlob.Upload(download.Content);
Console.WriteLine($"{blob.Name} copied successfully.");
}
}
}
else
{
Console.WriteLine($"Source blob container {sourceContainer} does not exist.");
}
Console.WriteLine("*** Copy completed ***");
}
Console.WriteLine("*** Press any key to exit ***");
Console.Read();
}
// </Snippet_ReadBlobMetadata>
//-------------------------------------------------
// Metadata menu
//-------------------------------------------------
public async Task <bool> MenuAsync()
{
var connectionString = Constants.connectionString;
BlobServiceClient blobServiceClient = new BlobServiceClient(connectionString);
BlobContainerClient container = blobServiceClient.GetBlobContainerClient(Constants.containerName);
BlobClient blob = container.GetBlobClient(Constants.blobName);
Console.Clear();
Console.WriteLine("Choose a properties or metadata scenario:");
Console.WriteLine("1) Read container properties");
Console.WriteLine("2) Set metadata on container");
Console.WriteLine("3) Read metadata on container");
Console.WriteLine("4) Set blob properties");
Console.WriteLine("5) Read blob properties");
Console.WriteLine("6) Add metadata on blob");
Console.WriteLine("7) Read blob metadata");
Console.WriteLine("X) Exit to main menu");
Console.Write("\r\nSelect an option: ");
switch (Console.ReadLine())
{
case "1":
await ReadContainerPropertiesAsync(container);
Console.WriteLine("Press enter to continue");
Console.ReadLine();
return(true);
case "2":
await AddContainerMetadataAsync(container);
Console.WriteLine("Press enter to continue");
Console.ReadLine();
return(true);
case "3":
await ReadContainerMetadataAsync(container);
Console.WriteLine("Press enter to continue");
Console.ReadLine();
return(true);
case "4":
await SetBlobPropertiesAsync(blob);
Console.WriteLine("Press enter to continue");
Console.ReadLine();
return(true);
case "5":
await GetBlobPropertiesAsync(blob);
Console.WriteLine("Press enter to continue");
Console.ReadLine();
return(true);
case "6":
await AddBlobMetadataAsync(blob);
Console.WriteLine("Press enter to continue");
Console.ReadLine();
return(true);
case "7":
await ReadBlobMetadataAsync(blob);
Console.WriteLine("Press enter to continue");
Console.ReadLine();
return(true);
case "x":
case "X":
return(false);
default:
return(true);
}
}
private static Func <string, IDistributedLock> CreateAzureDistributedLockFactory(IServiceProvider services, Uri blobContainerUrl)
{
var container = new BlobContainerClient(blobContainerUrl);
return(name => new AzureBlobLeaseDistributedLock(container, name));
}
/// <summary>
/// Runs the sample using the specified Event Hubs and Azure storage connection information.
/// </summary>
///
/// <param name="eventHubsConnectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that the sample should run against.</param>
/// <param name="blobStorageConnectionString">The connection string for the storage account where checkpoints and state should be persisted.</param>
/// <param name="blobContainerName">The name of the blob storage container where checkpoints and state should be persisted.</param>
///
public async Task RunAsync(string eventHubsConnectionString,
string eventHubName,
string blobStorageConnectionString,
string blobContainerName)
{
// In order to ensure efficient communication with the Event Hubs service and the best throughput possible for dispatching events to be processed,
// the Event Processor client is eagerly reading from each partition of the Event Hub and staging events. The processor will dispatch an event
// to the "ProcessEvent" handler immediately when one is available. Each call to the handler passes a single event and the context of the partition
// from which the event was read. This pattern is intended to allow developers to act on an event as soon as possible, and present a straightforward
// and understandable interface.
//
// This approach is optimized for scenarios where the processing of events can be performed quickly and without heavy resource costs. For scenarios
// where that is not the case, it may be advantageous to collect the events into batches and send them to be processed outside of the context
// of the "ProcessEvent" handler.
//
// In this example, our "ProcessEvent" handler will group events into batches by partition, sending them for downstream processing when the desired
// batch size was reached or when no event was available for more than a maximum wait time interval.
int desiredBatchSize = 3;
TimeSpan maximumWaitTime = TimeSpan.FromMilliseconds(150);
// The Event Processor client will preserve the order that events were enqueued in a partition by waiting for the "ProcessEvent" handler to
// complete before it is invoked with a new event for the same partition. However, partitions are processed concurrently, so the
// handler is likely to be executing for multiple partitions at the same time.
//
// To account for this, we'll use a concurrent dictionary to track batches, grouping them by partition.
ConcurrentDictionary <string, List <ProcessEventArgs> > eventBatches = new ConcurrentDictionary <string, List <ProcessEventArgs> >();
// Create our Event Processor client, specifying the maximum wait time as an option to ensure that
// our handler is invoked when no event was available.
EventProcessorClientOptions clientOptions = new EventProcessorClientOptions
{
MaximumWaitTime = maximumWaitTime
};
string consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
BlobContainerClient storageClient = new BlobContainerClient(blobStorageConnectionString, blobContainerName);
EventProcessorClient processor = new EventProcessorClient(storageClient, consumerGroup, eventHubsConnectionString, eventHubName, clientOptions);
// For this example, we'll create a simple event handler that writes to the
// console each time it was invoked.
int eventIndex = 0;
async Task processEventHandler(ProcessEventArgs eventArgs)
{
if (eventArgs.CancellationToken.IsCancellationRequested)
{
return;
}
try
{
// Retrieve or create the active batch for the current partition.
//
// NOTE: There is a bug in the Event Hubs Processor preview 6 library causing the "Partition"
// property to be null when no event was available. This sample will work when run against the
// referenced project but will require adjustments to work with the preview 6 package.
List <ProcessEventArgs> currentBatch = eventBatches.GetOrAdd(eventArgs.Partition.PartitionId, _ => new List <ProcessEventArgs>());
bool sendBatchForProcessing = false;
// If there was an event emitted, add the event and check to see if the size of the batch has reached the desired
// size. If so, it will need to be sent.
//
// NOTE: There is a bug in the Event Hubs preview 6 library causing "HasEvents" to return the
// wrong value. We'll substitute a check against the "Data" property to work around it.
//
// if (eventArgs.HasEvents) {} is the preferred snippet.
//
if (eventArgs.Data != null)
{
currentBatch.Add(eventArgs);
sendBatchForProcessing = (currentBatch.Count >= desiredBatchSize);
}
else
{
// There was no event available within the interval requested by the maximum
// wait time, send the batch for processing if it contains any events.
sendBatchForProcessing = (currentBatch.Count > 0);
}
// It is important to be aware that no events for the partition will be processed until the handler returns,
// so you may wish to delegate processing to a downstream service or background task in order to maintain
// throughput.
//
// In this example, if the batch is to be sent, we'll delegate to simple helper method that will write
// to the console. If sending our batch completed successfully, we'll checkpoint using the last
// event and clear the batch.
if (sendBatchForProcessing)
{
await SendEventBatchAsync(currentBatch.Select(item => item.Data));
await currentBatch[currentBatch.Count - 1].UpdateCheckpointAsync();
currentBatch.Clear();
}
}
catch (Exception ex)
{
Console.WriteLine();
Console.WriteLine($"An error was observed while processing events. Message: { ex.Message }");
Console.WriteLine();
}
++eventIndex;
};
// For this example, exceptions will just be logged to the console.
Task processErrorHandler(ProcessErrorEventArgs eventArgs)
{
if (eventArgs.CancellationToken.IsCancellationRequested)
{
return(Task.CompletedTask);
}
Console.WriteLine();
Console.WriteLine("===============================");
Console.WriteLine($"The error handler was invoked during the operation: { eventArgs.Operation ?? "Unknown" }, for Exception: { eventArgs.Exception.Message }");
Console.WriteLine("===============================");
Console.WriteLine();
return(Task.CompletedTask);
}
processor.ProcessEventAsync += processEventHandler;
processor.ProcessErrorAsync += processErrorHandler;
try
{
Console.WriteLine("Starting the Event Processor client...");
Console.WriteLine();
eventIndex = 0;
await processor.StartProcessingAsync();
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(90));
// We'll publish a batch of events for our processor to receive. We'll split the events into a couple of batches to
// increase the chance they'll be spread around to different partitions and introduce a delay between batches to
// allow for the handler to be invoked without an available event interleaved.
var expectedEvents = new List <EventData>()
{
new EventData(Encoding.UTF8.GetBytes("First Event, First Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, First Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, First Batch")),
new EventData(Encoding.UTF8.GetBytes("First Event, Second Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, Second Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, Second Batch")),
new EventData(Encoding.UTF8.GetBytes("First Event, Third Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, Third Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, Third Batch")),
new EventData(Encoding.UTF8.GetBytes("First Event, Fourth Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, Fourth Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, Fourth Batch")),
new EventData(Encoding.UTF8.GetBytes("First Event, Fifth Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, Fifth Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, Fifth Batch")),
new EventData(Encoding.UTF8.GetBytes("First Event, Fifth Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, Fifth Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, Fifth Batch")),
new EventData(Encoding.UTF8.GetBytes("First Event, Sixth Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, Sixth Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, Sixth Batch")),
new EventData(Encoding.UTF8.GetBytes("First Event, Seventh Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, Seventh Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, Seventh Batch"))
};
int sentIndex = 0;
int numberOfBatches = 3;
int eventsPerBatch = (expectedEvents.Count / numberOfBatches);
await using (var producer = new EventHubProducerClient(eventHubsConnectionString, eventHubName))
{
while (sentIndex < expectedEvents.Count)
{
using EventDataBatch eventBatch = await producer.CreateBatchAsync();
for (int index = 0; index < eventsPerBatch; ++index)
{
eventBatch.TryAdd(expectedEvents[sentIndex]);
++sentIndex;
}
await producer.SendAsync(eventBatch);
await Task.Delay(250, cancellationSource.Token);
}
}
// We'll allow the Event Processor client to read and dispatch the events that we published, along with
// ensuring a few invocations with no event. Note that, due to non-determinism in the timing, we may or may
// not see all of the events from our batches read.
while ((!cancellationSource.IsCancellationRequested) && (eventIndex <= expectedEvents.Count + 5))
{
await Task.Delay(TimeSpan.FromMilliseconds(250));
}
// Once we arrive at this point, either cancellation was requested or we have processed all of our events. In
// both cases, we'll want to shut down the processor.
Console.WriteLine();
Console.WriteLine("Stopping the processor...");
await processor.StopProcessingAsync();
}
finally
{
// It is encouraged that you unregister your handlers when you have finished
// using the Event Processor to ensure proper cleanup. This is especially
// important when using lambda expressions or handlers in any form that may
// contain closure scopes or hold other references.
processor.ProcessEventAsync -= processEventHandler;
processor.ProcessErrorAsync -= processErrorHandler;
}
// The Event Processor client has been stopped and is not explicitly disposable; there
// is nothing further that we need to do for cleanup.
Console.WriteLine();
}
public async Task DefaultAzureCredential()
{
await using var eventHubScope = await EventHubScope.CreateAsync(1);
await using var storageScope = await StorageScope.CreateAsync();
#region Snippet:EventHubs_Processor_Sample05_DefaultAzureCredential
TokenCredential credential = new DefaultAzureCredential();
var storageEndpoint = "<< STORAGE ENDPOINT (likely similar to {your-account}.blob.core.windows.net) >>";
var blobContainerName = "<< NAME OF THE BLOB CONTAINER >>";
/*@@*/
/*@@*/ storageEndpoint = new BlobServiceClient(StorageTestEnvironment.Instance.StorageConnectionString).Uri.ToString();
/*@@*/ blobContainerName = storageScope.ContainerName;
var fullyQualifiedNamespace = "<< NAMESPACE (likely similar to {your-namespace}.servicebus.windows.net) >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = "<< NAME OF THE EVENT HUB CONSUMER GROUP >>";
/*@@*/
/*@@*/ fullyQualifiedNamespace = EventHubsTestEnvironment.Instance.FullyQualifiedNamespace;
/*@@*/ eventHubName = eventHubScope.EventHubName;
/*@@*/ consumerGroup = eventHubScope.ConsumerGroups.First();
/*@@*/ credential = EventHubsTestEnvironment.Instance.Credential;
var blobUriBuilder = new BlobUriBuilder(new Uri(storageEndpoint));
blobUriBuilder.BlobContainerName = blobContainerName;
var storageClient = new BlobContainerClient(
blobUriBuilder.ToUri(),
credential);
var processor = new EventProcessorClient(
storageClient,
consumerGroup,
fullyQualifiedNamespace,
eventHubName,
credential);
try
{
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
// The event handlers are not relevant for this sample; for
// illustration, they're delegating the implementation to the
// host application.
processor.ProcessEventAsync += Application.ProcessorEventHandler;
processor.ProcessErrorAsync += Application.ProcessorErrorHandler;
try
{
await processor.StartProcessingAsync(cancellationSource.Token);
await Task.Delay(Timeout.Infinite, cancellationSource.Token);
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
// This may take up to the length of time defined
// as part of the configured TryTimeout of the processor;
// by default, this is 60 seconds.
await processor.StopProcessingAsync();
}
}
catch
{
// If this block is invoked, then something external to the
// processor was the source of the exception.
}
finally
{
// It is encouraged that you unregister your handlers when you have
// finished using the Event Processor to ensure proper cleanup.
processor.ProcessEventAsync -= Application.ProcessorEventHandler;
processor.ProcessErrorAsync -= Application.ProcessorErrorHandler;
}
#endregion
}
public async Task PostAsync([FromBody] string name)
{
BlobContainerClient blobContainer = await _client.CreateBlobContainerAsync(name);
}
/// <inheritdoc/>
public async Task <ITenant> CreateWellKnownChildTenantAsync(
string parentTenantId,
Guid wellKnownChildTenantGuid,
string name)
{
(ITenant parentTenant, BlobContainerClient container) =
await this.GetContainerAndTenantForChildTenantsOfAsync(parentTenantId).ConfigureAwait(false);
// We need to copy blob storage settings for the Tenancy container definition from the parent to the new child
// to support the tenant blob store provider. We would expect this to be overridden by clients that wanted to
// establish their own settings.
bool configIsInV3 = true;
LegacyV2BlobStorageConfiguration?v2TenancyStorageConfiguration = null;
if (!parentTenant.Properties.TryGet(TenancyV3ConfigKey, out BlobContainerConfiguration tenancyStorageConfiguration))
{
configIsInV3 = false;
if (!parentTenant.Properties.TryGet(TenancyV2ConfigKey, out v2TenancyStorageConfiguration))
{
throw new InvalidOperationException($"No configuration found for ${TenancyV3ConfigKey} or ${TenancyV2ConfigKey}");
}
}
IPropertyBag childProperties;
if (parentTenantId == this.Root.Id && this.propagateRootStorageConfigAsV2)
{
configIsInV3 = false;
v2TenancyStorageConfiguration = new LegacyV2BlobStorageConfiguration
{
Container = tenancyStorageConfiguration.Container,
};
if (tenancyStorageConfiguration.ConnectionStringPlainText != null)
{
v2TenancyStorageConfiguration.AccountName = tenancyStorageConfiguration.ConnectionStringPlainText;
}
else if (tenancyStorageConfiguration.AccountName != null)
{
v2TenancyStorageConfiguration.AccountName = tenancyStorageConfiguration.AccountName;
v2TenancyStorageConfiguration.KeyVaultName = tenancyStorageConfiguration.AccessKeyInKeyVault?.VaultName;
v2TenancyStorageConfiguration.AccountKeySecretName = tenancyStorageConfiguration.AccessKeyInKeyVault?.SecretName;
}
}
if (configIsInV3)
{
childProperties = this.propertyBagFactory.Create(values =>
values.Append(new KeyValuePair <string, object>(TenancyV3ConfigKey, tenancyStorageConfiguration)));
}
else
{
childProperties = this.propertyBagFactory.Create(values =>
values.Append(new KeyValuePair <string, object>(TenancyV2ConfigKey, v2TenancyStorageConfiguration !)));
}
var child = new Tenant(
parentTenantId.CreateChildId(wellKnownChildTenantGuid),
name,
childProperties);
// TODO: this needs thinking through.
BlobContainerClient newTenantBlobContainer = await this.GetBlobContainer(child).ConfigureAwait(false);
await newTenantBlobContainer.CreateIfNotExistsAsync().ConfigureAwait(false);
// As we create the new blob, we need to ensure there isn't already a tenant with the same Id. We do this by
// providing an If-None-Match header passing a "*", which will cause a storage exception with a 409 status
// code if a blob with the same Id already exists.
BlockBlobClient blob = GetLiveTenantBlockBlobReference(child.Id, container);
var content = new MemoryStream();
using (var sw = new StreamWriter(content, UTF8WithoutBom, leaveOpen: true))
using (JsonWriter writer = new JsonTextWriter(sw))
{
this.jsonSerializer.Serialize(writer, child);
}
content.Position = 0;
try
{
Response <BlobContentInfo> response = await blob.UploadAsync(
content,
new BlobUploadOptions { Conditions = new BlobRequestConditions {
IfNoneMatch = ETag.All
} })
.ConfigureAwait(false);
child.ETag = response.Value.ETag.ToString("H");
}
catch (global::Azure.RequestFailedException x)
when(x.ErrorCode == "BlobAlreadyExists")
{
// This exception is thrown because there's already a tenant with the same Id. This should never happen when
// this method has been called from CreateChildTenantAsync as the Guid will have been generated and the
// chances of it matching one previously generated are miniscule. However, it could happen when calling this
// method directly with a wellKnownChildTenantGuid that's already in use. In this case, the fault is with
// the client code - creating tenants with well known Ids is something one would expect to happen under
// controlled conditions, so it's only likely that a conflict will occur when either the client code has made
// a mistake or someone is actively trying to cause problems.
throw new ArgumentException(
$"A child tenant of '{parentTenantId}' with a well known Guid of '{wellKnownChildTenantGuid}' already exists.",
nameof(wellKnownChildTenantGuid));
}
return(child);
}
/// <summary>
/// This method is called each polling interval for all containers. The method divides the
/// budget of allocated number of blobs to query, for each container we query a page of
/// that size and we keep the continuation token for the next time. AS a curser, we use
/// the time stamp when the current cycle on the container started. blobs newer than that
/// time will be considered new and registrations will be notified
/// </summary>
/// <param name="container"></param>
/// <param name="containerScanInfo"> Information that includes the last cycle start
/// the continuation token and the current cycle start for a container</param>
/// <param name="clientRequestId"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task <IEnumerable <BlobBaseClient> > PollNewBlobsAsync(
BlobContainerClient container, ContainerScanInfo containerScanInfo, string clientRequestId, CancellationToken cancellationToken)
{
IEnumerable <BlobItem> currentBlobs;
int blobPollLimitPerContainer = _scanBlobLimitPerPoll / _scanInfo.Count;
string continuationToken = containerScanInfo.ContinuationToken;
Page <BlobItem> page;
// if starting the cycle, reset the sweep time
if (continuationToken == null)
{
containerScanInfo.CurrentSweepCycleLatestModified = DateTime.MinValue;
}
Stopwatch sw = Stopwatch.StartNew();
try
{
AsyncPageable <BlobItem> blobsAsyncPageable = container.GetBlobsAsync(cancellationToken: cancellationToken);
IAsyncEnumerable <Page <BlobItem> > pages = blobsAsyncPageable.AsPages(continuationToken: continuationToken, pageSizeHint: blobPollLimitPerContainer);
IAsyncEnumerator <Page <BlobItem> > pagesEnumerator = pages.GetAsyncEnumerator(cancellationToken);
if (await pagesEnumerator.MoveNextAsync().ConfigureAwait(false))
{
page = pagesEnumerator.Current;
currentBlobs = page.Values;
}
else
{
return(Enumerable.Empty <BlobBaseClient>());
}
}
catch (RequestFailedException exception)
{
if (exception.IsNotFound())
{
Logger.ContainerDoesNotExist(_logger, container.Name);
return(Enumerable.Empty <BlobBaseClient>());
}
else
{
throw;
}
}
List <BlobBaseClient> newBlobs = new List <BlobBaseClient>();
// Type cast to IStorageBlob is safe due to useFlatBlobListing: true above.
foreach (BlobItem currentBlob in currentBlobs)
{
cancellationToken.ThrowIfCancellationRequested();
var properties = currentBlob.Properties;
DateTime lastModifiedTimestamp = properties.LastModified.Value.UtcDateTime;
if (lastModifiedTimestamp > containerScanInfo.CurrentSweepCycleLatestModified)
{
containerScanInfo.CurrentSweepCycleLatestModified = lastModifiedTimestamp;
}
// Blob timestamps are rounded to the nearest second, so make sure we continue to check
// the previous timestamp to catch any blobs that came in slightly after our previous poll.
if (lastModifiedTimestamp >= containerScanInfo.LastSweepCycleLatestModified)
{
newBlobs.Add(container.GetBlobClient(currentBlob.Name));
}
}
Logger.PollBlobContainer(_logger, container.Name, containerScanInfo.LastSweepCycleLatestModified, clientRequestId,
newBlobs.Count, sw.ElapsedMilliseconds, !string.IsNullOrWhiteSpace(page.ContinuationToken));
// record continuation token for next chunk retrieval
containerScanInfo.ContinuationToken = page.ContinuationToken;
// if ending a cycle then copy currentSweepCycleStartTime to lastSweepCycleStartTime, if changed
if (page.ContinuationToken == null &&
containerScanInfo.CurrentSweepCycleLatestModified > containerScanInfo.LastSweepCycleLatestModified)
{
containerScanInfo.LastSweepCycleLatestModified = containerScanInfo.CurrentSweepCycleLatestModified;
}
return(newBlobs);
}
public BlobContainerService(BlobContainerClient blobContainerClient)
{
_blobContainerClient = blobContainerClient;
}
public async Task ClaimOwnershipAsyncReturnsOnlyTheSuccessfullyClaimedOwnership()
{
await using (StorageScope storageScope = await StorageScope.CreateAsync())
{
var storageConnectionString = StorageTestEnvironment.StorageConnectionString;
var containerClient = new BlobContainerClient(storageConnectionString, storageScope.ContainerName);
var partitionManager = new BlobPartitionManager(containerClient);
var ownershipList = new List <PartitionOwnership>();
var ownershipCount = 5;
for (int i = 0; i < ownershipCount; i++)
{
ownershipList.Add(
new MockPartitionOwnership
(
"namespace",
"eventHubName",
"consumerGroup",
"ownerIdentifier",
$"partitionId_{ i }"
));
}
await partitionManager.ClaimOwnershipAsync(ownershipList);
// The ETags must have been set by the partition manager.
var eTags = ownershipList.Select(ownership => ownership.ETag).ToList();
ownershipList.Clear();
// Use a valid eTag when 'i' is odd. This way, we can expect 'ownershipCount / 2' successful
// claims (rounded down).
var expectedClaimedCount = ownershipCount / 2;
for (int i = 0; i < ownershipCount; i++)
{
ownershipList.Add(
new MockPartitionOwnership
(
"namespace",
"eventHubName",
"consumerGroup",
"ownerIdentifier",
$"partitionId_{ i }",
offset: i,
eTag: i % 2 == 1 ? eTags[i] : null
));
}
IEnumerable <PartitionOwnership> claimedOwnershipList = await partitionManager.ClaimOwnershipAsync(ownershipList);
IEnumerable <PartitionOwnership> expectedOwnership = ownershipList.Where(ownership => ownership.Offset % 2 == 1);
Assert.That(claimedOwnershipList, Is.Not.Null);
Assert.That(claimedOwnershipList.Count, Is.EqualTo(expectedClaimedCount));
var index = 0;
foreach (PartitionOwnership ownership in claimedOwnershipList.OrderBy(ownership => ownership.PartitionId))
{
Assert.That(ownership.IsEquivalentTo(expectedOwnership.ElementAt(index)), Is.True, $"Ownership of partition '{ ownership.PartitionId }' should be equivalent.");
++index;
}
}
}
public AzureBlobStorageService(IConfiguration config)
{
_config = config;
_blobClient = new BlobServiceClient(config["AzureBlobSettings:ConnectionString"]);
_container = _blobClient.GetBlobContainerClient(config["AzureBlobSettings:ContainerName"]);
}
