public void InitializeContainer(string container)
{
var blobClient = StorageAccount.CreateCloudBlobClient();
blobContainer = blobClient.GetContainerReference(container);
blobContainer.CreateIfNotExists();
blobContainer.SetPermissions(new BlobContainerPermissions() { PublicAccess = BlobContainerPublicAccessType.Blob });
}
public BlobHelper(String container)
{
// get or create blobContainer to communicate with Azure storage service
var blobClient = StorageAccount.CreateCloudBlobClient();
blobContainer = blobClient.GetContainerReference(container);
blobContainer.CreateIfNotExists();
blobContainer.SetPermissions(new BlobContainerPermissions() { PublicAccess = BlobContainerPublicAccessType.Blob });
}
/// <summary>
/// Initializes a new instance of the <see cref="AzureBlobStorage" /> class.
/// </summary>
/// <param name="account">The Azure account to use.</param>
/// <param name="containerAddress">The name of the Azure blob container to use for uploaded blobs.</param>
public AzureBlobStorage(CloudStorageAccount account, string containerAddress) {
Requires.NotNull(account, "account");
Requires.NotNullOrEmpty(containerAddress, "containerAddress");
Requires.Argument(DesktopUtilities.IsValidBlobContainerName(containerAddress), "containerAddress", "Invalid container name.");
this.account = account;
this.client = this.account.CreateCloudBlobClient();
this.container = this.client.GetContainerReference(containerAddress);
}
public void SetUp() {
AzureStorageConfig.RegisterConfiguration();
var testContainerName = "unittests" + Guid.NewGuid().ToString();
var testTableName = "unittests" + Guid.NewGuid().ToString().Replace("-", string.Empty);
var account = CloudStorageAccount.DevelopmentStorageAccount;
var client = account.CreateCloudBlobClient();
this.tableClient = account.CreateCloudTableClient();
this.tableClient.GetTableReference(testTableName).CreateIfNotExists();
this.container = client.GetContainerReference(testContainerName);
this.container.CreateContainerWithPublicBlobsIfNotExistAsync();
this.controller = new InboxControllerForTest(this.container.Name, testTableName, CloudConfigurationName);
}
public static IEnumerable<CloudBlobDirectory> ListSubdirectories(CloudBlobContainer cloudBlobContainer)
{
throw new NotImplementedException();
var l = cloudBlobContainer.Uri.AbsolutePath.Length;
return (from blob in cloudBlobContainer.ListBlobs().Select(each => each.Uri.AbsolutePath.Substring(l + 1))
let i = blob.IndexOf('/')
where i > -1
select blob.Substring(0, i)).Distinct().Select(cloudBlobContainer.GetDirectoryReference);
}
private static async System.Threading.Tasks.Task MoveBlobAsync(ICloudBlob sourceBlobRef, CloudBlobContainer sourceContainer, CloudBlobContainer destContainer)
{
CloudBlockBlob destBlob = destContainer.GetBlockBlobReference(sourceBlobRef.Name);
await destBlob.StartCopyAsync(new Uri(GetShareAcccessUri(sourceBlobRef.Name + ".zip", sourceContainer)));
ICloudBlob destBlobRef = await destContainer.GetBlobReferenceFromServerAsync(sourceBlobRef.Name);
while (destBlobRef.CopyState.Status == CopyStatus.Pending)
{
Console.WriteLine($"Blob: {destBlobRef.Name}, Copied: {destBlobRef.CopyState.BytesCopied ?? 0} of {destBlobRef.CopyState.TotalBytes ?? 0}");
await System.Threading.Tasks.Task.Delay(500);
destBlobRef = await destContainer.GetBlobReferenceFromServerAsync(sourceBlobRef.Name);
}
Console.WriteLine($"Blob: {destBlob.Name} Complete");
// Remove the source blob
bool blobExisted = await sourceBlobRef.DeleteIfExistsAsync();
}
/// <summary>
/// Get blob with copy status by name
/// </summary>
/// <param name="containerName">Container name</param>
/// <param name="blobName">blob name</param>
/// <returns>CloudBlob object</returns>
private CloudBlob GetCloudBlobObject(string containerName, string blobName)
{
CloudBlobContainer container = Channel.GetContainerReference(containerName);
return(GetCloudBlobObject(container, blobName));
}
/// <summary>
/// Loads the most recent Device telemetry.
/// </summary>
/// <param name="deviceId">
/// The ID of the Device for which telemetry should be returned.
/// </param>
/// <param name="minTime">
/// The minimum time of record of the telemetry that should be returned.
/// </param>
/// <returns>
/// Telemetry for the Device specified by deviceId, inclusively since
/// minTime.
/// </returns>
public async Task <IEnumerable <DeviceTelemetryModel> > LoadLatestDeviceTelemetryAsync(
string deviceId,
IList <DeviceTelemetryFieldModel> telemetryFields,
DateTime minTime)
{
IEnumerable <DeviceTelemetryModel> result = new DeviceTelemetryModel[0];
CloudBlobContainer container =
await BlobStorageHelper.BuildBlobContainerAsync(this._telemetryStoreConnectionString, this._telemetryContainerName);
IEnumerable <IListBlobItem> blobs =
await BlobStorageHelper.LoadBlobItemsAsync(
async (token) =>
{
return(await container.ListBlobsSegmentedAsync(
this._telemetryDataPrefix,
true,
BlobListingDetails.None,
null,
token,
null,
null));
});
blobs = blobs
.OrderByDescending(t => BlobStorageHelper.ExtractBlobItemDate(t));
CloudBlockBlob blockBlob;
IEnumerable <DeviceTelemetryModel> blobModels;
foreach (IListBlobItem blob in blobs)
{
if ((blockBlob = blob as CloudBlockBlob) == null)
{
continue;
}
// Translate LastModified to local time zone. DateTimeOffsets
// don't do this automatically. This is for equivalent behavior
// with parsed DateTimes.
if ((blockBlob.Properties != null) &&
blockBlob.Properties.LastModified.HasValue &&
(blockBlob.Properties.LastModified.Value.LocalDateTime < minTime))
{
break;
}
try
{
blobModels = await LoadBlobTelemetryModelsAsync(blockBlob, telemetryFields);
}
catch
{
continue;
}
if (blobModels == null)
{
break;
}
int preFilterCount = blobModels.Count();
blobModels =
blobModels.Where(
t =>
(t != null) &&
t.Timestamp.HasValue &&
t.Timestamp.Value >= minTime);
if (preFilterCount == 0)
{
break;
}
result = result.Concat(blobModels);
}
if (!string.IsNullOrEmpty(deviceId))
{
result = result.Where(t => t.DeviceId == deviceId);
}
return(result);
}
public async static void Run([BlobTrigger("testinvocation/{name}", Connection = "AzureWebJobsStorage")] Stream myBlob, string name, ILogger log)
{
try
{
if (name.Contains(".test"))
{
Engine engine = new Engine(log);
Search search = new Search(engine);
Model model = new Model(engine, search);
// Instanciate test labeling class that matches the configured labeling solution.
// Note: test labeling class must follow the nameing convention - {environment variable naming soluition name}TestLabeler
// If this convention is not followed ML Professoar cannot find the test labeling class to instanciate.
string labelingSolutionName = engine.GetEnvironmentVariable("labelingSolutionName");
Type classType = Type.GetType($"semisupervisedFramework.{labelingSolutionName}TestLabeler");
Test test = (Test)Activator.CreateInstance(classType, new Object[] { engine, search, model });
// Inititialize test result variables
string noTrainedModelTestResults = "";
string trainModelTestResults = "";
string evaluatePassingDataTestResults = "";
string evaluateFailingDataTestResults = "";
string labelDataTestResults = "";
string loadLabeledDataTestResults = "";
string LoadLabelsTestResults = "";
// get a reference to the invocation blob file so that it can be deleted after the test is launched.
string storageConnection = engine.GetEnvironmentVariable("AzureWebJobsStorage");
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnection);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer testDataContainer = blobClient.GetContainerReference("testinvocation");
CloudBlockBlob testInitiationBlob = testDataContainer.GetBlockBlobReference(name);
string testResults = "Initialized";
testInitiationBlob.DeleteIfExists();
switch (name)
{
case "TestAll.test":
noTrainedModelTestResults = await test.NoTrainedModel();
labelDataTestResults = await test.LabelData();
loadLabeledDataTestResults = await test.LoadLabeledData();
trainModelTestResults = await test.TrainModel();
evaluatePassingDataTestResults = await test.EvaluatePassingData();
evaluateFailingDataTestResults = await test.EvaluateFailingData();
if (noTrainedModelTestResults.Contains("Failed:") || trainModelTestResults.Contains("Failed:") || evaluatePassingDataTestResults.Contains("Failed:"))
{
testResults = $"\nFailed: Some test failures exist:\n{noTrainedModelTestResults}\n{labelDataTestResults}\n{trainModelTestResults}\n{evaluatePassingDataTestResults}\n{evaluateFailingDataTestResults}";
log.LogInformation(testResults);
}
else
{
testResults = $"\nAll test passed! with results:\n{noTrainedModelTestResults}\n{labelDataTestResults}\n{trainModelTestResults}\n{evaluatePassingDataTestResults}\n{evaluateFailingDataTestResults}";
log.LogInformation(testResults);
}
break;
case "NoTrainedModel.test":
noTrainedModelTestResults = await test.NoTrainedModel();
if (noTrainedModelTestResults.Contains("Failed:"))
{
testResults = $"\nNo Trained Model test failed: Some test failures exist:\n{noTrainedModelTestResults}";
log.LogInformation(testResults);
}
else
{
testResults = $"\nNo Trained Model test passed! with results: {noTrainedModelTestResults}";
log.LogInformation(testResults);
}
break;
case "TrainModel.test":
trainModelTestResults = await test.TrainModel();
if (trainModelTestResults.Contains("Failed:"))
{
testResults = $"\nTrain Model test failed: Some test failures exist:\n{trainModelTestResults}";
log.LogInformation(testResults);
}
else
{
testResults = $"\nTrain Model test passed! with results: {trainModelTestResults}";
log.LogInformation(testResults);
}
break;
case "EvaluatePassingData.test":
evaluatePassingDataTestResults = await test.EvaluatePassingData();
if (evaluatePassingDataTestResults.Contains("Failed:"))
{
testResults = $"\nEvaluate Passing Data test failed: Some test failures exist:\n{evaluatePassingDataTestResults}";
log.LogInformation(testResults);
}
else
{
testResults = $"\nEvaluate Passing Data test passed! with results: {evaluatePassingDataTestResults}";
log.LogInformation(testResults);
}
break;
case "EvaluateFailingData.test":
evaluateFailingDataTestResults = await test.EvaluateFailingData();
if (evaluateFailingDataTestResults.Contains("Failed:"))
{
testResults = $"\nEvaluate Failing Data test failed: Some test failures exist:\n{evaluateFailingDataTestResults}";
log.LogInformation(testResults);
}
else
{
testResults = $"\nEvaluate Failing Data test passed! with results: {evaluateFailingDataTestResults}";
log.LogInformation(testResults);
}
break;
case "LabelData.test":
labelDataTestResults = await test.LabelData();
if (labelDataTestResults.Contains("Failed:"))
{
testResults = $"\nLabel Data test failed: Some test failures exist:\n{labelDataTestResults}";
log.LogInformation(testResults);
}
else
{
testResults = $"\nLabel Data test passed! with results: {labelDataTestResults}";
log.LogInformation(testResults);
}
break;
case "LoadLabeledData.test":
loadLabeledDataTestResults = await test.LoadLabeledData();
if (loadLabeledDataTestResults.Contains("Failed:"))
{
testResults = $"\nLoad Labeled Data test failed: Some test failures exist:\n{loadLabeledDataTestResults}";
log.LogInformation(testResults);
}
else
{
testResults = $"\nLoad Labeled Data test passed! with results: {loadLabeledDataTestResults}";
log.LogInformation(testResults);
}
break;
case "LoadLabels.test":
LoadLabelsTestResults = await test.LoadLabels();
if (LoadLabelsTestResults.Contains("Failed:"))
{
testResults = $"\nLoad Labeles test failed: Some test failures exist:\n{LoadLabelsTestResults}";
log.LogInformation(testResults);
}
else
{
testResults = $"\nLoad Labeles test passed! with results: {LoadLabelsTestResults}";
log.LogInformation(testResults);
}
break;
}
//await Task.Delay(75000);
//int waitingAttempts = 0;
//do
//{
// waitingAttempts++;
// await Task.Delay(5000);
// log.LogInformation($"noTrainedModelTest checked {waitingAttempts} times. Will check 30 times before continuing.");
//} while (waitingAttempts <= 30);
CloudBlockBlob testResultsCloudBlob = testDataContainer.GetBlockBlobReference("TestResults-" + Guid.NewGuid().ToString() + ".txt");
Stream MemStream = new MemoryStream(Encoding.UTF8.GetBytes(testResults));
if (MemStream.Length != 0)
{
testResultsCloudBlob.UploadFromStream(MemStream);
}
else
{
throw (new ZeroLengthFileException("\nencoded JSON memory stream is zero length and cannot be writted to blob storage"));
}
log.LogInformation($"TestTrigger complete.");
}
}
catch (Exception e)
{
log.LogInformation($"TestTrigger failed with {e.Message}.");
}
}
public AzureBlobStorage(string connectionString, string containerName)
{
_container = GetContainerReference(connectionString, containerName);
}
public AzureCacheProvider(CloudBlobContainer container)
{
_container = container;
}
/// <summary>
/// Gets the latest BLOB.
/// </summary>
/// <param name="container">The container.</param>
/// <returns></returns>
private CloudBlockBlob GetLatestBlob(CloudBlobContainer container)
{
container.FetchAttributes();
var latest = container.Metadata[Constants.LastUploadedVersion];
return container.GetBlockBlobReference(latest);
}
public TestFixture()
{
RandomNameResolver nameResolver = new RandomNameResolver();
JobHostConfiguration hostConfiguration = new JobHostConfiguration()
{
NameResolver = nameResolver,
TypeLocator = new FakeTypeLocator(typeof(BlobBindingEndToEndTests)),
};
Config = hostConfiguration;
_storageAccount = CloudStorageAccount.Parse(hostConfiguration.StorageConnectionString);
CloudBlobClient blobClient = _storageAccount.CreateCloudBlobClient();
CloudBlobContainer blobContainer = blobClient.GetContainerReference(nameResolver.ResolveInString(ContainerName));
Assert.False(blobContainer.Exists());
blobContainer.Create();
CloudBlobContainer pageBlobContainer = blobClient.GetContainerReference(nameResolver.ResolveInString(PageBlobContainerName));
Assert.False(pageBlobContainer.Exists());
pageBlobContainer.Create();
CloudBlobContainer hierarchicalBlobContainer = blobClient.GetContainerReference(nameResolver.ResolveInString(HierarchicalBlobContainerName));
Assert.False(hierarchicalBlobContainer.Exists());
hierarchicalBlobContainer.Create();
Host = new JobHost(hostConfiguration);
Host.Start();
// upload some test blobs
CloudBlockBlob blob = blobContainer.GetBlockBlobReference("blob1");
blob.UploadText(TestData);
blob = blobContainer.GetBlockBlobReference("blob2");
blob.UploadText(TestData);
blob = blobContainer.GetBlockBlobReference("blob3");
blob.UploadText(TestData);
blob = blobContainer.GetBlockBlobReference("file1");
blob.UploadText(TestData);
blob = blobContainer.GetBlockBlobReference("file2");
blob.UploadText(TestData);
// add a couple hierarchical blob paths
blob = hierarchicalBlobContainer.GetBlockBlobReference("sub/blob1");
blob.UploadText(TestData);
blob = hierarchicalBlobContainer.GetBlockBlobReference("sub/blob2");
blob.UploadText(TestData);
blob = hierarchicalBlobContainer.GetBlockBlobReference("sub/sub/blob3");
blob.UploadText(TestData);
blob = hierarchicalBlobContainer.GetBlockBlobReference("blob4");
blob.UploadText(TestData);
byte[] bytes = new byte[512];
byte[] testBytes = Encoding.UTF8.GetBytes(TestData);
for (int i = 0; i < testBytes.Length; i++)
{
bytes[i] = testBytes[i];
}
CloudPageBlob pageBlob = pageBlobContainer.GetPageBlobReference("blob1");
pageBlob.UploadFromByteArray(bytes, 0, bytes.Length);
pageBlob = pageBlobContainer.GetPageBlobReference("blob2");
pageBlob.UploadFromByteArray(bytes, 0, bytes.Length);
}
public void RequestResultVerifyXml()
{
Uri baseAddressUri = new Uri(TestBase.TargetTenantConfig.BlobServiceEndpoint);
CloudBlobClient blobClient = new CloudBlobClient(baseAddressUri, TestBase.StorageCredentials);
CloudBlobContainer container = blobClient.GetContainerReference(Guid.NewGuid().ToString("N"));
OperationContext opContext = new OperationContext();
Assert.IsNull(opContext.LastResult);
container.Exists(null, opContext);
Assert.IsNotNull(opContext.LastResult);
// We do not have precision at milliseconds level. Hence, we need
// to recreate the start DateTime to be able to compare it later.
DateTime start = opContext.LastResult.StartTime;
start = new DateTime(start.Year, start.Month, start.Day, start.Hour, start.Minute, start.Second);
DateTime end = opContext.LastResult.EndTime;
end = new DateTime(end.Year, end.Month, end.Day, end.Hour, end.Minute, end.Second);
XmlWriterSettings settings = new XmlWriterSettings();
settings.Indent = true;
StringBuilder sb = new StringBuilder();
using (XmlWriter writer = XmlWriter.Create(sb, settings))
{
opContext.LastResult.WriteXml(writer);
}
RequestResult retrResult = new RequestResult();
using (XmlReader reader = XmlReader.Create(new StringReader(sb.ToString())))
{
retrResult.ReadXml(reader);
}
Assert.AreEqual(opContext.LastResult.RequestDate, retrResult.RequestDate);
Assert.AreEqual(opContext.LastResult.ServiceRequestID, retrResult.ServiceRequestID);
Assert.AreEqual(start, retrResult.StartTime);
Assert.AreEqual(end, retrResult.EndTime);
Assert.AreEqual(opContext.LastResult.HttpStatusCode, retrResult.HttpStatusCode);
Assert.AreEqual(opContext.LastResult.HttpStatusMessage, retrResult.HttpStatusMessage);
Assert.AreEqual(opContext.LastResult.ContentMd5, retrResult.ContentMd5);
Assert.AreEqual(opContext.LastResult.Etag, retrResult.Etag);
// Now test with no indentation
sb = new StringBuilder();
using (XmlWriter writer = XmlWriter.Create(sb))
{
opContext.LastResult.WriteXml(writer);
}
retrResult = new RequestResult();
using (XmlReader reader = XmlReader.Create(new StringReader(sb.ToString())))
{
retrResult.ReadXml(reader);
}
Assert.AreEqual(opContext.LastResult.RequestDate, retrResult.RequestDate);
Assert.AreEqual(opContext.LastResult.ServiceRequestID, retrResult.ServiceRequestID);
Assert.AreEqual(start, retrResult.StartTime);
Assert.AreEqual(end, retrResult.EndTime);
Assert.AreEqual(opContext.LastResult.HttpStatusCode, retrResult.HttpStatusCode);
Assert.AreEqual(opContext.LastResult.HttpStatusMessage, retrResult.HttpStatusMessage);
Assert.AreEqual(opContext.LastResult.ContentMd5, retrResult.ContentMd5);
Assert.AreEqual(opContext.LastResult.Etag, retrResult.Etag);
}
public static void CloudBlobContainerBinding_WithModelBinding(
[QueueTrigger("testqueue")] TestPoco poco,
[Blob("{A}")] CloudBlobContainer container)
{
CloudBlobContainerBinding(container);
}
public BlobAppendOnlyStore(CloudBlobContainer container)
{
_container = container;
}
public string HiveOutput(string q)
{
Trace.WriteLine("Entering HiveOutput method");
Trace.TraceInformation("Executing HiveOutput method at " + DateTime.Now.ToLongTimeString());
//Defining MapReduce Job
HiveJobCreateParameters hiveJobDefinition = new HiveJobCreateParameters()
{
JobName = "job",
StatusFolder = "/TableListFolder",
Query = q
};
Guid subscriptionId = new Guid("44fbb137-edbb-4044-9db9-0e1333e137cf"); //your-subscription-id
string clusterName = "tbanihumcluster";
// Get the certificate object from certificate store using the friendly name to identify it
X509Store store = new X509Store(StoreName.My);
store.Open(OpenFlags.ReadOnly);
X509Certificate2 cert = store.Certificates.Cast <X509Certificate2>().First(item => item.FriendlyName == "Azdem187U23713U-1-8-2015-credentials");
var creds = new JobSubmissionCertificateCredential(subscriptionId, cert, clusterName);
// Create a hadoop client to connect to HDInsight
var jobClient = JobSubmissionClientFactory.Connect(creds);
// Run the MapReduce job
JobCreationResults jobResults = jobClient.CreateHiveJob(hiveJobDefinition);
// Wait for the job to complete
WaitForJobCompletion(jobResults, jobClient);
// Hive job output
System.IO.Stream stream = jobClient.GetJobOutput(jobResults.JobId);
System.IO.StreamReader reader = new System.IO.StreamReader(stream);
string value = reader.ReadToEnd();
value = value.Replace('\t', ',');
// Retrieve storage account from connection string.
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString"));
// Create the blob client.
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Retrieve reference to a previously created container.
CloudBlobContainer container = blobClient.GetContainerReference("mycontainer");
container.CreateIfNotExists();
// Retrieve reference to a blob named "myblob".
CloudBlockBlob blockBlob = container.GetBlockBlobReference("myblob");
blockBlob.UploadText(value);
Trace.WriteLine("Leaving HiveOutput method");
return(value);
}
/// <summary>Initializes a new instance of the <see cref="StorageBlobContainer"/> class.</summary>
/// <param name="parent">The parent blob client.</param>
/// <param name="sdk">The SDK container to wrap.</param>
public StorageBlobContainer(IStorageBlobClient parent, CloudBlobContainer sdk)
{
_parent = parent;
_sdk = sdk;
}
public AzureStorageMultipartFormDataStreamProvider(CloudBlobContainer blobContainer, string folder) : base("azure")
{
_blobContainer = blobContainer;
this.folder = folder;
}
/// <summary>
/// Checks if a container exists.
/// </summary>
/// <param name="container">Container to check for</param>
/// <returns>Flag indicating the existence of the container</returns>
private static bool Exists(CloudBlobContainer container)
{
try
{
container.FetchAttributes();
return true;
}
catch (StorageClientException e)
{
if (e.ErrorCode == StorageErrorCode.ResourceNotFound)
{
return false;
}
throw;
}
}
public static void GetAnalyticsLogs(CloudBlobClient blobClient, CloudTableClient tableClient, StorageAccount strAccount)
{
try
{
DateTime time = DateTime.UtcNow;
int inactivityDays = Int16.Parse(InactivityDaysForStorageAccount) * -1;
CloudAnalyticsClient analyticsClient = new CloudAnalyticsClient(blobClient.StorageUri, tableClient.StorageUri, tableClient.Credentials);
IEnumerable <ICloudBlob> results = analyticsClient.ListLogs(StorageService.Blob, time.AddDays(inactivityDays), null, LoggingOperations.All, BlobListingDetails.Metadata, null, null);
List <ICloudBlob> logs = results.ToList();
int nonLogEntries = 0;
int onlyListLogEntries = 0;
//Creating folders for storage account
string storageName = blobClient.BaseUri.Host.Split('.')[0];
string tempPath = Path.GetTempPath();
string storagePath = (tempPath + "/logs/" + storageName + "/");
Directory.CreateDirectory(storagePath);
//Download the log files
foreach (var item in logs)
{
string name = ((CloudBlockBlob)item).Name;
CloudBlobContainer container = blobClient.GetContainerReference("$logs");
CloudBlockBlob blockBlob = container.GetBlockBlobReference(name);
//specify the directory without file name
string sub_folder = name.Remove(name.LastIndexOf("/") + 1);
string path = (storagePath + sub_folder);
//create the directory if it does not exist.
Directory.CreateDirectory(path);
//specify the file full path
string file_path = (storagePath + name);
using (var fileStream = File.Create(file_path))
{
blockBlob.DownloadToStream(fileStream);
}
}
if (logs.Count > 0)
{
foreach (string file in Directory.GetFiles(storagePath, "*.log", SearchOption.AllDirectories))
{
var contents = File.ReadLines(file);
foreach (var line in contents)
{
string operationType = line.Split(';')[2];
//Ignoring the log entries for fetching logs from logs container & BlobPreflightRequest logs also.
if (!(line.Contains("$logs") || line.Contains("%24logs") || line.Contains("BlobPreflightRequest")))
{
if (operationType.Equals("GetBlobServiceProperties") || operationType.Equals("ListBlobs") || operationType.Equals("ListContainers") || operationType.Equals("GetContainerServiceMetadata"))
{
onlyListLogEntries++;
}
else
{
nonLogEntries++;
}
}
}
}
}
ServiceProperties serviceProperties = blobClient.GetServiceProperties();
bool isLoggingDisabled = serviceProperties.Logging.LoggingOperations.ToString() == "None";
strAccount.isDiagnosticsEnabled = true;
if (isLoggingDisabled)
{
Console.WriteLine("Information: Diagnostics Settings are enabled for this account but logging is still disabled. Please change the Diagnostics Settings.");
}
else
{
strAccount.isLoggingEnabled = true;
if (logs.Count == 0)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("Warning: Diagnostics Settings are enabled for this account. Either this storage account has not been used from last " + InactivityDaysForStorageAccount + " days or Logs are not available for this storage account.");
Console.ResetColor();
strAccount.isNotUsed = true;
}
else
{
//If there are log entries other than the list logs means the storage account is in use
if (onlyListLogEntries > 0 && nonLogEntries == 0)
{
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine("Information: There are only log entries for GetBlobServiceProperties, ListBlobs, ListContainers from last " + InactivityDaysForStorageAccount + " days for this storage account.");
Console.ResetColor();
strAccount.onlyListLogEntries = true;
}
else if (onlyListLogEntries == 0 && nonLogEntries == 0)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("Warning: Either the storage account has not been used from last " + InactivityDaysForStorageAccount + " days for this storage account or complete logs are not available.");
Console.ResetColor();
strAccount.isNotUsed = true;
}
else
{
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("Storage account is being used in last " + InactivityDaysForStorageAccount + " days!");
Console.ResetColor();
strAccount.isUsed = true;
}
}
}
}
catch (StorageException e)
{
//If logs container is not present, means diagnostics are not enabled
if (e.RequestInformation.ErrorCode == "ContainerNotFound")
{
strAccount.isDiagnosticsEnabled = false;
Console.WriteLine("Information: Please enable Diagnostics Settings to check the logs/activity of the storage account.");
}
else
{
Console.WriteLine("Exception occured:" + e.RequestInformation.ErrorCode);
}
}
finally
{
storageAccounts.Add(strAccount);
}
}
public async Task <IHttpActionResult> PostReport(eLublin.Web.Models.Api.CommitReport commitReport)
{
if (!ModelState.IsValid)
{
return(BadRequest(ModelState));
}
decimal tempDec = 0;
if (!decimal.TryParse(commitReport.lat, System.Globalization.NumberStyles.AllowDecimalPoint, System.Globalization.NumberFormatInfo.InvariantInfo, out tempDec) ||
!decimal.TryParse(commitReport.lng, System.Globalization.NumberStyles.AllowDecimalPoint, System.Globalization.NumberFormatInfo.InvariantInfo, out tempDec))
{
return(BadRequest("zle koordynaty"));
}
else
{
//return BadRequest("cos zjebane"+ decimal.Parse(commitReport.lat));
}
// Retrieve storage account from connection string.
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(System.Configuration.ConfigurationManager.ConnectionStrings["StorageConnectionString"].ConnectionString);
// Create the blob client.
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Retrieve reference to a previously created container.
CloudBlobContainer container = blobClient.GetContainerReference("elublin");
// Retrieve reference to a blob named "myblob".
CloudBlockBlob blockBlob = container.GetBlockBlobReference(Guid.NewGuid().ToString() + ".jpg");
// Create or overwrite the "myblob" blob with contents from a local file.
await blockBlob.UploadFromByteArrayAsync(commitReport.image, 0, commitReport.image.Length);
var userEmail = User.Identity.GetUserName();
var _userInfo = await db.UserInfoes.FirstAsync(x => x.email == userEmail);
var report = new Report {
glosy = 0,
imagePath = blockBlob.StorageUri.PrimaryUri.ToString(),
lat = commitReport.lat,
lng = commitReport.lng,
tekst = commitReport.tekst,
userInfoId = _userInfo.id,
timeAdded = DateTime.Now
};
//FileStream fs = new FileStream(@"C:\Users\Dom\Desktop\dupa.jpg", FileMode.OpenOrCreate, FileAccess.ReadWrite);
//fs.Write(commitReport.image, 0, commitReport.image.Length);
//fs.Close();
db.Reports.Add(report);
try
{
await db.SaveChangesAsync();
}
catch (DbUpdateException)
{
if (ReportExists(report.id))
{
return(Conflict());
}
else
{
throw;
}
}
return(CreatedAtRoute("DefaultApi", new { id = report.id }, report));
}
/// <summary>
/// remove azure blob
/// </summary>
/// <param name="container">CloudBlobContainer object</param>
/// <param name="blobName">blob name</param>
/// <returns>true if the blob is removed successfully, false if user cancel the remove operation</returns>
internal async Task RemoveAzureBlob(long taskId, IStorageBlobManagement localChannel, CloudBlobContainer container, string blobName)
{
if (!NameUtil.IsValidBlobName(blobName))
{
throw new ArgumentException(String.Format(Resources.InvalidBlobName, blobName));
}
ValidatePipelineCloudBlobContainer(container);
AccessCondition accessCondition = null;
BlobRequestOptions requestOptions = null;
CloudBlob blob = null;
try
{
blob = await localChannel.GetBlobReferenceFromServerAsync(container, blobName, accessCondition,
requestOptions, OperationContext, CmdletCancellationToken);
}
catch (InvalidOperationException)
{
blob = null;
}
if (null == blob && container.ServiceClient.Credentials.IsSharedKey)
{
throw new ResourceNotFoundException(String.Format(Resources.BlobNotFound, blobName, container.Name));
}
else
{
//Construct the blob as CloudBlockBlob no matter what's the real blob type
//We can't get the blob type if Credentials only have the delete permission and don't have read permission.
blob = container.GetBlockBlobReference(blobName);
}
await RemoveAzureBlob(taskId, localChannel, blob, true);
}
/// <summary>
/// Retrieves collection of Task Definitions, ordered by version, latest first
/// </summary>
/// <param name="takeCount">default 1 signifies latest, -1 all</param>
/// <returns></returns>
public IEnumerable <TaskModel> LoadTaskDefinitionsOrderByLatestVersion(int takeCount = 1)
{
List <TaskModel> tasks = new List <TaskModel>();
var storageConnectionString = WebConfigurationManager.AppSettings["storageConnectionString"];
var containerName = "task-definitions";
CloudStorageAccount cloudStorageAccount;
if (CloudStorageAccount.TryParse(storageConnectionString, account: out cloudStorageAccount))
{
CloudBlobClient blobClient = cloudStorageAccount.CreateCloudBlobClient();
CloudBlobContainer cloudBlobContainer = blobClient.GetContainerReference(containerName);
foreach (IListBlobItem blobItem in cloudBlobContainer.ListBlobs())
{
if (blobItem is CloudBlobDirectory)
{
CloudBlobDirectory directory = (CloudBlobDirectory)blobItem;
IEnumerable <IListBlobItem> blobs = directory.ListBlobs(true);
ICloudBlob blockBlob;
foreach (var blob in blobs)
{
if (blob is CloudBlockBlob)
{
blockBlob = blob as CloudBlockBlob;
if (blockBlob.Name.Contains(".tasks"))
{
//string text = blockBlob.d.DownloadTextAsync().Result;
var parts = blockBlob.Name.Split('/');
tasks.Add(new TaskModel()
{
Task = Path.GetFileNameWithoutExtension(blockBlob.Name),
FullName = blockBlob.Name,
Path = blockBlob.StorageUri.PrimaryUri.ToString(),
Project = parts[parts.Length - 3],
Definition = "N/A",
Version = parts[parts.Length - 2],
});
}
}
}
}
}
}
else
{
//ToDo: define logger
// Otherwise, let the user know that they need to define the environment variable.
//Console.WriteLine(
// "A connection string has not been defined in the system environment variables. " +
// "Add a environment variable named 'storageconnectionstring' with your storage " +
// "connection string as a value.");
//Console.WriteLine("Press any key to exit the sample application.");
//Console.ReadLine();
}
return(tasks.OrderByDescending(e => AsVersionNumber(e.Version)));
}
/// <summary>
/// remove azure blob
/// </summary>
/// <param name="containerName">container name</param>
/// <param name="blobName">blob name</param>
/// <returns>true if the blob is removed successfully, false if user cancel the remove operation</returns>
internal async Task RemoveAzureBlob(long taskId, IStorageBlobManagement localChannel, string containerName, string blobName)
{
CloudBlobContainer container = localChannel.GetContainerReference(containerName);
await RemoveAzureBlob(taskId, localChannel, container, blobName);
}
/// <summary>
/// Initializes a new instance of the <see cref="AzureBlobCache"/> class.
/// </summary>
/// <param name="requestPath">
/// The request path for the image.
/// </param>
/// <param name="fullPath">
/// The full path for the image.
/// </param>
/// <param name="querystring">
/// The query string containing instructions.
/// </param>
public AzureBlobCache(string requestPath, string fullPath, string querystring)
: base(requestPath, fullPath, querystring)
{
if (!(cloudCachedBlobContainer is null))
{
return;
}
lock (SyncLock)
{
if (!(cloudCachedBlobContainer is null))
{
return;
}
// Retrieve storage accounts from connection string.
var cloudCachedStorageAccount = CloudStorageAccount.Parse(this.Settings["CachedStorageAccount"]);
// Create the blob clients.
cloudCachedBlobClient = cloudCachedStorageAccount.CreateCloudBlobClient();
// Set cloud cache container.
BlobContainerPublicAccessType accessType = this.Settings.ContainsKey("AccessType")
? (BlobContainerPublicAccessType)Enum.Parse(typeof(BlobContainerPublicAccessType), this.Settings["AccessType"])
: BlobContainerPublicAccessType.Blob;
cloudCachedBlobContainer = CreateContainer(cloudCachedBlobClient, this.Settings["CachedBlobContainer"], accessType);
// Set cloud source container.
string sourceAccount = this.Settings.ContainsKey("SourceStorageAccount")
? this.Settings["SourceStorageAccount"]
: string.Empty;
// Repeat for source if it exists
if (!string.IsNullOrWhiteSpace(sourceAccount))
{
var cloudSourceStorageAccount = CloudStorageAccount.Parse(this.Settings["SourceStorageAccount"]);
CloudBlobClient cloudSourceBlobClient = cloudSourceStorageAccount.CreateCloudBlobClient();
cloudSourceBlobContainer = cloudSourceBlobClient.GetContainerReference(this.Settings["SourceBlobContainer"]);
}
// This setting was added to facilitate streaming of the blob resource directly instead of a redirect. This is beneficial for CDN purposes
// but caution should be taken if not used with a CDN as it will add quite a bit of overhead to the site.
// See: https://github.com/JimBobSquarePants/ImageProcessor/issues/161
// If it's private, we also stream the response rather than redirect.
streamCachedImage = accessType.Equals(BlobContainerPublicAccessType.Off) || (this.Settings.ContainsKey("StreamCachedImage") && string.Equals(this.Settings["StreamCachedImage"], "true", StringComparison.OrdinalIgnoreCase));
cachedCdnRoot = this.Settings.ContainsKey("CachedCDNRoot")
? this.Settings["CachedCDNRoot"]
: cloudCachedBlobContainer.Uri.ToString().TrimEnd(cloudCachedBlobContainer.Name.ToCharArray());
if (this.Settings.ContainsKey("CachedCDNTimeout"))
{
int.TryParse(this.Settings["CachedCDNTimeout"], out int t);
timeout = t;
}
// Do we insert the cache container? This seems to break some setups.
useCachedContainerInUrl = this.Settings.ContainsKey("UseCachedContainerInUrl") &&
!string.Equals(this.Settings["UseCachedContainerInUrl"], "false", StringComparison.OrdinalIgnoreCase);
}
}
public static CloudBlobDirectory GetDirectory(Guid applicationId, Guid sequenceId, Guid sectionId, string pageId, string questionId, CloudBlobContainer container)
{
var applicationFolder = container.GetDirectoryReference(applicationId.ToString());
var sequenceFolder = applicationFolder.GetDirectoryReference(sequenceId.ToString());
var sectionFolder = sequenceFolder.GetDirectoryReference(sectionId.ToString());
var pageFolder = sectionFolder.GetDirectoryReference(pageId.ToLower());
if (questionId is null)
{
return(pageFolder);
}
var questionFolder = pageFolder.GetDirectoryReference(questionId.ToLower());
return(questionFolder);
}
/// <summary>
/// Loads the most recent DeviceTelemetrySummaryModel for a specified Device.
/// </summary>
/// <param name="deviceId">
/// The ID of the Device for which a telemetry summary model should be
/// returned.
/// </param>
/// <param name="minTime">
/// If provided the the minimum time stamp of the summary data that should
/// be loaded.
/// </param>
/// <returns>
/// The most recent DeviceTelemetrySummaryModel for the Device,
/// specified by deviceId.
/// </returns>
public async Task <DeviceTelemetrySummaryModel> LoadLatestDeviceTelemetrySummaryAsync(
string deviceId,
DateTime?minTime)
{
DeviceTelemetrySummaryModel summaryModel = null;
CloudBlobContainer container =
await BlobStorageHelper.BuildBlobContainerAsync(
this._telemetryStoreConnectionString,
this._telemetryContainerName);
IEnumerable <IListBlobItem> blobs =
await BlobStorageHelper.LoadBlobItemsAsync(
async (token) =>
{
return(await container.ListBlobsSegmentedAsync(
this._telemetrySummaryPrefix,
true,
BlobListingDetails.None,
null,
token,
null,
null));
});
blobs = blobs.OrderByDescending(t => BlobStorageHelper.ExtractBlobItemDate(t));
IEnumerable <DeviceTelemetrySummaryModel> blobModels;
CloudBlockBlob blockBlob;
foreach (IListBlobItem blob in blobs)
{
if ((blockBlob = blob as CloudBlockBlob) == null)
{
continue;
}
// Translate LastModified to local time zone. DateTimeOffsets
// don't do this automatically. This is for equivalent behavior
// with parsed DateTimes.
if (minTime.HasValue &&
(blockBlob.Properties != null) &&
blockBlob.Properties.LastModified.HasValue &&
(blockBlob.Properties.LastModified.Value.LocalDateTime < minTime.Value))
{
break;
}
try
{
blobModels = await LoadBlobTelemetrySummaryModelsAsync(blockBlob);
}
catch
{
continue;
}
if (blobModels == null)
{
break;
}
blobModels = blobModels.Where(t => t != null);
if (!string.IsNullOrEmpty(deviceId))
{
blobModels = blobModels.Where(t => t.DeviceId == deviceId);
}
summaryModel = blobModels.LastOrDefault();
if (summaryModel != null)
{
break;
}
}
return(summaryModel);
}
public IEnumerable <IListBlobItem> ListBlob(CloudBlobContainer container)
{
return(container.ListBlobs(null, false));
}
public override void Configure(IFunctionsHostBuilder builder)
{
builder.Services.AddLogging();
builder.Services.AddOptions <AppSettingsModel>()
.Configure <IConfiguration>((settings, configuration) =>
{
configuration.Bind(settings);
});
builder.Services.AddHttpClient();
builder.Services.AddHttpClient <IPexApiClient, PexApiClient>((client) =>
{
client.BaseAddress = new Uri(Environment.GetEnvironmentVariable("PEXAPIBaseURL", EnvironmentVariableTarget.Process));
})
.AddPolicyHandler(GetPexRetryPolicy());
builder.Services.AddScoped <IStorageMappingService>(
provider => new StorageMappingService(
provider.GetService <IDataProtectionProvider>()
));
string storageConnectionString = Environment.GetEnvironmentVariable("StorageConnectionString", EnvironmentVariableTarget.Process);
builder.Services.AddSingleton(provider =>
new Pex2AplosMappingStorage(
Environment.GetEnvironmentVariable("StorageConnectionString", EnvironmentVariableTarget.Process),
provider.GetService <IStorageMappingService>(),
provider.GetService <ILogger <Pex2AplosMappingStorage> >())
.InitTable());
builder.Services.AddSingleton(provider => new PexOAuthSessionStorage(Environment.GetEnvironmentVariable("StorageConnectionString", EnvironmentVariableTarget.Process)).InitTable());
builder.Services.AddSingleton(provider => new Pex2AplosMappingQueue(Environment.GetEnvironmentVariable("StorageConnectionString", EnvironmentVariableTarget.Process)).InitQueue());
builder.Services.AddSingleton(provider => new SyncResultStorage(Environment.GetEnvironmentVariable("StorageConnectionString", EnvironmentVariableTarget.Process)).InitTable());
builder.Services.AddScoped <IAccessTokenDecryptor>(provider => new AplosAccessTokenDecryptor());
builder.Services.AddSingleton(provider =>
{
var syncTransactionsInterval =
Environment.GetEnvironmentVariable("SyncTransactionsIntervalDays", EnvironmentVariableTarget.Process);
if (!int.TryParse(syncTransactionsInterval, out var syncTransactionsIntervalDays))
{
syncTransactionsIntervalDays = 60;
}
var result = new SyncSettingsModel
{
SyncTransactionsIntervalDays = syncTransactionsIntervalDays
};
return(result);
});
builder.Services.AddScoped <IAplosApiClientFactory>(provider => new AplosApiClientFactory(
provider.GetService <IHttpClientFactory>(),
provider.GetService <IAccessTokenDecryptor>(),
provider.GetService <ILogger <AplosApiClientFactory> >()));
builder.Services.AddScoped <IAplosIntegrationMappingService>(provider => new AplosIntegrationMappingService());
builder.Services.AddScoped <IAplosIntegrationService>(provider => new AplosIntegrationService(
provider.GetService <IOptions <AppSettingsModel> >(),
provider.GetService <IAplosApiClientFactory>(),
provider.GetService <IAplosIntegrationMappingService>(),
provider.GetService <IPexApiClient>(),
provider.GetService <SyncResultStorage>(),
provider.GetService <Pex2AplosMappingStorage>()));
var dataProtectionApplicationName = Environment.GetEnvironmentVariable("DataProtectionApplicationName", EnvironmentVariableTarget.Process);
var dataProtectionBlobContainer = Environment.GetEnvironmentVariable("DataProtectionBlobContainer", EnvironmentVariableTarget.Process);
var dataProtectionBlobName = Environment.GetEnvironmentVariable("DataProtectionBlobName", EnvironmentVariableTarget.Process);
var dataProtectionKeyIdentifier = Environment.GetEnvironmentVariable("DataProtectionKeyIdentifier", EnvironmentVariableTarget.Process);
var storageAccount = CloudStorageAccount.Parse(storageConnectionString);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer blobContainer = blobClient.GetContainerReference(dataProtectionBlobContainer);
blobContainer.CreateIfNotExistsAsync();
var keyVaultClient = new KeyVaultClient(new KeyVaultClient.AuthenticationCallback(new AzureServiceTokenProvider().KeyVaultTokenCallback));
builder.Services
.AddDataProtection()
.SetApplicationName(dataProtectionApplicationName)
.PersistKeysToAzureBlobStorage(
blobContainer,
dataProtectionBlobName)
.ProtectKeysWithAzureKeyVault(
keyVaultClient,
dataProtectionKeyIdentifier)
.DisableAutomaticKeyGeneration();
}
public IEnumerable <T> ListBlob <T>(CloudBlobContainer container)
{
return(container.ListBlobs(null, false).Cast <T>());
}
public static async Task <HttpResponseMessage> Run([HttpTrigger("POST")] HttpRequestMessage req, [Blob("emails")] CloudBlobContainer emailContainer, [Queue("emails")] IAsyncCollector <string> emailQueue, [Queue("emailtriggers")] IAsyncCollector <string> triggerQueue, TraceWriter log, ExecutionContext context)
{
var config = Config.Instance(context.FunctionDirectory);
var spreadsheet = new BehaviourSpreadsheet(config.BehaviourSpreadsheetId, config.Key);
await spreadsheet.GetTrackerDataAsync();
#if !DEBUG
if (spreadsheet.Tracker.LastExported >= DateTime.Today)
{
log.Error("Export has already run today.");
return(req.CreateResponse(HttpStatusCode.Conflict, new { error = "Export has already run today." }));
}
#endif
// Append new expectations incidents
var newIncidents = await ReadIncidentsAsync(req.Content);
log.Info($"Request contains {newIncidents[true].Count()} expectation and {newIncidents[false].Count()} behaviour incidents.");
await spreadsheet.AddIncidentsAsync(SheetNames.Expectations, newIncidents[true].ToList());
// Fetch and parse spreadsheet data
var sheets = await spreadsheet.GetSheetsAsync($"{SheetNames.Expectations}!{spreadsheet.Tracker.StartRow}:{MAX_ROWS}", SheetNames.Contacts, SheetNames.DetentionDays, SheetNames.EmailTemplates);
var(unresolved, newStartRow) = Readers.ReadUnresolved(sheets[0], spreadsheet.Tracker.ExpectationColumns, spreadsheet.Tracker.StartRow);
var contacts = Readers.ReadContacts(sheets[1]);
var detentionDays = Readers.ReadDetentionDays(sheets[2]);
var emailTemplates = Readers.ReadTemplates(sheets[3]);
// Append new behaviour incidents
var escalated = Escalate(unresolved.Where(o => o.Status == Status.Escalate).ToList());
await spreadsheet.AddIncidentsAsync(SheetNames.Incidents, newIncidents[false].Union(escalated).ToList());
// Set detentions
CalculateDetentions(unresolved, detentionDays);
await spreadsheet.UpdateDetentionsAsync(unresolved.Where(o => o.IsChanged).ToList());
// Delete cancelled detentions
await spreadsheet.DeleteIncidentsAsync(unresolved.Where(o => o.Status == Status.Cancel).ToList());
// Email students, parents and contacts
var mailer = new Mailer(config.SenderEmail, emailQueue, emailContainer, emailTemplates, contacts, config.BehaviourSpreadsheetId, config.DebugRecipientEmail);
await mailer.QueueEmailsAsync(unresolved.ToList());
await triggerQueue.AddAsync(string.Empty);
// Sort expectations sheet
await spreadsheet.SortExpectationsSheet(spreadsheet.Tracker.ExpectationColumns);
// Write last edit date and new start row
await spreadsheet.WriteTrackerDataAsync(newStartRow);
// Update calendar events
var calendarChanges = unresolved.Where(o => o.EmailRequired || o.Status == Status.Cancel || o.PreviousDetentionDate != null).ToList();
if (config.DebugRecipientEmail == null)
{
await StudentCalendar.UpdateDetentionsAsync(calendarChanges, config.Key, config.DetentionStartTimeSpan, config.DetentionEndTimeSpan, config.DebugRecipientEmail);
}
log.Info("Function completed successfully.");
return(req.CreateResponse(HttpStatusCode.OK));
}
private void UpdateLatestVersionFlags(CloudBlobContainer container)
{
var blobsWithMetadata = container.ListBlobs().OfType<CloudBlockBlob>().Select(x => new
{
Blob = x,
Metadata = _packageSerializer.ReadFromMetadata(x)
}).ToList();
foreach (var blobWithMetadata in blobsWithMetadata)
blobWithMetadata.Metadata.IsLatestVersion = false;
var latest = blobsWithMetadata.OrderBy(x => x.Metadata.Version).Reverse().First();
latest.Metadata.IsLatestVersion = true;
latest.Metadata.IsAbsoluteLatestVersion = true;
foreach (var blobWithMetadata in blobsWithMetadata)
_packageSerializer.SaveToMetadata(blobWithMetadata.Metadata,blobWithMetadata.Blob);
}
public StatelessAzureQueueWriter(CloudBlobContainer container, CloudQueue queue, string name)
{
_cloudBlob = container;
_queue = queue;
Name = name;
}
public static async Task PurgeExpiredAsync(CloudBlobContainer inboxContainer)
{
Requires.NotNull(inboxContainer, "inboxContainer");
var deleteBlobsExpiringBefore = DateTime.UtcNow;
int purgedBlobCount = 0;
var searchExpiredBlobs = new TransformManyBlock<CloudBlobContainer, ICloudBlob>(
async c =>
{
try
{
var results = await c.ListBlobsSegmentedAsync(
string.Empty,
useFlatBlobListing: true,
pageSize: 50,
details: BlobListingDetails.Metadata,
options: new BlobRequestOptions(),
operationContext: null);
return from blob in results.OfType<ICloudBlob>()
let expires = DateTime.Parse(blob.Metadata[ExpirationDateMetadataKey])
where expires < deleteBlobsExpiringBefore
select blob;
}
catch (StorageException ex)
{
var webException = ex.InnerException as WebException;
if (webException != null)
{
var httpResponse = (HttpWebResponse)webException.Response;
if (httpResponse.StatusCode == HttpStatusCode.NotFound)
{
// it's legit that some tests never created the container to begin with.
return Enumerable.Empty<ICloudBlob>();
}
}
throw;
}
},
new ExecutionDataflowBlockOptions
{
BoundedCapacity = 4,
});
var deleteBlobBlock = new ActionBlock<ICloudBlob>(
blob =>
{
Interlocked.Increment(ref purgedBlobCount);
return blob.DeleteAsync();
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 4,
BoundedCapacity = 100,
});
searchExpiredBlobs.LinkTo(deleteBlobBlock, new DataflowLinkOptions { PropagateCompletion = true });
searchExpiredBlobs.Post(inboxContainer);
searchExpiredBlobs.Complete();
await deleteBlobBlock.Completion;
}
public CloudBlobContainerWrapper(CloudBlobContainer blobContainer)
{
_blobContainer = blobContainer;
}
/// <summary>
/// Get DestinationBlob with specified copy id
/// </summary>
/// <param name="container">CloudBlobContainer object</param>
/// <param name="blobName">Blob name</param>
/// <param name="copyId">Current CopyId</param>
/// <returns>Destination CloudBlob object</returns>
private CloudBlob GetDestinationBlobWithCopyId(IStorageBlobManagement destChannel, CloudBlobContainer container, string blobName)
{
AccessCondition accessCondition = null;
BlobRequestOptions options = RequestOptions;
CloudBlob blob = destChannel.GetBlobReferenceFromServer(container, blobName, accessCondition, options, OperationContext);
return(blob);
}
public static IEnumerable<IListBlobItem> ListAllBlob(CloudBlobContainer reference)
{
return reference.ListBlobs();
}
/// <summary>
/// Updates the container metadata.
/// </summary>
/// <param name="package">The package.</param>
/// <param name="container">The container.</param>
/// <param name="exists">if set to <c>true</c> [exists].</param>
private void UpdateContainerMetadata(IPackage package, CloudBlobContainer container, bool exists)
{
container.Metadata[Constants.LatestModificationDate] = DateTimeOffset.Now.ToString();
if (!exists)
{
container.Metadata[Constants.Created] = DateTimeOffset.Now.ToString();
}
container.Metadata[Constants.LastUploadedVersion] = package.Version.ToString();
container.Metadata[Constants.PackageId] = package.Id;
container.SetMetadata();
}
/// <summary>
/// Start copy using transfer mangager by source uri
/// </summary>
/// <param name="uri">source uri</param>
/// <param name="destContainer">Destination CloudBlobContainer object</param>
/// <param name="destBlobName">Destination blob name</param>
/// <returns>Destination CloudBlob object</returns>
private async Task StartCopyInTransferManager(long taskId, IStorageBlobManagement destChannel, Uri uri, CloudBlobContainer destContainer, string destBlobName)
{
NameUtil.ValidateContainerName(destContainer.Name);
NameUtil.ValidateBlobName(destBlobName);
Dictionary <string, string> BlobPath = new Dictionary <string, string>()
{
{ "Container", destContainer.Name },
{ "Blob", destBlobName }
};
DataMovementUserData data = new DataMovementUserData()
{
Data = BlobPath,
TaskId = taskId,
Channel = destChannel,
Record = null
};
TransferJob startCopyJob = new TransferJob(
new TransferLocation(uri),
new TransferLocation(destContainer.GetBlockBlobReference(destBlobName)),
TransferMethod.AsyncCopyInAzureStorageWithoutMonitor);
await this.EnqueueStartCopyJob(startCopyJob, data);
}
// A production app would also include an OnStop override to provide for
// graceful shut-downs of worker-role VMs. See
// http://azure.microsoft.com/en-us/documentation/articles/cloud-services-dotnet-multi-tier-app-storage-3-web-role/#restarts
public override bool OnStart()
{
// Set the maximum number of concurrent connections.
ServicePointManager.DefaultConnectionLimit = 12;
// Read database connection string and open database.
var dbConnString = CloudConfigurationManager.GetSetting("ContosoAdsDbConnectionString");
db = new ContosoAdsContext(dbConnString);
// Open storage account using credentials from .cscfg file.
var storageAccount = CloudStorageAccount.Parse
(RoleEnvironment.GetConfigurationSettingValue("StorageConnectionString"));
Trace.TraceInformation("Creating images blob container");
var blobClient = storageAccount.CreateCloudBlobClient();
imagesBlobContainer = blobClient.GetContainerReference("images");
if (imagesBlobContainer.CreateIfNotExists())
{
// Enable public access on the newly created "images" container.
imagesBlobContainer.SetPermissions(
new BlobContainerPermissions
{
PublicAccess = BlobContainerPublicAccessType.Blob
});
}
Trace.TraceInformation("Creating images queue");
CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient();
imagesQueue = queueClient.GetQueueReference("images");
imagesQueue.CreateIfNotExists();
Trace.TraceInformation("Storage initialized");
return base.OnStart();
}
