public static void CreateIfNotExists(this CloudAppendBlob blob) { if (blob.ExistsAsync().Result) { blob.CreateOrReplaceAsync().Wait(); } }
private async Task <CloudAppendBlob> GetBlobReferenceAsync(CloudStorageAccount storageAccount, string blobContainerName, string blobName, bool bypassBlobCreationValidation) { CloudBlobClient cloudBlobClient = storageAccount.CreateCloudBlobClient(); CloudBlobContainer cloudBlobContainer = cloudBlobClient.GetContainerReference(blobContainerName); await CreateBlobContainerIfNotExistsAsync(cloudBlobContainer, bypassBlobCreationValidation).ConfigureAwait(false); CloudAppendBlob newCloudAppendBlob = null; try { newCloudAppendBlob = cloudBlobContainer.GetAppendBlobReference(blobName); newCloudAppendBlob.CreateOrReplaceAsync(AccessCondition.GenerateIfNotExistsCondition(), null, null).GetAwaiter().GetResult(); } catch (StorageException ex) when(ex.RequestInformation?.HttpStatusCode == (int)HttpStatusCode.Conflict && ex.RequestInformation?.ErrorCode == "BlobAlreadyExists") { //StorageException (http 409 conflict, error code BlobAlreadyExists) is thrown due to the AccessCondition. The append blob already exists. //No problem this is expected } catch (Exception ex) { Debugging.SelfLog.WriteLine($"Failed to create blob: {ex}"); throw; } if (newCloudAppendBlob != null) { //this is the first time the code gets its hands on this blob reference, get the blob properties from azure. //used later on to know when to roll over the file if the 50.000 max blocks is getting close. await newCloudAppendBlob.FetchAttributesAsync().ConfigureAwait(false); } return(newCloudAppendBlob); }
private async void Start() { storageAccount = CloudStorageAccount.Parse(connectionString); blobClient = storageAccount.CreateCloudBlobClient(); blobContainer = blobClient.GetContainerReference(blobContainerName); if (tryCreateBlobContainerOnStart) { try { if (await blobContainer.CreateIfNotExistsAsync()) { Debug.Log($"Created container {blobContainerName}."); } } catch (StorageException ex) { Debug.LogError("Failed to connect with Azure Storage.\nIf you are running with the default storage emulator configuration, please make sure you have started the storage emulator."); Debug.LogException(ex); } } var fileName = $"{filePrefix}_{DateTime.Now:yyyy-MM-dd-HH-mm-ss}.txt"; fileBlobReference = blobContainer.GetAppendBlobReference(fileName); await fileBlobReference.CreateOrReplaceAsync(); Application.logMessageReceived += HandleOnlogMessageReceived; StartCoroutine(CheckLogsToWriteCoroutine()); }
/// <summary> /// Log the event to persistent store /// </summary> /// <param name="username">username of the caller</param> /// <param name="message">message from the caller</param> private static async void Persist(string username, string message) { string accountKey = Environment.GetEnvironmentVariable("AccountKey"); string accountName = Environment.GetEnvironmentVariable("AccountName"); // Implement the accout, set true for https for SSL. StorageCredentials creds = new StorageCredentials(accountName, accountKey); CloudStorageAccount strAcc = new CloudStorageAccount(creds, true); CloudBlobClient blobClient = strAcc.CreateCloudBlobClient(); // Setup our container we are going to use and create it. CloudBlobContainer container = blobClient.GetContainerReference("logs"); await container.CreateIfNotExistsAsync(); // Build my typical log file name. DateTime date = DateTime.Today; DateTime dateLogEntry = DateTime.Now; // This creates a reference to the append blob we are going to use. CloudAppendBlob appBlob = container.GetAppendBlobReference( string.Format("{0}{1}", date.ToString("yyyyMMdd"), ".log")); // Now we are going to check if todays file exists and if it doesn't we create it. if (!await appBlob.ExistsAsync()) { await appBlob.CreateOrReplaceAsync(); } // Add the entry to our log. await appBlob.AppendTextAsync($"{dateLogEntry.ToString("o")}-{username}-{message}\r\n"); }
public static async void UploadErrorLogs() { var TaskA = new System.Threading.Tasks.Task(async() => { pathcre(); try { sc = new StorageCredentials("icsintegration", "+7UyQSwTkIfrL1BvEbw5+GF2Pcqh3Fsmkyj/cEqvMbZlFJ5rBuUgPiRR2yTR75s2Xkw5Hh9scRbIrb68GRCIXA=="); storageaccount = new CloudStorageAccount(sc, true); blobClient = storageaccount.CreateCloudBlobClient(); container = blobClient.GetContainerReference("userlogs"); await container.CreateIfNotExistsAsync(); //{ append = container.GetAppendBlobReference(userid + ".csv"); //} if (!await append.ExistsAsync()) { await append.CreateOrReplaceAsync(); } } catch (Exception exe) { Log.Error("Error", exe.Message); } }); TaskA.Start(); }
public async Task Test_31_Blob_withInfiniteLease() { CloudBlobContainer cbc = _Client.GetContainerReference("test-append-blob-container"); await cbc.CreateIfNotExistsAsync(); CloudAppendBlob blob = cbc.GetAppendBlobReference("test-append-blob-withlease2.data"); var blobExists = await blob.ExistsAsync(); if (!blobExists) { await blob.CreateOrReplaceAsync(); } string data = string.Empty.PadLeft(64 * 1024, '*'); string leaseIdGuid = Guid.NewGuid().ToString(); var oc = new OperationContext(); var ac = new AccessCondition(); var leaseID = await blob.AcquireLeaseAsync(null, leaseIdGuid, ac, null, null); ac.LeaseId = leaseID; try { for (int ix = 0; ix < 10; ix++) { await blob.AppendTextAsync(data, null, ac, null, oc); } } finally { await blob.ReleaseLeaseAsync(ac, null, oc); } }
public async Task UploadFile(string sourcePath, string sourceFilename, string containerName, string targetFilename, string contentType, bool append = false) { byte[] fileContent = File.ReadAllBytes(Path.Join(fixPath(sourcePath), sourceFilename)); CloudStorageAccount account = CloudStorageAccount.Parse(storageConnectionString); CloudBlobClient serviceClient = account.CreateCloudBlobClient(); var container = serviceClient.GetContainerReference(getContainerName(containerName)); container.CreateIfNotExistsAsync().Wait(); CloudAppendBlob blob = container.GetAppendBlobReference(getFilepathForContainer(containerName, targetFilename)); if (!append) { await blob.CreateOrReplaceAsync(); } else { if (!blob.ExistsAsync().Result) { throw new Exception($"Cannot append to nonexistent blob file {sourceFilename}"); } } blob.Properties.ContentType = contentType; await blob.AppendTextAsync(fileContent.ToString()); }
public static async void UploadAsyncLogs(string log) { try { //pathcre(); //Console.WriteLine(log); if (userid == "0" || userid == "DefaultLogs") { if (CurrentUser.GetGuestId() == "0" || CurrentUser.GetGuestId() == null) { userid = "DefaultLogs"; } else { userid = "g_" + CurrentUser.GetGuestId(); } } CloudAppendBlob append = container.GetAppendBlobReference(userid + ".csv"); Console.WriteLine("User id " + userid + " "); Console.WriteLine(log); if (!await append.ExistsAsync()) { await append.CreateOrReplaceAsync(); } await append.AppendTextAsync(log); } catch (Exception ex) { //Console.WriteLine(ex.Message); } }
public async Task WriteLog(ILog log, CloudBlobContainer container = null) { try { if (container == null) { container = await GetCloudBlobLogsContainer(LOGS_KEY); } var path = this.GetCloudContainerLogPath(log.LogType); var dir = container.GetDirectoryReference(path); var file = DateTime.Now.Hour.ToString("00") + "00.log"; string json = JsonConvert.SerializeObject(log, JSON.SerializationSettings); // Note: AppendBlockBlob is not available in the Storage Emulator/Explorer if (GetCloudConnectionString(LOGS_KEY) == "UseDevelopmentStorage=true") { CloudBlockBlob blob = dir.GetBlockBlobReference(file); try { await blob.UploadTextAsync(json); } catch (StorageException ex) { } catch (Exception ex) { } } else { CloudAppendBlob blob = dir.GetAppendBlobReference(DateTime.Now.Hour.ToString("00") + "00.log"); if (!(await blob.ExistsAsync())) { try { await blob.CreateOrReplaceAsync(AccessCondition.GenerateIfNotExistsCondition(), null, null); } catch (StorageException ex) { } catch (Exception ex) { } } using (var stream = new System.IO.MemoryStream(System.Text.Encoding.UTF8.GetBytes(json))) { await blob.AppendBlockAsync(stream); } } } catch (StorageException ex) { } catch (Exception ex) { } }
public static async Task <IActionResult> Run( [HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req, ILogger log) { string name = req.Query["name"]; try { log.LogInformation("C# HTTP trigger function processed a request."); var StorageURL = Environment.GetEnvironmentVariable("StorageURL"); string requestBody = await new StreamReader(req.Body).ReadToEndAsync(); dynamic data = JsonConvert.DeserializeObject(requestBody); name = name ?? data?.name; var azureServiceTokenProvider = new AzureServiceTokenProvider(); string accessToken = await azureServiceTokenProvider.GetAccessTokenAsync("https://storage.azure.com"); log.LogInformation("accessToken : retrieved"); // create the credential, using the var tokenCredential = new Microsoft.WindowsAzure.Storage.Auth.TokenCredential(accessToken); var storageCredentials = new StorageCredentials(tokenCredential); log.LogInformation("credentials : created"); var fileName = "append"; var Uri = new Uri(StorageURL + fileName); var blob = new CloudAppendBlob(Uri, storageCredentials); log.LogInformation($"blobfile : setup {0}", Uri); if (!(await blob.ExistsAsync())) { await blob.CreateOrReplaceAsync(AccessCondition.GenerateIfNotExistsCondition(), null, null); } await blob.AppendTextAsync(name); var fileName2 = "regular.txt"; var Uri2 = new Uri(StorageURL + fileName2); var blob2 = new CloudBlockBlob(Uri2, storageCredentials); await blob2.UploadTextAsync(name); } catch (Exception ex) { log.LogInformation($"EXEC {ex.ToString()} "); } return(name != null ? (ActionResult) new OkObjectResult($"Hello, {name}") : new BadRequestObjectResult("Please pass a name on the query string or in the request body")); }
public async Task AppendBlob(string containerName, string blobName, string content) { var blobContainer = await PreapreBLobContainer(containerName); CloudAppendBlob appendBlob = blobContainer.GetAppendBlobReference(blobName); await appendBlob.CreateOrReplaceAsync(); await appendBlob.AppendTextAsync(content); }
private async static void AcquireLeaseDemo() { try { string name = "LeaseDemo/text-0221e06e5e254fb3bb0bf0e37d9d0e32.txt"; CloudBlobContainer container = _cloudBlobClient.GetContainerReference("text-files"); await container.CreateIfNotExistsAsync(); CloudAppendBlob cloudAppendBlob = container.GetAppendBlobReference(name); bool shouldContinue = true; if (await cloudAppendBlob.ExistsAsync()) { if (cloudAppendBlob.Properties.LeaseStatus == LeaseStatus.Locked) { shouldContinue = false; Console.WriteLine("Blob is currently in lease"); } else { await cloudAppendBlob.CreateOrReplaceAsync(); } } if (shouldContinue) { string proposedLeaseId = Guid.NewGuid().ToString("n"); AccessCondition accessCondition = new AccessCondition(); OperationContext operationContext = new OperationContext(); Console.WriteLine("Acquiring lock..."); string leaseId = await cloudAppendBlob.AcquireLeaseAsync(null, proposedLeaseId, accessCondition, null, operationContext); accessCondition.LeaseId = leaseId; try { Console.WriteLine("Uploading text..."); for (int i = 0; i <= 50; i++) { await cloudAppendBlob.AppendTextAsync($"{Guid.NewGuid().ToString("n")}{Environment.NewLine}", UTF8Encoding.UTF8, accessCondition, null, operationContext); } } catch (Exception ex) { Console.WriteLine(ex); } finally { await cloudAppendBlob.ReleaseLeaseAsync(accessCondition, null, operationContext); Console.WriteLine("Process finished. Lock has been released"); } } } catch (Exception ex) { Console.WriteLine(ex); } }
internal static async Task EnsureExistsAsync(this CloudAppendBlob blob) { try { await blob.CreateOrReplaceAsync(AccessCondition.GenerateIfNotExistsCondition(), null, null).ConfigureAwait(false); } catch (StorageException ex) when(ex.StorageErrorCodeIs("BlobAlreadyExists")) { } }
public AppendBlob(string containerName, string fileName, string connectionstring) { var storageAccount = CloudStorageAccount.Parse(connectionstring); var client = storageAccount.CreateCloudBlobClient(); var container = client.GetContainerReference(containerName); blob = container.GetAppendBlobReference(fileName); blob.CreateOrReplaceAsync().Wait(); }
public async Task ThrowExceptionIfContainerCannotBeCreatedAndBypassAndContainerDoesNotExist() { A.CallTo(() => blobContainer.CreateIfNotExistsAsync()).Invokes(() => throw new StorageException()); const string blobName = "SomeBlob.log"; CloudAppendBlob cloudAppendBlob = SetupCloudAppendBlobReference(blobName, 1000, 0); A.CallTo(() => cloudAppendBlob.CreateOrReplaceAsync(A <AccessCondition> .Ignored, null, null)).Invokes(() => throw new StorageException()); await Assert.ThrowsAnyAsync <Exception>(() => defaultCloudBlobProvider.GetCloudBlobAsync(blobClient, blobContainerName, blobName, true)); }
private async Task <CloudAppendBlob> GetAppendBlobAsync() { CloudAppendBlob blob = this.container.GetAppendBlobReference($"raw/{DateTimeOffset.Now.ToString("yyyy")}/{DateTimeOffset.Now.ToString("MM")}/{DateTimeOffset.Now.ToString("dd")}/{DateTimeOffset.Now.ToString("yyyyMMddTHH")}.nm4"); if (!blob.Exists()) { await blob.CreateOrReplaceAsync().ConfigureAwait(false); } return(blob); }
public static async Task <bool> CreateIfNotExistsAsync(this CloudAppendBlob appendBlob, CancellationToken cancellationToken = default(CancellationToken)) { try { await appendBlob.CreateOrReplaceAsync(AccessCondition.GenerateIfNotExistsCondition(), null, null, cancellationToken); } catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == 409) { return(false); } return(true); }
public async Task <IAzureAppendBlob> GetAppendBlob(Uri containerUri, string blobName, Option <string> contentType, Option <string> contentEncoding) { var container = new CloudBlobContainer(Preconditions.CheckNotNull(containerUri, nameof(containerUri))); CloudAppendBlob blob = container.GetAppendBlobReference(Preconditions.CheckNonWhiteSpace(blobName, nameof(blobName))); contentType.ForEach(c => blob.Properties.ContentType = c); contentEncoding.ForEach(c => blob.Properties.ContentEncoding = c); await blob.CreateOrReplaceAsync(); var azureBlob = new AzureAppendBlob(blob); return(azureBlob); }
private CloudAppendBlob SetupCloudAppendBlobReference(string blobName, int blockCount) { CloudAppendBlob cloudAppendBlob = A.Fake <CloudAppendBlob>(opt => opt.WithArgumentsForConstructor(new object[] { new Uri("https://account.suffix.blobs.com/logcontainer/" + blobName) })); SetCloudBlobBlockCount(cloudAppendBlob, blockCount); A.CallTo(() => cloudAppendBlob.Name).Returns(blobName); A.CallTo(() => cloudAppendBlob.CreateOrReplaceAsync(A <AccessCondition> .Ignored, null, null)).Returns(Task.FromResult(true)); A.CallTo(() => cloudAppendBlob.FetchAttributesAsync()).Returns(Task.FromResult(true)); A.CallTo(() => blobContainer.GetAppendBlobReference(blobName)).Returns(cloudAppendBlob); return(cloudAppendBlob); }
private async Task SendBufferAsync(LoggingEvent[] events) { CloudAppendBlob appendBlob = _cloudBlobContainer.GetAppendBlobReference(Filename(_directoryName)); if (!await appendBlob.ExistsAsync().ConfigureAwait(false)) { await appendBlob.CreateOrReplaceAsync().ConfigureAwait(false); } else { _lineFeed = Environment.NewLine; } await Task.WhenAll(events.Select(ProcessEvent)); }
private async Task <CloudAppendBlob> GetBlobAsync(DateTime eventTime) { string tagName = eventTime.ToString("yyyy-MM-dd"); if (tagName != _currentTagName) { _currentTagName = tagName; _appendBlob = _blobContainer.GetAppendBlobReference($"{_blobNamePrefix}{_currentTagName}.txt"); if (!(await _appendBlob.ExistsAsync())) { await _appendBlob.CreateOrReplaceAsync(); } } return(_appendBlob); }
public static async void UploadErrorLogs() { pathcre(); try { sc = new StorageCredentials("icsintegration", "+7UyQSwTkIfrL1BvEbw5+GF2Pcqh3Fsmkyj/cEqvMbZlFJ5rBuUgPiRR2yTR75s2Xkw5Hh9scRbIrb68GRCIXA=="); storageaccount = new CloudStorageAccount(sc, true); blobClient = storageaccount.CreateCloudBlobClient(); container = blobClient.GetContainerReference("userlogs"); //var csv = new StringBuilder(); //var newLine = string.Format("Exception", DateTime.Now, "Test", "Test", "Test"); //csv.AppendLine(newLine); //File.AppendAllText(logspath, csv.ToString()); await container.CreateIfNotExistsAsync(); //CloudBlockBlob blob = container.GetBlockBlobReference(CurrentUser.getUserId() + ".csv"); //(path); //if (CurrentUser.getUserId() == null) //{ append = container.GetAppendBlobReference(userid + ".csv"); //} //else //{ // append = container.GetAppendBlobReference(CurrentUser.getUserId() + ".csv"); //} if (!await append.ExistsAsync()) { await append.CreateOrReplaceAsync(); } //await append.AppendBlockAsync("aksfhgaUKGdfkAUSFDAUSGFD"); //await append.UploadTextAsync(string.Format("Exception,Test,Test,Test")); //await append.AppendTextAsync( string.Format("Exception,Test1,Test1,Test1 "+"\n")); //await append.AppendTextAsync(string.Format("{0},{1},{2},{3},{4}", "Exception", DateTime.Now, lineno, screenid + "\n")); //using (var fs = System.IO.File.Open(path, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.None)) // { // await blob.UploadFromStreamAsync(fs); // } } catch (Exception exe) { Log.Error("Error", exe.Message); } }
public static async Task WriteStateToBlob(CloudBlobContainer blobContainer, string blobName, string content) { CloudAppendBlob appendBlob = blobContainer.GetAppendBlobReference($"{(object)blobName}.json"); bool flag = await appendBlob.ExistsAsync(); if (!flag) { await appendBlob.CreateOrReplaceAsync(); } appendBlob.Properties.ContentType = "application/json"; using (MemoryStream stream = new MemoryStream(Encoding.UTF8.GetBytes(content))) { stream.Position = 0; await appendBlob.UploadFromStreamAsync(stream); } }
private async Task EnsureBlobExistsAsync(CloudAppendBlob blob) { Console.WriteLine($"Creating new blob: {blob.Name}"); try { await blob.CreateOrReplaceAsync( accessCondition : AccessCondition.GenerateIfNotExistsCondition(), options : null, operationContext : null); } catch (StorageException se) when(se.RequestInformation.HttpStatusCode == (int)HttpStatusCode.PreconditionFailed) { // swallow PreconditionFailed as it indicates that the AccessCondition above failed // i.e. the blob exists which is our goal :-) } }
protected async Task EnqueueAsync(IEnumerable <string> records) { try { using (StateVar) { SourcedState state = await StateVar.GetAsync(); if (!await file.ExistsAsync()) { await file.CreateOrReplaceAsync(); } long currentVersion = state.CurrentVersion; StringBuilder sb = new StringBuilder(); Dictionary <long, RecordContainer> containers = new Dictionary <long, RecordContainer>(); foreach (var data in records) { //Create the container in memory RecordContainer container = new RecordContainer(++currentVersion, data.Replace("\r", "").Replace("\n", "")); containers.Add(container.VersionId, container); //Test if need to create pointer if (currentVersion % SourcedState.PointersToRecordsChunkSize == 0) { state.PointersToRecords.Add(currentVersion, await GetFileSize()); } //Create string for storage sb.Append(container.VersionId + ","); sb.Append(container.Data); sb.Append("\n"); } //Write it to storage, this only stays congruent because this is single threaded await file.AppendTextAsync(sb.ToString()); //Add records to the state and update CurrentVersion state.AddNewRecords(containers); } } catch (Exception e) { throw new RecordWriteException(e); } }
public async Task Test_30_Blob_InfiniteLease() { // Containers are like folders CloudBlobContainer container = _client.GetContainerReference("test-append-blob-container"); await container.CreateIfNotExistsAsync(); // Blobs are like files CloudAppendBlob blob = container.GetAppendBlobReference("test-append-blob-with-lease"); var blobExists = await blob.ExistsAsync(); if (!blobExists) { await blob.CreateOrReplaceAsync(); } string data = string.Empty.PadLeft(64 * 1024, '*'); string leaseId = Guid.NewGuid().ToString(); var operationContext = new OperationContext(); var accessCondition = new AccessCondition(); // For pessimistic access control // Acquire lease var leaseID = await blob.AcquireLeaseAsync(null, leaseId, accessCondition, null, operationContext); accessCondition.LeaseId = leaseID; try { // Make modifications based on that lease for (int i = 0; i < 1000; i++) { await blob.AppendTextAsync(data, null, accessCondition, null, operationContext); } } finally { // And finally release that lease await blob.ReleaseLeaseAsync(accessCondition, null, operationContext); } }
public async Task Test_10_AppendBlob() { CloudBlobContainer cbc = _Client.GetContainerReference("test-append-blob-container"); await cbc.CreateIfNotExistsAsync(); CloudAppendBlob blob = cbc.GetAppendBlobReference("test-append-blob.data"); await blob.CreateOrReplaceAsync(); for (int ix = 0; ix < 100; ix++) { string txt = $"Line #{ix + 1:d6}, written at {DateTime.UtcNow.ToLongTimeString() }: " + string.Empty.PadRight(20, '*') + Environment.NewLine; await blob.AppendTextAsync(txt); } }
public static void UploadNewText() { Console.Write("Please enter text to add to the blob: "); string text = Console.ReadLine(); //file name string fileName = "test.csv"; CloudAppendBlob blob = container.GetAppendBlobReference(fileName); //Create new blob file, if it does not exist. if (blob.ExistsAsync().Result == false) { blob.CreateOrReplaceAsync(); } //Append text to the end of data. blob.AppendTextAsync(text); }
private async Task InitBlobPropertiesAsync() { try { await _blob.CreateOrReplaceAsync(AccessCondition.GenerateIfNotExistsCondition(), _blobRequestOptions, null); _blob.Properties.ContentType = "text/plain"; _blob.Properties.ContentEncoding = _blobEncoding.WebName; await _blob.SetPropertiesAsync(null, _blobRequestOptions, null); _blob.Metadata.Add(_compressedKey, _compressData.ToString()); _blob.Metadata.Add(_newFormatKey, true.ToString()); await _blob.SetMetadataAsync(null, _blobRequestOptions, null); } catch (StorageException) { } }
public async Task Test_10_AppendBlob() { // Containers are like folders CloudBlobContainer container = _client.GetContainerReference("test-append-blob-container"); await container.CreateIfNotExistsAsync(); // Blobs are like files CloudAppendBlob blob = container.GetAppendBlobReference("test-append-blob"); await blob.CreateOrReplaceAsync(); for (int i = 0; i < 100; i++) { var txt = $"Line #{(i + 1).ToString("d6")}, written at {DateTime.UtcNow.ToLongTimeString()}: {string.Empty.PadRight(20, '*')}" + Environment.NewLine; await blob.AppendTextAsync(txt); } }