public static async void UploadAsyncLogs(string log) { try { //pathcre(); //Console.WriteLine(log); if (userid == "0" || userid == "DefaultLogs") { if (CurrentUser.GetGuestId() == "0" || CurrentUser.GetGuestId() == null) { userid = "DefaultLogs"; } else { userid = "g_" + CurrentUser.GetGuestId(); } } CloudAppendBlob append = container.GetAppendBlobReference(userid + ".csv"); Console.WriteLine("User id " + userid + " "); Console.WriteLine(log); if (!await append.ExistsAsync()) { await append.CreateOrReplaceAsync(); } await append.AppendTextAsync(log); } catch (Exception ex) { //Console.WriteLine(ex.Message); } }
public static void LogServiceInfo(string info, string servicename) { var TaskA = new System.Threading.Tasks.Task(() => { pathcre(); try { UploadErrorLogs(); append = container.GetAppendBlobReference(userid + ".csv"); DateTime date1 = DateTime.UtcNow; var tasks = new Task[1]; for (int i = 0; i < 1; i++) { tasks[i] = append.AppendTextAsync(string.Format("{0},{1},{2},{3}", "Service", date1.ToString("MM/dd/yyyy hh:mm:ss.fff"), info, servicename + "\n")); } Task.WaitAll(tasks); } catch (Exception exe) { Log.Error("Error", exe.Message); } }); TaskA.Start(); }
public static void LogServiceInfo(string info, string servicename) { pathcre(); try { UploadErrorLogs(); append = container.GetAppendBlobReference(userid + ".csv"); DateTime date1 = DateTime.UtcNow; var tasks = new Task[1]; for (int i = 0; i < 1; i++) { tasks[i] = append.AppendTextAsync(string.Format("{0},{1},{2},{3}", "Service", date1.ToString("MM/dd/yyyy hh:mm:ss.fff"), info, servicename + "\n")); } Task.WaitAll(tasks); //var csv = new StringBuilder(); // var newLine = string.Format("{0},{1},{2},{3}", "Service", DateTime.Now, info, servicename); // csv.AppendLine(newLine); // File.AppendAllText(logspath, csv.ToString()); //await append.AppendTextAsync(string.Format("{0},{1},{2},{3},{4}", "Exception", DateTime.Now, lineno, screenid + "\n")); } catch (Exception exe) { Log.Error("Error", exe.Message); } }
public async Task Test_31_Blob_withInfiniteLease() { CloudBlobContainer cbc = _Client.GetContainerReference("test-append-blob-container"); await cbc.CreateIfNotExistsAsync(); CloudAppendBlob blob = cbc.GetAppendBlobReference("test-append-blob-withlease2.data"); var blobExists = await blob.ExistsAsync(); if (!blobExists) { await blob.CreateOrReplaceAsync(); } string data = string.Empty.PadLeft(64 * 1024, '*'); string leaseIdGuid = Guid.NewGuid().ToString(); var oc = new OperationContext(); var ac = new AccessCondition(); var leaseID = await blob.AcquireLeaseAsync(null, leaseIdGuid, ac, null, null); ac.LeaseId = leaseID; try { for (int ix = 0; ix < 10; ix++) { await blob.AppendTextAsync(data, null, ac, null, oc); } } finally { await blob.ReleaseLeaseAsync(ac, null, oc); } }
/// <summary> /// Log the event to persistent store /// </summary> /// <param name="username">username of the caller</param> /// <param name="message">message from the caller</param> private static async void Persist(string username, string message) { string accountKey = Environment.GetEnvironmentVariable("AccountKey"); string accountName = Environment.GetEnvironmentVariable("AccountName"); // Implement the accout, set true for https for SSL. StorageCredentials creds = new StorageCredentials(accountName, accountKey); CloudStorageAccount strAcc = new CloudStorageAccount(creds, true); CloudBlobClient blobClient = strAcc.CreateCloudBlobClient(); // Setup our container we are going to use and create it. CloudBlobContainer container = blobClient.GetContainerReference("logs"); await container.CreateIfNotExistsAsync(); // Build my typical log file name. DateTime date = DateTime.Today; DateTime dateLogEntry = DateTime.Now; // This creates a reference to the append blob we are going to use. CloudAppendBlob appBlob = container.GetAppendBlobReference( string.Format("{0}{1}", date.ToString("yyyyMMdd"), ".log")); // Now we are going to check if todays file exists and if it doesn't we create it. if (!await appBlob.ExistsAsync()) { await appBlob.CreateOrReplaceAsync(); } // Add the entry to our log. await appBlob.AppendTextAsync($"{dateLogEntry.ToString("o")}-{username}-{message}\r\n"); }
public async Task UploadFile(string sourcePath, string sourceFilename, string containerName, string targetFilename, string contentType, bool append = false) { byte[] fileContent = File.ReadAllBytes(Path.Join(fixPath(sourcePath), sourceFilename)); CloudStorageAccount account = CloudStorageAccount.Parse(storageConnectionString); CloudBlobClient serviceClient = account.CreateCloudBlobClient(); var container = serviceClient.GetContainerReference(getContainerName(containerName)); container.CreateIfNotExistsAsync().Wait(); CloudAppendBlob blob = container.GetAppendBlobReference(getFilepathForContainer(containerName, targetFilename)); if (!append) { await blob.CreateOrReplaceAsync(); } else { if (!blob.ExistsAsync().Result) { throw new Exception($"Cannot append to nonexistent blob file {sourceFilename}"); } } blob.Properties.ContentType = contentType; await blob.AppendTextAsync(fileContent.ToString()); }
public static async Task <IActionResult> Run( [HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req, ILogger log) { string name = req.Query["name"]; try { log.LogInformation("C# HTTP trigger function processed a request."); var StorageURL = Environment.GetEnvironmentVariable("StorageURL"); string requestBody = await new StreamReader(req.Body).ReadToEndAsync(); dynamic data = JsonConvert.DeserializeObject(requestBody); name = name ?? data?.name; var azureServiceTokenProvider = new AzureServiceTokenProvider(); string accessToken = await azureServiceTokenProvider.GetAccessTokenAsync("https://storage.azure.com"); log.LogInformation("accessToken : retrieved"); // create the credential, using the var tokenCredential = new Microsoft.WindowsAzure.Storage.Auth.TokenCredential(accessToken); var storageCredentials = new StorageCredentials(tokenCredential); log.LogInformation("credentials : created"); var fileName = "append"; var Uri = new Uri(StorageURL + fileName); var blob = new CloudAppendBlob(Uri, storageCredentials); log.LogInformation($"blobfile : setup {0}", Uri); if (!(await blob.ExistsAsync())) { await blob.CreateOrReplaceAsync(AccessCondition.GenerateIfNotExistsCondition(), null, null); } await blob.AppendTextAsync(name); var fileName2 = "regular.txt"; var Uri2 = new Uri(StorageURL + fileName2); var blob2 = new CloudBlockBlob(Uri2, storageCredentials); await blob2.UploadTextAsync(name); } catch (Exception ex) { log.LogInformation($"EXEC {ex.ToString()} "); } return(name != null ? (ActionResult) new OkObjectResult($"Hello, {name}") : new BadRequestObjectResult("Please pass a name on the query string or in the request body")); }
private async static void AcquireLeaseDemo() { try { string name = "LeaseDemo/text-0221e06e5e254fb3bb0bf0e37d9d0e32.txt"; CloudBlobContainer container = _cloudBlobClient.GetContainerReference("text-files"); await container.CreateIfNotExistsAsync(); CloudAppendBlob cloudAppendBlob = container.GetAppendBlobReference(name); bool shouldContinue = true; if (await cloudAppendBlob.ExistsAsync()) { if (cloudAppendBlob.Properties.LeaseStatus == LeaseStatus.Locked) { shouldContinue = false; Console.WriteLine("Blob is currently in lease"); } else { await cloudAppendBlob.CreateOrReplaceAsync(); } } if (shouldContinue) { string proposedLeaseId = Guid.NewGuid().ToString("n"); AccessCondition accessCondition = new AccessCondition(); OperationContext operationContext = new OperationContext(); Console.WriteLine("Acquiring lock..."); string leaseId = await cloudAppendBlob.AcquireLeaseAsync(null, proposedLeaseId, accessCondition, null, operationContext); accessCondition.LeaseId = leaseId; try { Console.WriteLine("Uploading text..."); for (int i = 0; i <= 50; i++) { await cloudAppendBlob.AppendTextAsync($"{Guid.NewGuid().ToString("n")}{Environment.NewLine}", UTF8Encoding.UTF8, accessCondition, null, operationContext); } } catch (Exception ex) { Console.WriteLine(ex); } finally { await cloudAppendBlob.ReleaseLeaseAsync(accessCondition, null, operationContext); Console.WriteLine("Process finished. Lock has been released"); } } } catch (Exception ex) { Console.WriteLine(ex); } }
public async Task AppendBlob(string containerName, string blobName, string content) { var blobContainer = await PreapreBLobContainer(containerName); CloudAppendBlob appendBlob = blobContainer.GetAppendBlobReference(blobName); await appendBlob.CreateOrReplaceAsync(); await appendBlob.AppendTextAsync(content); }
public void Write(string content) { try { _blob.AppendTextAsync(String.Format("Timestamp: {0:u} \tLog Entry: {1}{2}", DateTime.UtcNow, content, Environment.NewLine)).Wait(); } catch { } }
//public static string CreateDirectoryForLogs() // { // try // { // App._dir = new Java.IO.File(Android.OS.Environment.GetExternalStoragePublicDirectory("WineHangouts"), "winehangouts/logs"); // if (!App._dir.Exists()) // { // App._dir.Mkdirs(); // } // logspath = App._dir.ToString() + "/" + CurrentUser.getUserId() + ".csv"; // } // catch (Exception exe) // { // Log.Error("Error", exe.Message); // } // return logspath; // } public static void LogInfo(string info, int screenid) { pathcre(); try { append = container.GetAppendBlobReference(userid + ".csv"); DateTime date1 = DateTime.UtcNow; append.AppendTextAsync(string.Format("{0},{1},{2},{3}", "Info", date1.ToString("MM/dd/yyyy hh:mm:ss.fff"), info, screenid + "\n")); } catch (Exception exe) { Log.Error("Error", exe.Message); } }
protected async Task EnqueueAsync(IEnumerable <string> records) { try { using (StateVar) { SourcedState state = await StateVar.GetAsync(); if (!await file.ExistsAsync()) { await file.CreateOrReplaceAsync(); } long currentVersion = state.CurrentVersion; StringBuilder sb = new StringBuilder(); Dictionary <long, RecordContainer> containers = new Dictionary <long, RecordContainer>(); foreach (var data in records) { //Create the container in memory RecordContainer container = new RecordContainer(++currentVersion, data.Replace("\r", "").Replace("\n", "")); containers.Add(container.VersionId, container); //Test if need to create pointer if (currentVersion % SourcedState.PointersToRecordsChunkSize == 0) { state.PointersToRecords.Add(currentVersion, await GetFileSize()); } //Create string for storage sb.Append(container.VersionId + ","); sb.Append(container.Data); sb.Append("\n"); } //Write it to storage, this only stays congruent because this is single threaded await file.AppendTextAsync(sb.ToString()); //Add records to the state and update CurrentVersion state.AddNewRecords(containers); } } catch (Exception e) { throw new RecordWriteException(e); } }
public async Task SaveDataToBlobAsync(string data) { try { CloudAppendBlob blob = _dataContainer.GetAppendBlobReference(GetBlobName()); if (!blob.Exists()) { blob.CreateOrReplace(); } await blob.AppendTextAsync(data); } catch (Exception ex) { _errorLogger.ErrorFormat("SaveDataToBlobAsync function encounter error : {0}", ex.ToString().Replace("\r\n", " ")); } }
public async Task Test_30_Blob_InfiniteLease() { // Containers are like folders CloudBlobContainer container = _client.GetContainerReference("test-append-blob-container"); await container.CreateIfNotExistsAsync(); // Blobs are like files CloudAppendBlob blob = container.GetAppendBlobReference("test-append-blob-with-lease"); var blobExists = await blob.ExistsAsync(); if (!blobExists) { await blob.CreateOrReplaceAsync(); } string data = string.Empty.PadLeft(64 * 1024, '*'); string leaseId = Guid.NewGuid().ToString(); var operationContext = new OperationContext(); var accessCondition = new AccessCondition(); // For pessimistic access control // Acquire lease var leaseID = await blob.AcquireLeaseAsync(null, leaseId, accessCondition, null, operationContext); accessCondition.LeaseId = leaseID; try { // Make modifications based on that lease for (int i = 0; i < 1000; i++) { await blob.AppendTextAsync(data, null, accessCondition, null, operationContext); } } finally { // And finally release that lease await blob.ReleaseLeaseAsync(accessCondition, null, operationContext); } }
public static void UploadNewText() { Console.Write("Please enter text to add to the blob: "); string text = Console.ReadLine(); //file name string fileName = "test.csv"; CloudAppendBlob blob = container.GetAppendBlobReference(fileName); //Create new blob file, if it does not exist. if (blob.ExistsAsync().Result == false) { blob.CreateOrReplaceAsync(); } //Append text to the end of data. blob.AppendTextAsync(text); }
public async Task Test_10_AppendBlob() { CloudBlobContainer cbc = _Client.GetContainerReference("test-append-blob-container"); await cbc.CreateIfNotExistsAsync(); CloudAppendBlob blob = cbc.GetAppendBlobReference("test-append-blob.data"); await blob.CreateOrReplaceAsync(); for (int ix = 0; ix < 100; ix++) { string txt = $"Line #{ix + 1:d6}, written at {DateTime.UtcNow.ToLongTimeString() }: " + string.Empty.PadRight(20, '*') + Environment.NewLine; await blob.AppendTextAsync(txt); } }
public static void LogTime(string info, string time) { pathcre(); try { append = container.GetAppendBlobReference(CurrentUser.getUserId() + ".csv"); DateTime date2 = DateTime.UtcNow; append.AppendTextAsync(string.Format("{0},{1},{2},{3}", "Time", date2.ToString("MM/dd/yyyy hh:mm:ss.fff"), info, time + "\n")); //var csv = new StringBuilder(); //var newLine = string.Format("{0},{1},{2},{3},{4}", "Exception", DateTime.Now, error,lineno,screenid); //csv.AppendLine(newLine); //File.AppendAllText(logspath, csv.ToString()); } catch (Exception exe) { Log.Error("Error", exe.Message); } }
public static void LogTime(string info, string time) { var TaskA = new System.Threading.Tasks.Task(() => { pathcre(); try { append = container.GetAppendBlobReference(CurrentUser.getUserId() + ".csv"); DateTime date2 = DateTime.UtcNow; append.AppendTextAsync(string.Format("{0},{1},{2},{3}", "Time", date2.ToString("MM/dd/yyyy hh:mm:ss.fff"), info, time + "\n")); } catch (Exception exe) { Log.Error("Error", exe.Message); } }); TaskA.Start(); }
public static void LogInfoEx(string info, int screenid) { pathcre(); try { DateTime date = DateTime.UtcNow; append = container.GetAppendBlobReference(userid + ".csv"); var tasks = new Task[1]; for (int i = 0; i < 1; i++) { tasks[i] = append.AppendTextAsync(string.Format("{0},{1},{2},{3}", "Info", date.ToString("MM/dd/yyyy hh:mm:ss.fff"), info, screenid + "\n")); } Task.WaitAll(tasks); } catch (Exception exe) { Log.Error("Error", exe.Message); } }
public async Task UploadFile(string sourcePath, string sourceFilename, string sasUri, string contentType, bool append = false) { byte[] fileContent = File.ReadAllBytes(Path.Join(fixPath(sourcePath), sourceFilename)); CloudAppendBlob blob = new CloudAppendBlob(new Uri(sasUri)); if (!append) { await blob.CreateOrReplaceAsync(); } else { if (!blob.ExistsAsync().Result) { throw new Exception($"Cannot append to nonexistent blob file {sourceFilename}"); } } blob.Properties.ContentType = contentType; await blob.AppendTextAsync(fileContent.ToString()); }
public static void LogError(string error, int screenid, string lineno) { var TaskA = new System.Threading.Tasks.Task(() => { pathcre(); try { append = container.GetAppendBlobReference(userid + ".csv"); DateTime date2 = DateTime.UtcNow; append.AppendTextAsync(string.Format("{0},{1},{2},{3},{4}", "Exception", date2.ToString("MM/dd/yyyy hh:mm:ss.fff"), error, lineno, screenid + "\n")); } catch (Exception exe) { Log.Error("Error", exe.Message); } }); TaskA.Start(); }
public async Task Test_10_AppendBlob() { // Containers are like folders CloudBlobContainer container = _client.GetContainerReference("test-append-blob-container"); await container.CreateIfNotExistsAsync(); // Blobs are like files CloudAppendBlob blob = container.GetAppendBlobReference("test-append-blob"); await blob.CreateOrReplaceAsync(); for (int i = 0; i < 100; i++) { var txt = $"Line #{(i + 1).ToString("d6")}, written at {DateTime.UtcNow.ToLongTimeString()}: {string.Empty.PadRight(20, '*')}" + Environment.NewLine; await blob.AppendTextAsync(txt); } }
private async Task <int> UploadToAzureStorage(String messageString) { try { // create Azure Storage CloudStorageAccount storageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=https;AccountName=norkartstorageaccount;AccountKey=ywvq9KDDK/F1mqiz1NJ/vEj9lT7wuVrEqWtO1f3hi+i4vw9gOCvJG1rOjVKpjx/Ki1q6rVjG7uUalT+hWdXxCA=="); // create a blob client. CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); // create a container CloudBlobContainer container = blobClient.GetContainerReference("norkartstorageaccount"); CloudAppendBlob appBlob = container.GetAppendBlobReference("Temperature&Humidity/" + DateTime.Now.ToString("yyyy-dd-M") + ".json"); //CloudAppendBlob appBlob = container.GetAppendBlobReference("TestFile.txt"); System.Diagnostics.Debug.WriteLine("APPBLOB EXISTS: " + appBlob.ExistsAsync().Result.ToString()); if (appBlob.ExistsAsync().Result.ToString() != "True") { await appBlob.CreateOrReplaceAsync(); } // create a local file //StorageFile file = await ApplicationData.Current.LocalFolder.CreateFileAsync(DateTime.Now.ToString("yyyy-dd-M--HH-mm-ss"), CreationCollisionOption.GenerateUniqueName); //StorageFile file = await ApplicationData.Current.LocalFolder.CreateFileAsync("TestFile.txt", CreationCollisionOption.GenerateUniqueName); await appBlob.AppendTextAsync(messageString + Environment.NewLine); return(1); } catch { // return error System.Diagnostics.Debug.WriteLine("STORAGE ERROR"); return(0); } }
protected override void Write(LogEventInfo logEvent) { _client = _client ?? CloudStorageAccount.Parse(ConnectionString.Render(logEvent)).CreateCloudBlobClient(); if (_client == null) { return; } var containerName = Container.Render(logEvent); var blobName = BlobName.Render(logEvent); if (_container == null || _container.Name != containerName) { _container = _client.GetContainerReference(containerName); _container.CreateIfNotExistsAsync().Wait(); _blob = null; } if (_blob == null || _blob.Name != blobName || _forceCheck) { _blob = _container.GetAppendBlobReference(blobName); if (!_blob.ExistsAsync().Result) { try { _blob.Properties.ContentType = "text/plain"; _blob.CreateOrReplaceAsync().Wait(); _blob.SetPropertiesAsync().Wait(); } catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Conflict) { // to be expected } } } _blob.AppendTextAsync(Layout.Render(logEvent) + "\r\n").Wait(); }
public static void appendNewTextGP() { //Saving string as responses // /* * Here I found a problem, without reading previous RESPNUM(respondant number) and REF(ID), * I cannot add next RESPNUM and REF. * * Solution 1, * in a database, We need to add the number of responses, * so we can keep track of numbers when we add to csv. * * Solution 2, * Making a log file * * Solution 3, * Do not keep ID and numbers * * Solution 4, * Download a file, load into a memory and add ID. * * Having discussed, we remove RESPNUM and REF(instead of REF, SurveyID is added) */ string response = "1," + DateTime.Now.ToString("yyyyMMdd") + "," + DateTime.Now.ToString("hh:mm:ss") + "," + "1,2,2,1,2,1,5,1,7,5,1,5,4,4,1,2,2,2,2,2,4,3,3,32514, good" + System.Environment.NewLine; //file name string fileName = "GP_test.csv"; CloudAppendBlob blob = container.GetAppendBlobReference(fileName); //Create new blob file, if it does not exist. if (blob.ExistsAsync().Result == false) { blob.CreateOrReplaceAsync(); } //Append responses to the end of data. blob.AppendTextAsync(response); }
private async static void UploadText() { try { CloudBlobContainer textFilesContainer = _cloudBlobClient.GetContainerReference("text-files"); if (await textFilesContainer.ExistsAsync()) { CloudAppendBlob myTextBlob = textFilesContainer.GetAppendBlobReference($"text-{Guid.NewGuid().ToString("N")}.txt"); await myTextBlob.CreateOrReplaceAsync(); Console.WriteLine("Appending data to the file.."); for (int i = 1; i <= 100; i++) { string text = $"Line #{i} -> {Guid.NewGuid().ToString("N")}, @ {DateTime.UtcNow.ToLongTimeString()}{Environment.NewLine}"; await myTextBlob.AppendTextAsync(text); } Console.WriteLine("Finished. Now, adding some metadata"); myTextBlob.Metadata.Add("author", "Mirza Ghulam Rasyid"); myTextBlob.Metadata.Add("fileName", myTextBlob.Name); myTextBlob.Metadata.Add("creationDateTime", DateTime.UtcNow.ToString()); await myTextBlob.SetMetadataAsync(); Console.WriteLine("Finished"); } else { Console.WriteLine("Container doesn't exist"); } } catch (Exception ex) { Console.WriteLine(ex); } }
private async Task <bool> AppendResponse(object data) { CSVResponseAppendModel responseToAppend = jsonHelper.FromJson <CSVResponseAppendModel>(data.ToString()); //CSVResponseAppendModel responseToAppend = JsonConvert.DeserializeObject<CSVResponseAppendModel>(data.ToString()); bool result = false; string fileName = null; BaseResponseModel CSVResponse = responseService.GetSurveyFilename(responseToAppend.SurveyID); if (CSVResponse != null) { fileName = CSVResponse.ResponseCSV.Split("/").Last(); } try { CloudAppendBlob CSV = CONTAINER.GetAppendBlobReference(fileName); if (CSV.ExistsAsync().Result == false) //Create new blob file, if it does not exist. { await CSV.CreateOrReplaceAsync(); } await CSV.AppendTextAsync(responseToAppend.Response + System.Environment.NewLine); //Append responses to the end of data. result = true; } catch (Exception ex) { System.Diagnostics.Debug.WriteLine("Exception Caught - " + ex.Message); result = false; } return(result); }
public override async T.Task <bool> ProcessAsync(TaskEventMessage message, DateTimeOffset?insertionTime, CancellationToken token) { this.logger = this.Logger.ForContext("Job", message.JobId).ForContext("Task", message.Id); this.jobsTable = this.Utilities.GetJobsTable(); this.nodesTable = this.Utilities.GetNodesTable(); var nodeName = this.ServerOptions.HostName; JobType jobType = message.JobType; int jobId = message.JobId; int taskId = message.Id; int requeueCount = message.RequeueCount; this.nodePartitionKey = this.Utilities.GetNodePartitionKey(nodeName); this.jobPartitionKey = this.Utilities.GetJobPartitionKey(message.JobType, jobId); var taskKey = this.Utilities.GetTaskKey(jobId, taskId, requeueCount); var taskInfoKey = this.Utilities.GetTaskInfoKey(jobId, taskId, requeueCount); var taskResultKey = this.Utilities.GetTaskResultKey(jobId, taskId, requeueCount); logger.Information("Do work {0} for Task {1} on node {2}", message.EventVerb, taskKey, nodeName); if (insertionTime != null && insertionTime + TimeSpan.FromSeconds(10) < DateTimeOffset.UtcNow) { // Only when the insertion time is 10 seconds ago, we check the job status. var job = await this.jobsTable.RetrieveAsync <Job>(jobPartitionKey, this.Utilities.JobEntryKey, token); if (job.State != JobState.Running) { logger.Warning("Trying to start a task {0} when {1} Job {2} is in state {3}", taskKey, job.Type, job.Id, job.State); return(true); } } var task = await this.jobsTable.RetrieveAsync <Task>(this.jobPartitionKey, taskKey, token); int?exitCode = null; CloudAppendBlob taskResultBlob = null; var cmd = task.CommandLine; DateTimeOffset startTime = DateTimeOffset.UtcNow; var taskResult = new ComputeClusterTaskInformation() { ExitCode = -1, Message = "Running", CommandLine = cmd, JobId = jobId, TaskId = taskId, NodeName = nodeName, ResultKey = taskResultKey, StartTime = startTime, }; try { if (task.State != TaskState.Dispatching && task.State != TaskState.Running && task.State != TaskState.Queued) { Logger.Information("Job {0} task {1} state {2}, skip Executing command {3}", jobId, taskId, task.State, cmd); return(true); } var taskInfo = await this.jobsTable.RetrieveAsync <TaskStartInfo>(this.jobPartitionKey, taskInfoKey, token); Logger.Information("Executing command {0}", cmd); var rawResult = new StringBuilder(); using (var monitor = string.IsNullOrEmpty(cmd) ? null : this.Monitor.StartMonitorTask(jobId, taskKey, async(output, eof, cancellationToken) => { try { if (rawResult.Length < MaxRawResultLength) { rawResult.Append(output); } taskResultBlob = taskResultBlob ?? await this.Utilities.CreateOrReplaceJobOutputBlobAsync(jobType, taskResultKey, token); await taskResultBlob.AppendTextAsync(output, Encoding.UTF8, null, null, null, cancellationToken); if (eof) { taskResultBlob.Metadata[TaskOutputPage.EofMark] = eof.ToString(); await taskResultBlob.SetMetadataAsync(null, null, null, cancellationToken); } } catch (Exception ex) { logger.Error(ex, "Error happened when append to blob {0}", taskResultBlob.Name); } })) { if (!await this.PersistTaskResult(taskResultKey, taskResult, token)) { return(false); } logger.Information("Call startjobandtask for task {0}", taskKey); (taskInfo.StartInfo.environmentVariables ?? (taskInfo.StartInfo.environmentVariables = new Dictionary <string, string>())) .Add("blobEndpoint", this.Utilities.Account.BlobEndpoint.AbsoluteUri); taskInfo.StartInfo.stdout = taskInfo.StartInfo.stderr = $"{this.Communicator.Options.AgentUriBase}/output/{jobId}/{taskKey}"; await this.Utilities.UpdateTaskAsync(jobPartitionKey, taskKey, t => t.State = TaskState.Dispatching, token, this.logger); await this.Communicator.StartJobAndTaskAsync( nodeName, new StartJobAndTaskArg(new int[0], taskInfo.JobId, taskInfo.Id), taskInfo.UserName, taskInfo.Password, taskInfo.StartInfo, taskInfo.PrivateKey, taskInfo.PublicKey, token); logger.Information("Update task state to running"); await this.Utilities.UpdateTaskAsync(jobPartitionKey, taskKey, t => t.State = TaskState.Running, token, this.logger); logger.Information("Waiting for response"); if (monitor == null) { return(true); } ComputeNodeTaskCompletionEventArgs taskResultArgs; try { if (monitor.Result.Execution == await T.Task.WhenAny(monitor.Result.Execution, T.Task.Delay(TimeSpan.FromSeconds(task.MaximumRuntimeSeconds)))) { taskResultArgs = monitor.Result.Execution.Result; } else { logger.Information("Task has timed out"); return(true); } } catch (AggregateException ex) when(ex.InnerExceptions.All(e => e is OperationCanceledException)) { logger.Information("Task has been canceled"); return(true); } catch (T.TaskCanceledException) { logger.Information("Task has been canceled"); return(true); } taskResult = taskResultArgs.TaskInfo ?? taskResult; logger.Information("Updating task state with exit code {0}", taskResult?.ExitCode); await this.Utilities.UpdateTaskAsync(jobPartitionKey, taskKey, t => t.State = taskResult?.ExitCode == 0?TaskState.Finished : TaskState.Failed, token, this.logger); if (taskResult != null) { taskResult.StartTime = startTime; exitCode = taskResult.ExitCode; taskResult.Message = rawResult.Length > MaxRawResultLength?rawResult.ToString(0, MaxRawResultLength) : rawResult.ToString(); taskResult.CommandLine = cmd; taskResult.JobId = jobId; taskResult.TaskId = taskId; taskResult.NodeName = nodeName; taskResult.ResultKey = taskResultKey; taskResult.EndTime = DateTimeOffset.UtcNow; logger.Information("Saving result"); if (!await this.PersistTaskResult(taskResultKey, taskResult, token)) { return(false); } } } } catch (StorageException ex) when(ex.IsCancellation()) { return(false); } catch (OperationCanceledException) when(token.IsCancellationRequested) { return(false); } catch (Exception ex) { taskResult.Message = ex.ToString(); taskResult.EndTime = DateTimeOffset.UtcNow; await this.PersistTaskResult(taskResultKey, taskResult, token); await this.Utilities.UpdateTaskAsync(jobPartitionKey, taskKey, t => t.State = t.State == TaskState.Dispatching || t.State == TaskState.Running?TaskState.Failed : t.State, token, this.logger); await this.Utilities.AddJobsEventAsync(jobType, jobId, $"Task {taskId}, exception {ex}", EventType.Warning, token); } finally { var queue = this.Utilities.GetJobTaskCompletionQueue(jobId); logger.Information("Adding task completion message"); await queue.AddMessageAsync(new CloudQueueMessage(JsonConvert.SerializeObject(new TaskCompletionMessage() { JobId = jobId, Id = taskId, ExitCode = exitCode, JobType = jobType, RequeueCount = requeueCount, ChildIds = task.ChildIds, }, Formatting.Indented)), null, null, null, null, token); taskResultBlob = taskResultBlob ?? await this.Utilities.CreateOrReplaceJobOutputBlobAsync(jobType, taskResultKey, token); taskResultBlob.Metadata[TaskOutputPage.EofMark] = true.ToString(); await taskResultBlob.SetMetadataAsync(null, null, null, token); logger.Information("Finished"); } return(true); }
private static async Task <int> BackUpCollection(CloudStorageAccount storageAccount, DateTime starttime) { Uri collectionURI = UriFactory.CreateDocumentCollectionUri(DatabaseName, CollectionName); //Our changes file daily roll example CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); //Setup our container we are going to use and create it. CloudBlobContainer container = blobClient.GetContainerReference("dbchanges"); await container.CreateIfNotExistsAsync(); DateTime changefiledate = DateTime.Today; CloudAppendBlob appBlob = container.GetAppendBlobReference( string.Format("{0}_{1}_{2}{3}", DatabaseName, CollectionName, changefiledate.ToString("yyyyMMdd"), ".json")); var exists = await appBlob.ExistsAsync(); if (!exists) { await appBlob.CreateOrReplaceAsync(); await appBlob.AppendTextAsync("["); //Store file as JSON Array for easy import } int x = 0; string pkRangesResponseContinuation = null; using (var client = new DocumentClient(new Uri(endpointUrl), authorizationKey, new ConnectionPolicy { ConnectionMode = ConnectionMode.Direct, ConnectionProtocol = Protocol.Tcp })) { var pkRangesResponse = await client.ReadPartitionKeyRangeFeedAsync(collectionURI, new FeedOptions { RequestContinuation = pkRangesResponseContinuation }); List <PartitionKeyRange> partitionKeyRanges = new List <PartitionKeyRange>(); partitionKeyRanges.AddRange(pkRangesResponse); pkRangesResponseContinuation = pkRangesResponse.ResponseContinuation; Dictionary <string, string> checkpoints = new Dictionary <string, string>(); bool comma = exists; foreach (PartitionKeyRange pkRange in partitionKeyRanges) { string continuation = null; checkpoints.TryGetValue(pkRange.Id, out continuation); IDocumentQuery <Document> query = client.CreateDocumentChangeFeedQuery( collectionURI, new ChangeFeedOptions { PartitionKeyRangeId = pkRange.Id, StartFromBeginning = true, RequestContinuation = continuation, MaxItemCount = -1, // Set reading time: only show change feed results modified since StartTime StartTime = starttime }); while (query.HasMoreResults) { FeedResponse <dynamic> readChangesResponse = query.ExecuteNextAsync <dynamic>().Result; foreach (dynamic changedDocument in readChangesResponse) { Console.WriteLine("document: {0}", changedDocument); await appBlob.AppendTextAsync((comma ? "," : "") + changedDocument); x++; comma = true; } checkpoints[pkRange.Id] = readChangesResponse.ResponseContinuation; } } } return(x); }
public static void AppendText(this CloudAppendBlob blob, string content) { blob.AppendTextAsync(content).Wait(); }