public static async void UploadAsyncLogs(string log)
{
try
{
//pathcre();
//Console.WriteLine(log);
if (userid == "0" || userid == "DefaultLogs")
{
if (CurrentUser.GetGuestId() == "0" || CurrentUser.GetGuestId() == null)
{
userid = "DefaultLogs";
}
else
{
userid = "g_" + CurrentUser.GetGuestId();
}
}
CloudAppendBlob append = container.GetAppendBlobReference(userid + ".csv");
Console.WriteLine("User id " + userid + " ");
Console.WriteLine(log);
if (!await append.ExistsAsync())
{
await append.CreateOrReplaceAsync();
}
await append.AppendTextAsync(log);
}
catch (Exception ex)
{
//Console.WriteLine(ex.Message);
}
}
public static void LogServiceInfo(string info, string servicename)
{
var TaskA = new System.Threading.Tasks.Task(() =>
{
pathcre();
try
{
UploadErrorLogs();
append = container.GetAppendBlobReference(userid + ".csv");
DateTime date1 = DateTime.UtcNow;
var tasks = new Task[1];
for (int i = 0; i < 1; i++)
{
tasks[i] = append.AppendTextAsync(string.Format("{0},{1},{2},{3}", "Service", date1.ToString("MM/dd/yyyy hh:mm:ss.fff"), info, servicename + "\n"));
}
Task.WaitAll(tasks);
}
catch (Exception exe)
{
Log.Error("Error", exe.Message);
}
});
TaskA.Start();
}
public static void LogServiceInfo(string info, string servicename)
{
pathcre();
try
{
UploadErrorLogs();
append = container.GetAppendBlobReference(userid + ".csv");
DateTime date1 = DateTime.UtcNow;
var tasks = new Task[1];
for (int i = 0; i < 1; i++)
{
tasks[i] = append.AppendTextAsync(string.Format("{0},{1},{2},{3}", "Service", date1.ToString("MM/dd/yyyy hh:mm:ss.fff"), info, servicename + "\n"));
}
Task.WaitAll(tasks);
//var csv = new StringBuilder();
// var newLine = string.Format("{0},{1},{2},{3}", "Service", DateTime.Now, info, servicename);
// csv.AppendLine(newLine);
// File.AppendAllText(logspath, csv.ToString());
//await append.AppendTextAsync(string.Format("{0},{1},{2},{3},{4}", "Exception", DateTime.Now, lineno, screenid + "\n"));
}
catch (Exception exe)
{
Log.Error("Error", exe.Message);
}
}
public async Task Test_31_Blob_withInfiniteLease()
{
CloudBlobContainer cbc = _Client.GetContainerReference("test-append-blob-container");
await cbc.CreateIfNotExistsAsync();
CloudAppendBlob blob = cbc.GetAppendBlobReference("test-append-blob-withlease2.data");
var blobExists = await blob.ExistsAsync();
if (!blobExists)
{
await blob.CreateOrReplaceAsync();
}
string data = string.Empty.PadLeft(64 * 1024, '*');
string leaseIdGuid = Guid.NewGuid().ToString();
var oc = new OperationContext();
var ac = new AccessCondition();
var leaseID = await blob.AcquireLeaseAsync(null, leaseIdGuid, ac, null, null);
ac.LeaseId = leaseID;
try
{
for (int ix = 0; ix < 10; ix++)
{
await blob.AppendTextAsync(data, null, ac, null, oc);
}
}
finally
{
await blob.ReleaseLeaseAsync(ac, null, oc);
}
}
/// <summary>
/// Log the event to persistent store
/// </summary>
/// <param name="username">username of the caller</param>
/// <param name="message">message from the caller</param>
private static async void Persist(string username, string message)
{
string accountKey = Environment.GetEnvironmentVariable("AccountKey");
string accountName = Environment.GetEnvironmentVariable("AccountName");
// Implement the accout, set true for https for SSL.
StorageCredentials creds = new StorageCredentials(accountName, accountKey);
CloudStorageAccount strAcc = new CloudStorageAccount(creds, true);
CloudBlobClient blobClient = strAcc.CreateCloudBlobClient();
// Setup our container we are going to use and create it.
CloudBlobContainer container = blobClient.GetContainerReference("logs");
await container.CreateIfNotExistsAsync();
// Build my typical log file name.
DateTime date = DateTime.Today;
DateTime dateLogEntry = DateTime.Now;
// This creates a reference to the append blob we are going to use.
CloudAppendBlob appBlob = container.GetAppendBlobReference(
string.Format("{0}{1}", date.ToString("yyyyMMdd"), ".log"));
// Now we are going to check if todays file exists and if it doesn't we create it.
if (!await appBlob.ExistsAsync())
{
await appBlob.CreateOrReplaceAsync();
}
// Add the entry to our log.
await appBlob.AppendTextAsync($"{dateLogEntry.ToString("o")}-{username}-{message}\r\n");
}
public async Task UploadFile(string sourcePath, string sourceFilename, string containerName, string targetFilename, string contentType, bool append = false)
{
byte[] fileContent = File.ReadAllBytes(Path.Join(fixPath(sourcePath), sourceFilename));
CloudStorageAccount account = CloudStorageAccount.Parse(storageConnectionString);
CloudBlobClient serviceClient = account.CreateCloudBlobClient();
var container = serviceClient.GetContainerReference(getContainerName(containerName));
container.CreateIfNotExistsAsync().Wait();
CloudAppendBlob blob = container.GetAppendBlobReference(getFilepathForContainer(containerName, targetFilename));
if (!append)
{
await blob.CreateOrReplaceAsync();
}
else
{
if (!blob.ExistsAsync().Result)
{
throw new Exception($"Cannot append to nonexistent blob file {sourceFilename}");
}
}
blob.Properties.ContentType = contentType;
await blob.AppendTextAsync(fileContent.ToString());
}
public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req,
ILogger log)
{
string name = req.Query["name"];
try
{
log.LogInformation("C# HTTP trigger function processed a request.");
var StorageURL = Environment.GetEnvironmentVariable("StorageURL");
string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
dynamic data = JsonConvert.DeserializeObject(requestBody);
name = name ?? data?.name;
var azureServiceTokenProvider = new AzureServiceTokenProvider();
string accessToken = await azureServiceTokenProvider.GetAccessTokenAsync("https://storage.azure.com");
log.LogInformation("accessToken : retrieved");
// create the credential, using the
var tokenCredential = new Microsoft.WindowsAzure.Storage.Auth.TokenCredential(accessToken);
var storageCredentials = new StorageCredentials(tokenCredential);
log.LogInformation("credentials : created");
var fileName = "append";
var Uri = new Uri(StorageURL + fileName);
var blob = new CloudAppendBlob(Uri, storageCredentials);
log.LogInformation($"blobfile : setup {0}", Uri);
if (!(await blob.ExistsAsync()))
{
await blob.CreateOrReplaceAsync(AccessCondition.GenerateIfNotExistsCondition(), null, null);
}
await blob.AppendTextAsync(name);
var fileName2 = "regular.txt";
var Uri2 = new Uri(StorageURL + fileName2);
var blob2 = new CloudBlockBlob(Uri2, storageCredentials);
await blob2.UploadTextAsync(name);
}
catch (Exception ex)
{
log.LogInformation($"EXEC {ex.ToString()} ");
}
return(name != null
? (ActionResult) new OkObjectResult($"Hello, {name}")
: new BadRequestObjectResult("Please pass a name on the query string or in the request body"));
}
private async static void AcquireLeaseDemo()
{
try
{
string name = "LeaseDemo/text-0221e06e5e254fb3bb0bf0e37d9d0e32.txt";
CloudBlobContainer container = _cloudBlobClient.GetContainerReference("text-files");
await container.CreateIfNotExistsAsync();
CloudAppendBlob cloudAppendBlob = container.GetAppendBlobReference(name);
bool shouldContinue = true;
if (await cloudAppendBlob.ExistsAsync())
{
if (cloudAppendBlob.Properties.LeaseStatus == LeaseStatus.Locked)
{
shouldContinue = false;
Console.WriteLine("Blob is currently in lease");
}
else
{
await cloudAppendBlob.CreateOrReplaceAsync();
}
}
if (shouldContinue)
{
string proposedLeaseId = Guid.NewGuid().ToString("n");
AccessCondition accessCondition = new AccessCondition();
OperationContext operationContext = new OperationContext();
Console.WriteLine("Acquiring lock...");
string leaseId = await cloudAppendBlob.AcquireLeaseAsync(null, proposedLeaseId, accessCondition, null, operationContext);
accessCondition.LeaseId = leaseId;
try
{
Console.WriteLine("Uploading text...");
for (int i = 0; i <= 50; i++)
{
await cloudAppendBlob.AppendTextAsync($"{Guid.NewGuid().ToString("n")}{Environment.NewLine}", UTF8Encoding.UTF8, accessCondition, null, operationContext);
}
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
finally
{
await cloudAppendBlob.ReleaseLeaseAsync(accessCondition, null, operationContext);
Console.WriteLine("Process finished. Lock has been released");
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
}
public async Task AppendBlob(string containerName, string blobName, string content)
{
var blobContainer = await PreapreBLobContainer(containerName);
CloudAppendBlob appendBlob = blobContainer.GetAppendBlobReference(blobName);
await appendBlob.CreateOrReplaceAsync();
await appendBlob.AppendTextAsync(content);
}
public void Write(string content)
{
try
{
_blob.AppendTextAsync(String.Format("Timestamp: {0:u} \tLog Entry: {1}{2}",
DateTime.UtcNow, content, Environment.NewLine)).Wait();
}
catch
{
}
}
//public static string CreateDirectoryForLogs()
// {
// try
// {
// App._dir = new Java.IO.File(Android.OS.Environment.GetExternalStoragePublicDirectory("WineHangouts"), "winehangouts/logs");
// if (!App._dir.Exists())
// {
// App._dir.Mkdirs();
// }
// logspath = App._dir.ToString() + "/" + CurrentUser.getUserId() + ".csv";
// }
// catch (Exception exe)
// {
// Log.Error("Error", exe.Message);
// }
// return logspath;
// }
public static void LogInfo(string info, int screenid)
{
pathcre();
try
{
append = container.GetAppendBlobReference(userid + ".csv");
DateTime date1 = DateTime.UtcNow;
append.AppendTextAsync(string.Format("{0},{1},{2},{3}", "Info", date1.ToString("MM/dd/yyyy hh:mm:ss.fff"), info, screenid + "\n"));
}
catch (Exception exe)
{
Log.Error("Error", exe.Message);
}
}
protected async Task EnqueueAsync(IEnumerable <string> records)
{
try
{
using (StateVar)
{
SourcedState state = await StateVar.GetAsync();
if (!await file.ExistsAsync())
{
await file.CreateOrReplaceAsync();
}
long currentVersion = state.CurrentVersion;
StringBuilder sb = new StringBuilder();
Dictionary <long, RecordContainer> containers = new Dictionary <long, RecordContainer>();
foreach (var data in records)
{
//Create the container in memory
RecordContainer container = new RecordContainer(++currentVersion, data.Replace("\r", "").Replace("\n", ""));
containers.Add(container.VersionId, container);
//Test if need to create pointer
if (currentVersion % SourcedState.PointersToRecordsChunkSize == 0)
{
state.PointersToRecords.Add(currentVersion, await GetFileSize());
}
//Create string for storage
sb.Append(container.VersionId + ",");
sb.Append(container.Data);
sb.Append("\n");
}
//Write it to storage, this only stays congruent because this is single threaded
await file.AppendTextAsync(sb.ToString());
//Add records to the state and update CurrentVersion
state.AddNewRecords(containers);
}
}
catch (Exception e)
{
throw new RecordWriteException(e);
}
}
public async Task SaveDataToBlobAsync(string data)
{
try
{
CloudAppendBlob blob = _dataContainer.GetAppendBlobReference(GetBlobName());
if (!blob.Exists())
{
blob.CreateOrReplace();
}
await blob.AppendTextAsync(data);
}
catch (Exception ex)
{
_errorLogger.ErrorFormat("SaveDataToBlobAsync function encounter error : {0}",
ex.ToString().Replace("\r\n", " "));
}
}
public async Task Test_30_Blob_InfiniteLease()
{
// Containers are like folders
CloudBlobContainer container = _client.GetContainerReference("test-append-blob-container");
await container.CreateIfNotExistsAsync();
// Blobs are like files
CloudAppendBlob blob = container.GetAppendBlobReference("test-append-blob-with-lease");
var blobExists = await blob.ExistsAsync();
if (!blobExists)
{
await blob.CreateOrReplaceAsync();
}
string data = string.Empty.PadLeft(64 * 1024, '*');
string leaseId = Guid.NewGuid().ToString();
var operationContext = new OperationContext();
var accessCondition = new AccessCondition();
// For pessimistic access control
// Acquire lease
var leaseID = await blob.AcquireLeaseAsync(null, leaseId, accessCondition, null, operationContext);
accessCondition.LeaseId = leaseID;
try
{
// Make modifications based on that lease
for (int i = 0; i < 1000; i++)
{
await blob.AppendTextAsync(data, null, accessCondition, null, operationContext);
}
}
finally
{
// And finally release that lease
await blob.ReleaseLeaseAsync(accessCondition, null, operationContext);
}
}
public static void UploadNewText()
{
Console.Write("Please enter text to add to the blob: ");
string text = Console.ReadLine();
//file name
string fileName = "test.csv";
CloudAppendBlob blob = container.GetAppendBlobReference(fileName);
//Create new blob file, if it does not exist.
if (blob.ExistsAsync().Result == false)
{
blob.CreateOrReplaceAsync();
}
//Append text to the end of data.
blob.AppendTextAsync(text);
}
public async Task Test_10_AppendBlob()
{
CloudBlobContainer cbc = _Client.GetContainerReference("test-append-blob-container");
await cbc.CreateIfNotExistsAsync();
CloudAppendBlob blob = cbc.GetAppendBlobReference("test-append-blob.data");
await blob.CreateOrReplaceAsync();
for (int ix = 0; ix < 100; ix++)
{
string txt =
$"Line #{ix + 1:d6}, written at {DateTime.UtcNow.ToLongTimeString() }: " +
string.Empty.PadRight(20, '*') +
Environment.NewLine;
await blob.AppendTextAsync(txt);
}
}
public static void LogTime(string info, string time)
{
pathcre();
try
{
append = container.GetAppendBlobReference(CurrentUser.getUserId() + ".csv");
DateTime date2 = DateTime.UtcNow;
append.AppendTextAsync(string.Format("{0},{1},{2},{3}", "Time", date2.ToString("MM/dd/yyyy hh:mm:ss.fff"), info, time + "\n"));
//var csv = new StringBuilder();
//var newLine = string.Format("{0},{1},{2},{3},{4}", "Exception", DateTime.Now, error,lineno,screenid);
//csv.AppendLine(newLine);
//File.AppendAllText(logspath, csv.ToString());
}
catch (Exception exe)
{
Log.Error("Error", exe.Message);
}
}
public static void LogTime(string info, string time)
{
var TaskA = new System.Threading.Tasks.Task(() =>
{
pathcre();
try
{
append = container.GetAppendBlobReference(CurrentUser.getUserId() + ".csv");
DateTime date2 = DateTime.UtcNow;
append.AppendTextAsync(string.Format("{0},{1},{2},{3}", "Time", date2.ToString("MM/dd/yyyy hh:mm:ss.fff"), info, time + "\n"));
}
catch (Exception exe)
{
Log.Error("Error", exe.Message);
}
});
TaskA.Start();
}
public static void LogInfoEx(string info, int screenid)
{
pathcre();
try
{
DateTime date = DateTime.UtcNow;
append = container.GetAppendBlobReference(userid + ".csv");
var tasks = new Task[1];
for (int i = 0; i < 1; i++)
{
tasks[i] = append.AppendTextAsync(string.Format("{0},{1},{2},{3}", "Info", date.ToString("MM/dd/yyyy hh:mm:ss.fff"), info, screenid + "\n"));
}
Task.WaitAll(tasks);
}
catch (Exception exe)
{
Log.Error("Error", exe.Message);
}
}
public async Task UploadFile(string sourcePath, string sourceFilename, string sasUri, string contentType, bool append = false)
{
byte[] fileContent = File.ReadAllBytes(Path.Join(fixPath(sourcePath), sourceFilename));
CloudAppendBlob blob = new CloudAppendBlob(new Uri(sasUri));
if (!append)
{
await blob.CreateOrReplaceAsync();
}
else
{
if (!blob.ExistsAsync().Result)
{
throw new Exception($"Cannot append to nonexistent blob file {sourceFilename}");
}
}
blob.Properties.ContentType = contentType;
await blob.AppendTextAsync(fileContent.ToString());
}
public static void LogError(string error, int screenid, string lineno)
{
var TaskA = new System.Threading.Tasks.Task(() =>
{
pathcre();
try
{
append = container.GetAppendBlobReference(userid + ".csv");
DateTime date2 = DateTime.UtcNow;
append.AppendTextAsync(string.Format("{0},{1},{2},{3},{4}", "Exception", date2.ToString("MM/dd/yyyy hh:mm:ss.fff"), error, lineno, screenid + "\n"));
}
catch (Exception exe)
{
Log.Error("Error", exe.Message);
}
});
TaskA.Start();
}
public async Task Test_10_AppendBlob()
{
// Containers are like folders
CloudBlobContainer container = _client.GetContainerReference("test-append-blob-container");
await container.CreateIfNotExistsAsync();
// Blobs are like files
CloudAppendBlob blob = container.GetAppendBlobReference("test-append-blob");
await blob.CreateOrReplaceAsync();
for (int i = 0; i < 100; i++)
{
var txt = $"Line #{(i + 1).ToString("d6")}, written at {DateTime.UtcNow.ToLongTimeString()}: {string.Empty.PadRight(20, '*')}" + Environment.NewLine;
await blob.AppendTextAsync(txt);
}
}
private async Task <int> UploadToAzureStorage(String messageString)
{
try
{
// create Azure Storage
CloudStorageAccount storageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=https;AccountName=norkartstorageaccount;AccountKey=ywvq9KDDK/F1mqiz1NJ/vEj9lT7wuVrEqWtO1f3hi+i4vw9gOCvJG1rOjVKpjx/Ki1q6rVjG7uUalT+hWdXxCA==");
// create a blob client.
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// create a container
CloudBlobContainer container = blobClient.GetContainerReference("norkartstorageaccount");
CloudAppendBlob appBlob = container.GetAppendBlobReference("Temperature&Humidity/" + DateTime.Now.ToString("yyyy-dd-M") + ".json");
//CloudAppendBlob appBlob = container.GetAppendBlobReference("TestFile.txt");
System.Diagnostics.Debug.WriteLine("APPBLOB EXISTS: " + appBlob.ExistsAsync().Result.ToString());
if (appBlob.ExistsAsync().Result.ToString() != "True")
{
await appBlob.CreateOrReplaceAsync();
}
// create a local file
//StorageFile file = await ApplicationData.Current.LocalFolder.CreateFileAsync(DateTime.Now.ToString("yyyy-dd-M--HH-mm-ss"), CreationCollisionOption.GenerateUniqueName);
//StorageFile file = await ApplicationData.Current.LocalFolder.CreateFileAsync("TestFile.txt", CreationCollisionOption.GenerateUniqueName);
await appBlob.AppendTextAsync(messageString + Environment.NewLine);
return(1);
}
catch
{
// return error
System.Diagnostics.Debug.WriteLine("STORAGE ERROR");
return(0);
}
}
protected override void Write(LogEventInfo logEvent)
{
_client = _client ?? CloudStorageAccount.Parse(ConnectionString.Render(logEvent)).CreateCloudBlobClient();
if (_client == null)
{
return;
}
var containerName = Container.Render(logEvent);
var blobName = BlobName.Render(logEvent);
if (_container == null || _container.Name != containerName)
{
_container = _client.GetContainerReference(containerName);
_container.CreateIfNotExistsAsync().Wait();
_blob = null;
}
if (_blob == null || _blob.Name != blobName || _forceCheck)
{
_blob = _container.GetAppendBlobReference(blobName);
if (!_blob.ExistsAsync().Result)
{
try
{
_blob.Properties.ContentType = "text/plain";
_blob.CreateOrReplaceAsync().Wait();
_blob.SetPropertiesAsync().Wait();
}
catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Conflict)
{
// to be expected
}
}
}
_blob.AppendTextAsync(Layout.Render(logEvent) + "\r\n").Wait();
}
public static void appendNewTextGP()
{
//Saving string as responses //
/*
* Here I found a problem, without reading previous RESPNUM(respondant number) and REF(ID),
* I cannot add next RESPNUM and REF.
*
* Solution 1,
* in a database, We need to add the number of responses,
* so we can keep track of numbers when we add to csv.
*
* Solution 2,
* Making a log file
*
* Solution 3,
* Do not keep ID and numbers
*
* Solution 4,
* Download a file, load into a memory and add ID.
*
* Having discussed, we remove RESPNUM and REF(instead of REF, SurveyID is added)
*/
string response = "1," + DateTime.Now.ToString("yyyyMMdd") + "," + DateTime.Now.ToString("hh:mm:ss")
+ "," + "1,2,2,1,2,1,5,1,7,5,1,5,4,4,1,2,2,2,2,2,4,3,3,32514, good" + System.Environment.NewLine;
//file name
string fileName = "GP_test.csv";
CloudAppendBlob blob = container.GetAppendBlobReference(fileName);
//Create new blob file, if it does not exist.
if (blob.ExistsAsync().Result == false)
{
blob.CreateOrReplaceAsync();
}
//Append responses to the end of data.
blob.AppendTextAsync(response);
}
private async static void UploadText()
{
try
{
CloudBlobContainer textFilesContainer = _cloudBlobClient.GetContainerReference("text-files");
if (await textFilesContainer.ExistsAsync())
{
CloudAppendBlob myTextBlob = textFilesContainer.GetAppendBlobReference($"text-{Guid.NewGuid().ToString("N")}.txt");
await myTextBlob.CreateOrReplaceAsync();
Console.WriteLine("Appending data to the file..");
for (int i = 1; i <= 100; i++)
{
string text = $"Line #{i} -> {Guid.NewGuid().ToString("N")}, @ {DateTime.UtcNow.ToLongTimeString()}{Environment.NewLine}";
await myTextBlob.AppendTextAsync(text);
}
Console.WriteLine("Finished. Now, adding some metadata");
myTextBlob.Metadata.Add("author", "Mirza Ghulam Rasyid");
myTextBlob.Metadata.Add("fileName", myTextBlob.Name);
myTextBlob.Metadata.Add("creationDateTime", DateTime.UtcNow.ToString());
await myTextBlob.SetMetadataAsync();
Console.WriteLine("Finished");
}
else
{
Console.WriteLine("Container doesn't exist");
}
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
}
private async Task <bool> AppendResponse(object data)
{
CSVResponseAppendModel responseToAppend = jsonHelper.FromJson <CSVResponseAppendModel>(data.ToString());
//CSVResponseAppendModel responseToAppend = JsonConvert.DeserializeObject<CSVResponseAppendModel>(data.ToString());
bool result = false;
string fileName = null;
BaseResponseModel CSVResponse = responseService.GetSurveyFilename(responseToAppend.SurveyID);
if (CSVResponse != null)
{
fileName = CSVResponse.ResponseCSV.Split("/").Last();
}
try
{
CloudAppendBlob CSV = CONTAINER.GetAppendBlobReference(fileName);
if (CSV.ExistsAsync().Result == false) //Create new blob file, if it does not exist.
{
await CSV.CreateOrReplaceAsync();
}
await CSV.AppendTextAsync(responseToAppend.Response + System.Environment.NewLine); //Append responses to the end of data.
result = true;
}
catch (Exception ex)
{
System.Diagnostics.Debug.WriteLine("Exception Caught - " + ex.Message);
result = false;
}
return(result);
}
public override async T.Task <bool> ProcessAsync(TaskEventMessage message, DateTimeOffset?insertionTime, CancellationToken token)
{
this.logger = this.Logger.ForContext("Job", message.JobId).ForContext("Task", message.Id);
this.jobsTable = this.Utilities.GetJobsTable();
this.nodesTable = this.Utilities.GetNodesTable();
var nodeName = this.ServerOptions.HostName;
JobType jobType = message.JobType;
int jobId = message.JobId;
int taskId = message.Id;
int requeueCount = message.RequeueCount;
this.nodePartitionKey = this.Utilities.GetNodePartitionKey(nodeName);
this.jobPartitionKey = this.Utilities.GetJobPartitionKey(message.JobType, jobId);
var taskKey = this.Utilities.GetTaskKey(jobId, taskId, requeueCount);
var taskInfoKey = this.Utilities.GetTaskInfoKey(jobId, taskId, requeueCount);
var taskResultKey = this.Utilities.GetTaskResultKey(jobId, taskId, requeueCount);
logger.Information("Do work {0} for Task {1} on node {2}", message.EventVerb, taskKey, nodeName);
if (insertionTime != null && insertionTime + TimeSpan.FromSeconds(10) < DateTimeOffset.UtcNow)
{
// Only when the insertion time is 10 seconds ago, we check the job status.
var job = await this.jobsTable.RetrieveAsync <Job>(jobPartitionKey, this.Utilities.JobEntryKey, token);
if (job.State != JobState.Running)
{
logger.Warning("Trying to start a task {0} when {1} Job {2} is in state {3}", taskKey, job.Type, job.Id, job.State);
return(true);
}
}
var task = await this.jobsTable.RetrieveAsync <Task>(this.jobPartitionKey, taskKey, token);
int?exitCode = null;
CloudAppendBlob taskResultBlob = null;
var cmd = task.CommandLine;
DateTimeOffset startTime = DateTimeOffset.UtcNow;
var taskResult = new ComputeClusterTaskInformation()
{
ExitCode = -1,
Message = "Running",
CommandLine = cmd,
JobId = jobId,
TaskId = taskId,
NodeName = nodeName,
ResultKey = taskResultKey,
StartTime = startTime,
};
try
{
if (task.State != TaskState.Dispatching && task.State != TaskState.Running && task.State != TaskState.Queued)
{
Logger.Information("Job {0} task {1} state {2}, skip Executing command {3}", jobId, taskId, task.State, cmd);
return(true);
}
var taskInfo = await this.jobsTable.RetrieveAsync <TaskStartInfo>(this.jobPartitionKey, taskInfoKey, token);
Logger.Information("Executing command {0}", cmd);
var rawResult = new StringBuilder();
using (var monitor = string.IsNullOrEmpty(cmd) ? null : this.Monitor.StartMonitorTask(jobId, taskKey, async(output, eof, cancellationToken) =>
{
try
{
if (rawResult.Length < MaxRawResultLength)
{
rawResult.Append(output);
}
taskResultBlob = taskResultBlob ?? await this.Utilities.CreateOrReplaceJobOutputBlobAsync(jobType, taskResultKey, token);
await taskResultBlob.AppendTextAsync(output, Encoding.UTF8, null, null, null, cancellationToken);
if (eof)
{
taskResultBlob.Metadata[TaskOutputPage.EofMark] = eof.ToString();
await taskResultBlob.SetMetadataAsync(null, null, null, cancellationToken);
}
}
catch (Exception ex)
{
logger.Error(ex, "Error happened when append to blob {0}", taskResultBlob.Name);
}
}))
{
if (!await this.PersistTaskResult(taskResultKey, taskResult, token))
{
return(false);
}
logger.Information("Call startjobandtask for task {0}", taskKey);
(taskInfo.StartInfo.environmentVariables ?? (taskInfo.StartInfo.environmentVariables = new Dictionary <string, string>()))
.Add("blobEndpoint", this.Utilities.Account.BlobEndpoint.AbsoluteUri);
taskInfo.StartInfo.stdout = taskInfo.StartInfo.stderr = $"{this.Communicator.Options.AgentUriBase}/output/{jobId}/{taskKey}";
await this.Utilities.UpdateTaskAsync(jobPartitionKey, taskKey, t => t.State = TaskState.Dispatching, token, this.logger);
await this.Communicator.StartJobAndTaskAsync(
nodeName,
new StartJobAndTaskArg(new int[0], taskInfo.JobId, taskInfo.Id), taskInfo.UserName, taskInfo.Password,
taskInfo.StartInfo, taskInfo.PrivateKey, taskInfo.PublicKey, token);
logger.Information("Update task state to running");
await this.Utilities.UpdateTaskAsync(jobPartitionKey, taskKey, t => t.State = TaskState.Running, token, this.logger);
logger.Information("Waiting for response");
if (monitor == null)
{
return(true);
}
ComputeNodeTaskCompletionEventArgs taskResultArgs;
try
{
if (monitor.Result.Execution == await T.Task.WhenAny(monitor.Result.Execution, T.Task.Delay(TimeSpan.FromSeconds(task.MaximumRuntimeSeconds))))
{
taskResultArgs = monitor.Result.Execution.Result;
}
else
{
logger.Information("Task has timed out");
return(true);
}
}
catch (AggregateException ex) when(ex.InnerExceptions.All(e => e is OperationCanceledException))
{
logger.Information("Task has been canceled");
return(true);
}
catch (T.TaskCanceledException)
{
logger.Information("Task has been canceled");
return(true);
}
taskResult = taskResultArgs.TaskInfo ?? taskResult;
logger.Information("Updating task state with exit code {0}", taskResult?.ExitCode);
await this.Utilities.UpdateTaskAsync(jobPartitionKey, taskKey,
t => t.State = taskResult?.ExitCode == 0?TaskState.Finished : TaskState.Failed,
token,
this.logger);
if (taskResult != null)
{
taskResult.StartTime = startTime;
exitCode = taskResult.ExitCode;
taskResult.Message = rawResult.Length > MaxRawResultLength?rawResult.ToString(0, MaxRawResultLength) : rawResult.ToString();
taskResult.CommandLine = cmd;
taskResult.JobId = jobId;
taskResult.TaskId = taskId;
taskResult.NodeName = nodeName;
taskResult.ResultKey = taskResultKey;
taskResult.EndTime = DateTimeOffset.UtcNow;
logger.Information("Saving result");
if (!await this.PersistTaskResult(taskResultKey, taskResult, token))
{
return(false);
}
}
}
}
catch (StorageException ex) when(ex.IsCancellation())
{
return(false);
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
return(false);
}
catch (Exception ex)
{
taskResult.Message = ex.ToString();
taskResult.EndTime = DateTimeOffset.UtcNow;
await this.PersistTaskResult(taskResultKey, taskResult, token);
await this.Utilities.UpdateTaskAsync(jobPartitionKey, taskKey, t => t.State = t.State == TaskState.Dispatching || t.State == TaskState.Running?TaskState.Failed : t.State, token, this.logger);
await this.Utilities.AddJobsEventAsync(jobType, jobId, $"Task {taskId}, exception {ex}", EventType.Warning, token);
}
finally
{
var queue = this.Utilities.GetJobTaskCompletionQueue(jobId);
logger.Information("Adding task completion message");
await queue.AddMessageAsync(new CloudQueueMessage(JsonConvert.SerializeObject(new TaskCompletionMessage()
{
JobId = jobId,
Id = taskId,
ExitCode = exitCode,
JobType = jobType,
RequeueCount = requeueCount,
ChildIds = task.ChildIds,
}, Formatting.Indented)), null, null, null, null, token);
taskResultBlob = taskResultBlob ?? await this.Utilities.CreateOrReplaceJobOutputBlobAsync(jobType, taskResultKey, token);
taskResultBlob.Metadata[TaskOutputPage.EofMark] = true.ToString();
await taskResultBlob.SetMetadataAsync(null, null, null, token);
logger.Information("Finished");
}
return(true);
}
private static async Task <int> BackUpCollection(CloudStorageAccount storageAccount, DateTime starttime)
{
Uri collectionURI = UriFactory.CreateDocumentCollectionUri(DatabaseName, CollectionName);
//Our changes file daily roll example
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
//Setup our container we are going to use and create it.
CloudBlobContainer container = blobClient.GetContainerReference("dbchanges");
await container.CreateIfNotExistsAsync();
DateTime changefiledate = DateTime.Today;
CloudAppendBlob appBlob = container.GetAppendBlobReference(
string.Format("{0}_{1}_{2}{3}", DatabaseName, CollectionName, changefiledate.ToString("yyyyMMdd"), ".json"));
var exists = await appBlob.ExistsAsync();
if (!exists)
{
await appBlob.CreateOrReplaceAsync();
await appBlob.AppendTextAsync("[");
//Store file as JSON Array for easy import
}
int x = 0;
string pkRangesResponseContinuation = null;
using (var client = new DocumentClient(new Uri(endpointUrl), authorizationKey,
new ConnectionPolicy {
ConnectionMode = ConnectionMode.Direct, ConnectionProtocol = Protocol.Tcp
}))
{
var pkRangesResponse = await client.ReadPartitionKeyRangeFeedAsync(collectionURI, new FeedOptions { RequestContinuation = pkRangesResponseContinuation });
List <PartitionKeyRange> partitionKeyRanges = new List <PartitionKeyRange>();
partitionKeyRanges.AddRange(pkRangesResponse);
pkRangesResponseContinuation = pkRangesResponse.ResponseContinuation;
Dictionary <string, string> checkpoints = new Dictionary <string, string>();
bool comma = exists;
foreach (PartitionKeyRange pkRange in partitionKeyRanges)
{
string continuation = null;
checkpoints.TryGetValue(pkRange.Id, out continuation);
IDocumentQuery <Document> query = client.CreateDocumentChangeFeedQuery(
collectionURI,
new ChangeFeedOptions
{
PartitionKeyRangeId = pkRange.Id,
StartFromBeginning = true,
RequestContinuation = continuation,
MaxItemCount = -1,
// Set reading time: only show change feed results modified since StartTime
StartTime = starttime
});
while (query.HasMoreResults)
{
FeedResponse <dynamic> readChangesResponse = query.ExecuteNextAsync <dynamic>().Result;
foreach (dynamic changedDocument in readChangesResponse)
{
Console.WriteLine("document: {0}", changedDocument);
await appBlob.AppendTextAsync((comma ? "," : "") + changedDocument);
x++;
comma = true;
}
checkpoints[pkRange.Id] = readChangesResponse.ResponseContinuation;
}
}
}
return(x);
}
public static void AppendText(this CloudAppendBlob blob, string content)
{
blob.AppendTextAsync(content).Wait();
}
