public static void Execute()
{
using (KLog collectorLog = new KLog("ClassBlobFileCollector-MethodExecute"))
{
try
{
var token = new BlobContinuationToken();
var blobCollection = BlobClient.BlobContainer.ListBlobsSegmentedAsync(null, token).GetAwaiter().GetResult();
List <string> blobPrefixNames = blobCollection.Results.OfType <CloudBlobDirectory>().Select(b => b.Prefix).ToList();
collectorLog.Info($"Collector => Prefix Count: {blobPrefixNames.Count()}");
// flush static object
BlobFileCollection.StaticFlush();
// for each dir in the blob container
foreach (var blobPrefixName in blobPrefixNames)
{
var prefixblobCollection = BlobClient.BlobContainer.ListBlobsSegmentedAsync(blobPrefixName, token).GetAwaiter().GetResult();
List <string> prefixblobFileNames = prefixblobCollection.Results.OfType <CloudBlockBlob>().Select(b => b.Name).ToList();
collectorLog.Info($"Collector => Parsing Prefix: {blobPrefixName}");
BlobFileParser.Execute(blobPrefixName, prefixblobFileNames);
}
}
catch (Exception ex)
{
collectorLog.Error($"BlobFileCollector Exception: {ex.ToString()}");
}
}
}
public static void Execute()
{
// Blob retention
using (KLog blobRetention = new KLog("BlobFileRetention-BlobRetention"))
{
try
{
IEnumerable <BlobFileModel> retentionFileCollection = BlobFileCollection.CurrentRetentionCount();
blobRetention.Info($"Retention Count: {retentionFileCollection.Count()}");
foreach (BlobFileModel blobFile in retentionFileCollection)
{
// for each file, confim check one more
blobRetention.Info($"Retention => File Name: {blobFile.CloudFile}");
string cloudFile = blobFile.CloudFile;
try
{
BlobClient.DeleteBlobFile(cloudFile);
blobRetention.Info($"File Deleted => File Name: {blobFile.CloudFile}");
}
catch (Exception ex)
{
blobRetention.Error($"Delete Failure: {blobFile.CloudFile} Exception: {ex}");
}
}
}
catch (Exception ex)
{
blobRetention.Error($"BlobFileRetention Exception: {ex.ToString()}");
}
}
// SQL Retention
using (KLog sqlRetention = new KLog("BlobFileRetention-SQLRetention"))
{
try
{
SQLResponseModel checkRetention = DataAccessor.Retention(Configuration.RetentionDays);
if (!checkRetention.Success)
{
sqlRetention.Error($"SQL Expection on [BlobFileRetention].[Retention] - Message: {checkRetention.Message}");
}
}
catch (Exception ex)
{
sqlRetention.Error($"BlobFileRetention Exception: {ex.ToString()}");
}
}
}
public static bool Execute(string line, Guid fileGuid, KLog uploaderLog)
{
uploaderLog.Warning($"Uploader => Footer Check Failed - Guid: {fileGuid.ToString()}");
BlobFileCollection.GetFiles().First(d => d.FileGuid == fileGuid).FooterStatus = false;
LogRecordModel record = null;
try
{
record = JsonConvert.DeserializeObject <LogRecordModel>(line);
}
catch (Exception ex)
{
uploaderLog.Error($" [UploadInstanceStop] Log Record Deserialize Exception: {ex.ToString()}");
return(false);
}
var newEndTime = record.EventTime;
InstanceModel newInstanceCloser = new InstanceModel();
newInstanceCloser.InstanceID = fileGuid;
newInstanceCloser.EventTime = newEndTime;
var checkUpdateInstaceClose = DataAccessor.UpdateInstanceStop(newInstanceCloser);
if (!checkUpdateInstaceClose.Success)
{
uploaderLog.Error($"SQL Expection on [UploadInstanceStop].[UpdateInstaceClose] - Message: {checkUpdateInstaceClose.Message}");
return(false);
}
var checkUpdateBlockStop = DataAccessor.UpdateBlockEmptyStop(record, fileGuid);
if (!checkUpdateBlockStop.Success)
{
uploaderLog.Error($"SQL Expection on [UploadInstanceStop].[UpdateBlockStop] - Message: {checkUpdateBlockStop.Message}");
return(false);
}
return(true);
}
public static void Execute()
{
using (KLog checkLog = new KLog("ClassBlobFileCheck-MethodExecute"))
{
try
{
foreach (var file in BlobFileCollection.GetFiles())
{
if (file.FileGuid != Guid.Empty)
{
var resultResponse = DataAccessor.CheckInstanceId(file.FileGuid);
if (!resultResponse.Success)
{
checkLog.Error($"SQL Expection on [BlobFileCheck].[CheckInstance] - Message: {resultResponse.Message}");
break;
}
if (resultResponse.Id != Guid.Empty)
{
BlobFileCollection.GetFiles().First(d => d.FileGuid == file.FileGuid).Exist = true;
checkLog.Info($"Instance Check => Guid: {file.FileGuid.ToString()} Result: true");
}
else
{
BlobFileCollection.GetFiles().First(d => d.FileGuid == file.FileGuid).Exist = false;
checkLog.Info($"Instance Check => Guid: {file.FileGuid.ToString()} Result: false");
}
}
else
{
BlobFileCollection.GetFiles().First(d => d.FileGuid == file.FileGuid).Exist = true;
checkLog.Info($"Instance Check => Guid: {file.FileGuid.ToString()} Result: true (empty guid)");
}
}
}
catch (Exception ex)
{
checkLog.Error($"BlobfileCheck Expection: {ex.ToString()}");
}
}
}
// this should pass in a single List<BlobFileModel>
public static void Execute(string blobPrefixName, List <string> prefixblobFileNames)
{
using (KLog parserLog = new KLog("ClassBlobFileParser-MethodExecute"))
{
try
{
foreach (var file in prefixblobFileNames)
{
var fileNameWithoutPrefix = file.Replace(blobPrefixName, "");
BlobFileModel blobFile = new BlobFileModel();
blobFile.File = fileNameWithoutPrefix;
blobFile.CloudFile = file;
// Check and parse the full log file name
if (fileNameWithoutPrefix.Count() == 47 && fileNameWithoutPrefix.Contains("KLOG_"))
{
var parseGuid = Guid.Parse(fileNameWithoutPrefix.Substring(7, 36));
var parseTag = fileNameWithoutPrefix.Substring(1, 6);
blobFile.FileGuid = parseGuid;
blobFile.Tag = parseTag;
blobFile.ParseStatus = true;
parserLog.Info($"Parser => File Name: {fileNameWithoutPrefix} Tag: {parseTag} Guid: {parseGuid}");
}
else
{
blobFile.ParseStatus = false;
}
BlobFileCollection.AddFile(blobFile);
}
}
catch (Exception ex)
{
parserLog.Error($"BlobFileParser Exception: {ex.ToString()}");
}
}
}
public static void Execute()
{
using (KLog uploaderLog = new KLog("ClassBlobFileUploader-MethodExecute"))
{
// Load files not currently uploaded
var existFileCollction = BlobFileCollection.CurrentExistFalseCount();
try
{
foreach (var blobFile in existFileCollction)
{
try
{
// Set variables
var cloudFile = blobFile.CloudFile;
var fileGuid = blobFile.FileGuid;
var lineCounter = 1;
List <LogRecordModel> recordModelList = new List <LogRecordModel>();
List <string> lines = new List <string>();
var document = BlobClient.GetDocument(cloudFile);
lines = ReadFile.Execute(document);
var lineCountTotal = lines.Count();
uploaderLog.Info($"Uploader => Starting Upload - Guid: {fileGuid.ToString()} Line Count: {lineCountTotal} ");
//
// Read each line in the log file. Check first and last line for KLog instance data. Break loop on any unexcpted data or failure.
//
foreach (var line in lines)
{
//
// Check first line -- expecting the KLog instane "header"
//
if (lineCounter == 1)
{
if (line.Contains("#KLOG_INSTANCE_STATUS#"))
{
var checkUploadFirstLine = UploadFirstLine.Execute(line, uploaderLog);
if (!checkUploadFirstLine)
{
uploaderLog.Error($"[BlobFileUploader].[checkUploadFirstLine] - GUID: {fileGuid}");
break;
}
}
else
{
UploadFirstLine.MarkFailure(fileGuid);
uploaderLog.Error($"Uploader => Header Check Failed - Guid: {fileGuid.ToString()} Line: {line}");
break;
}
lineCounter++;
}
//
// Check last line -- expecting the KLog instance "footer" and add all logs.
//
else if (lineCounter == lineCountTotal)
{
// Post all logs
var checkUploadAllLogs = UploadAllLogs.Execute(recordModelList, fileGuid, uploaderLog);
if (!checkUploadAllLogs)
{
uploaderLog.Error($"[BlobFileUploader].[UploadAllLogs] - GUID: {fileGuid}");
break;
}
// Check for a proper KLog instance "footer"
if (line.Contains("#KLOG_INSTANCE_STATUS#"))
{
var checkUploadLastLine = UploadLastLine.Execute(line, uploaderLog);
if (!checkUploadLastLine)
{
uploaderLog.Error($"[BlobFileUploader].[UploadLastLine] - GUID: {fileGuid}");
}
}
else
{
var checkUploadInstanceStop = UploadInstanceStop.Execute(line, fileGuid, uploaderLog);
if (!checkUploadInstanceStop)
{
uploaderLog.Error($"[BlobFileUploader].[UploadInstanceStop] - GUID: {fileGuid}");
}
}
uploaderLog.Info($"Log Loaded - Guid: {fileGuid.ToString()}");
}
//
// Check line for "normal" log record, add to collection to bulk add later.
//
else
{
var checkAddLogToCollection = AddLogToCollection.Execute(line, recordModelList, uploaderLog);
if (!checkAddLogToCollection)
{
uploaderLog.Error($"[BlobFileUploader].[AddLogToCollection] - GUID: {fileGuid}");
break;
}
lineCounter++;
}
}
}
catch (Exception ex)
{
uploaderLog.Error($"Blob Upload Failure - Exception: {ex}");
}
}
}
catch (Exception ex)
{
uploaderLog.Error($"BlobFileUploader Exception: {ex}");
}
}
}
/// <summary>
/// Mark the file has a failure.
/// </summary>
/// <param name="fileGuid"></param>
public static void MarkFailure(Guid fileGuid)
{
BlobFileCollection.GetFiles().First(d => d.FileGuid == fileGuid).HeaderStatus = false;
BlobFileCollection.GetFiles().First(d => d.FileGuid == fileGuid).LogStatus = false;
BlobFileCollection.GetFiles().First(d => d.FileGuid == fileGuid).FooterStatus = false;
}