private IList <CsvCounterRecord> ExtractPerfCsvData(FileObject fileObject)
{
List <CsvCounterRecord> csvRecords = new List <CsvCounterRecord>();
try
{
IList <string> allLines = fileObject.Stream.Read();
MatchCollection matchList = Regex.Matches(allLines[0], "\"(.+?)\"");
string[] headers = matchList.Cast <Match>().Select(match => match.Value).ToArray();
for (int csvRecordIndex = 1; csvRecordIndex < allLines.Count; csvRecordIndex++)
{
string[] counterValues = allLines[csvRecordIndex].Split(',');
for (int headerIndex = 1; headerIndex < headers.Length; headerIndex++)
{
if (counterValues.Length > headerIndex)
{
string stringValue = counterValues[headerIndex].Trim('"').Trim(' ');
if (!string.IsNullOrEmpty(stringValue))
{
try
{
csvRecords.Add(new CsvCounterRecord()
{
Timestamp = Convert.ToDateTime(counterValues[0].Trim('"').Trim(' ')),
CounterName = headers[headerIndex],
CounterValue = Decimal.Parse(stringValue, NumberStyles.AllowExponent | NumberStyles.AllowDecimalPoint),
NodeName = fileObject.NodeName,
FileType = fileObject.FileDataType.ToString(),
RelativeUri = fileObject.RelativeUri
});
}
catch (Exception ex)
{
Log.Exception($"stringValue:{stringValue} exception:{ex}");
}
}
}
}
}
return(csvRecords);
}
catch (Exception e)
{
Log.Exception($"{e}", csvRecords);
return(csvRecords);
}
}
public void QueueForIngest(FileObject fileObject)
{
Log.Debug("enter");
if (Config.IsKustoConfigured())
{
TaskManager.QueueTaskAction(() => Kusto.AddFile(fileObject));
}
if (Config.IsLogAnalyticsConfigured())
{
TaskManager.QueueTaskAction(() => LogAnalytics.AddFile(fileObject));
}
}
private void SaveToCache(FileObject fileObject, bool force = false)
{
try
{
if (force || (!Config.UseMemoryStream & !fileObject.Exists))
{
fileObject.Stream.SaveToFile();
}
}
catch (Exception e)
{
Log.Exception($"{e}", fileObject);
}
}
private bool PostData(FileObject fileObject, bool connectivityCheck = false)
{
Log.Debug("enter");
string jsonBody = Config.UseMemoryStream || connectivityCheck ? new StreamReader(fileObject.Stream.Get()).ReadToEnd() : File.ReadAllText(fileObject.FileUri);
fileObject.Stream.Close();
byte[] jsonBytes = Encoding.UTF8.GetBytes(jsonBody);
string date = DateTime.UtcNow.ToString("r");
string signature = BuildSignature(date, jsonBytes);
Log.Info($"signature length: {signature.Length} date: {date} json length: {jsonBytes.Length}\r\n file: {Path.GetFileName(fileObject.FileUri)}", ConsoleColor.Magenta);
try
{
string uri = "https://" + Config.LogAnalyticsId + _logAnalyticsCustomLogSuffix;
Log.Debug($"post uri:{uri}");
Dictionary <string, string> headers = new Dictionary <string, string>();
// send empty log name for connectivity check
// successful connect will respond with badrequest
headers.Add("Log-Type", connectivityCheck ? string.Empty : Config.LogAnalyticsName);
headers.Add("Authorization", signature);
headers.Add("x-ms-date", date);
headers.Add("time-generated-field", _timeStampField);
_httpClient.SendRequest(
uri: uri,
jsonBody: jsonBody,
httpMethod: HttpMethod.Post,
headers: headers,
okStatus: connectivityCheck ? HttpStatusCode.BadRequest : HttpStatusCode.OK);
Log.Info($"{_httpClient.Response?.ReasonPhrase}");
if (_httpClient.Success || (connectivityCheck && _httpClient.StatusCode == HttpStatusCode.BadRequest))
{
return(true);
}
Log.Error("unsuccessful response:", _httpClient.Response);
return(false);
}
catch (Exception e)
{
Log.Exception($"post exception:{e}");
return(false);
}
}
public IRecord Populate(FileObject fileObject, string traceRecord, string resourceUri = null)
{
Match match = Regex.Match(traceRecord, @"/(?<Type>\w+?)(\.\w+?)?\.(?<PID>\d+?)\.dmp");
Timestamp = fileObject.LastModified.UtcDateTime;
PID = Convert.ToInt32(match.Groups["PID"].Value);
Type = match.Groups["Type"].Value;
Text = traceRecord;
NodeName = fileObject.NodeName;
FileType = fileObject.FileDataType.ToString();
RelativeUri = fileObject.RelativeUri;
ResourceUri = resourceUri;
return(this);
}
public IRecord Populate(FileObject fileObject, string traceRecord, string resourceUri = null)
{
string[] fields = ParseRecord(traceRecord);
Timestamp = Convert.ToDateTime(fields[0].Replace("-", " "));
Level = fields[1];
PID = Convert.ToInt32(fields[2]);
Type = fields[3];
Text = fields[4];
NodeName = fileObject.NodeName;
FileType = fileObject.FileDataType.ToString();
RelativeUri = fileObject.RelativeUri;
ResourceUri = resourceUri;
return(this);
}
private FileObjectCollection FormatTraceFile <T>(FileObject fileObject) where T : IRecord, new()
{
Log.Debug($"enter:{fileObject.FileUri}");
IList <IRecord> records = new List <IRecord>();
// handles dtr, setup, and deployer file timestamp formats
string newEventPattern = @"^[0-9]{2,4}(-|/)[0-9]{1,2}(-|/)[0-9]{1,2}(-| )[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}";
Regex regex = new Regex(newEventPattern, RegexOptions.Compiled);
string record = string.Empty;
try
{
foreach (string tempLine in fileObject.Stream.Read())
{
if (regex.IsMatch(tempLine))
{
// new record, write old record
if (record.Length > 0)
{
records.Add(new T().Populate(fileObject, record, Config.ResourceUri));
}
record = string.Empty;
}
record += tempLine;
}
// last record
if (record.Length > 0)
{
records.Add(new T().Populate(fileObject, record, Config.ResourceUri));
}
Log.Debug($"finished format:{fileObject.FileUri}");
TotalFilesFormatted++;
TotalRecords += records.Count;
return(PopulateCollection(fileObject, records.Cast <IRecord>().ToList()));
}
catch (Exception e)
{
Log.Exception($"file:{fileObject.FileUri} exception:{e}");
return(new FileObjectCollection()
{
fileObject
});
}
}
private string SetIngestionMapping(FileObject fileObject)
{
string ingestionJsonString = null;
switch (Config.FileType)
{
case FileTypesEnum.counter:
ingestionJsonString = JsonConvert.SerializeObject(new KustoIngestionMappings(fileObject)
{
ResourceUri = Config.ResourceUri,
SetConstants = Config.KustoUseBlobAsSource,
}.CounterSchema());
break;
case FileTypesEnum.setup:
ingestionJsonString = JsonConvert.SerializeObject(new KustoIngestionMappings(fileObject)
{
ResourceUri = Config.ResourceUri,
SetConstants = Config.KustoUseBlobAsSource,
}.SetupSchema());
break;
case FileTypesEnum.table:
ingestionJsonString = JsonConvert.SerializeObject(new KustoIngestionMappings(fileObject)
{
ResourceUri = Config.ResourceUri
}.TableSchema());
break;
case FileTypesEnum.trace:
ingestionJsonString = JsonConvert.SerializeObject(new KustoIngestionMappings(fileObject)
{
ResourceUri = Config.ResourceUri,
SetConstants = Config.KustoUseBlobAsSource,
}.TraceSchema());
break;
}
Log.Debug($"Ingestion Mapping: {ingestionJsonString}");
return(ingestionJsonString);
}
public bool Connect()
{
if (Config.LogAnalyticsCreate | Config.LogAnalyticsRecreate | Config.Unique)
{
_arm.Authenticate();
GetCurrentWorkspace();
if (Config.LogAnalyticsRecreate)
{
return(RecreateWorkspace());
}
if (Config.LogAnalyticsCreate)
{
return(CreateWorkspace());
}
if (Config.Unique)
{
_ingestedUris.AddRange(PostQueryList($"['{Config.LogAnalyticsName}_CL']|distinct RelativeUri_s"));
Log.Info($"listResults:", _ingestedUris);
}
}
// send empty object to check connection
FileObject fileObject = new FileObject();
fileObject.Stream.Set(Encoding.UTF8.GetBytes("{}"));
Log.Info($"Checking connection to log analytics workspace {Config.LogAnalyticsId} ", ConsoleColor.Green);
if (!PostData(fileObject, true))
{
return(false);
}
if (Config.IsLogAnalyticsPurgeRequested())
{
_arm.Authenticate();
Purge();
return(false);
}
return(true);
}
public void AddFile(FileObject fileObject)
{
Log.Debug("enter");
if (!CanIngest(fileObject.RelativeUri))
{
Log.Warning($"file already ingested. skipping: {fileObject.RelativeUri}");
return;
}
if (Config.KustoUseBlobAsSource)
{
IngestSingleFile(fileObject);
}
else
{
IngestMultipleFiles(FileMgr.ProcessFile(fileObject));
}
}
public void QueueForIngest(FileObject fileObject)
{
Log.Debug("enter");
if (Config.IsKustoConfigured() | Config.IsLogAnalyticsConfigured())
{
if (Config.IsKustoConfigured())
{
_taskManager.QueueTaskAction(() => _kusto.AddFile(fileObject));
}
if (Config.IsLogAnalyticsConfigured())
{
_taskManager.QueueTaskAction(() => _logAnalytics.AddFile(fileObject));
}
}
else
{
_taskManager.QueueTaskAction(() => FileMgr.ProcessFile(fileObject));
}
}
private FileObjectCollection SerializeCsv <T>(FileObject fileObject, IList <T> records)
{
Log.Debug("enter");
FileObjectCollection collection = new FileObjectCollection()
{
fileObject
};
int counter = 0;
string sourceFile = fileObject.FileUri.ToLower().Replace(CsvExtension, "");
fileObject.FileUri = $"{sourceFile}{CsvExtension}";
List <byte> csvSerializedBytes = new List <byte>();
foreach (T record in records)
{
byte[] recordBytes = Encoding.UTF8.GetBytes(record.ToString());
if (csvSerializedBytes.Count + recordBytes.Length > MaxCsvTransmitBytes)
{
fileObject.Stream.Set(csvSerializedBytes.ToArray());
fileObject.Length = fileObject.Stream.Get().Length;
csvSerializedBytes.Clear();
fileObject = new FileObject($"{sourceFile}.{counter}{CsvExtension}", fileObject.BaseUri);
Log.Debug($"csv serialized size: {csvSerializedBytes.Count} file: {fileObject.FileUri}");
collection.Add(fileObject);
}
csvSerializedBytes.AddRange(recordBytes);
counter++;
}
fileObject.Stream.Set(csvSerializedBytes.ToArray());
fileObject.Length = fileObject.Stream.Get().Length;
Log.Debug($"csv serialized size: {csvSerializedBytes.Count} file: {fileObject.FileUri}");
return(collection);
}
private void InvokeCallback(IListBlobItem blob, FileObject fileObject)
{
if (!fileObject.Exists)
{
BlobRequestOptions blobRequestOptions = new BlobRequestOptions()
{
RetryPolicy = new IngestRetryPolicy(),
ParallelOperationThreadCount = Config.Threads
};
if (fileObject.Length > MaxStreamTransmitBytes)
{
fileObject.DownloadAction = () =>
{
if (!Directory.Exists(Path.GetDirectoryName(fileObject.FileUri)))
{
Directory.CreateDirectory(Path.GetDirectoryName(fileObject.FileUri));
}
((CloudBlockBlob)blob).DownloadToFileAsync(fileObject.FileUri, FileMode.Create, null, blobRequestOptions, null).Wait();
};
}
else
{
fileObject.DownloadAction = () =>
{
((CloudBlockBlob)blob).DownloadToStreamAsync(fileObject.Stream.Get(), null, blobRequestOptions, null).Wait();
};
}
IngestCallback?.Invoke(fileObject);
Interlocked.Increment(ref TotalFilesDownloaded);
}
else
{
Log.Warning($"destination file exists. skipping download:\r\n file: {fileObject}");
Interlocked.Increment(ref TotalFilesSkipped);
}
}
public void ImportJson(FileObject fileObject)
{
int retry = 0;
if (fileObject.Stream.Get().Length < 1 && (!fileObject.FileUri.ToLower().EndsWith(JsonExtension) | !fileObject.Exists))
{
Log.Warning($"no json data to send: {fileObject.FileUri}");
return;
}
while (!PostData(fileObject) & retry++ < RetryCount)
{
Log.Error($"error importing: {fileObject.FileUri} retry:{retry}");
if (retry == 1 && Config.LogDebug)
{
File.WriteAllBytes(fileObject.FileUri, fileObject.Stream.Get().ToArray());
Log.Error($"json saved to {fileObject.FileUri}");
}
TotalErrors++;
}
}
public MemoryStream Compress(FileObject fileObject = null)
{
fileObject = fileObject ?? _fileObject;
if (_fileObject.FileExtensionType.Equals(FileExtensionTypesEnum.zip))
{
string error = $"{_fileObject.FileUri} is already zip file";
Log.Error(error);
throw new ArgumentException();
}
Open();
_memoryStream.Position = 0;
Log.Debug($"compressing memoryStream start. start size: {fileObject.Length} dest size: {_memoryStream.Length} position: {_memoryStream.Position}");
fileObject.Length = _memoryStream.Length;
MemoryStream compressedStream = new MemoryStream();
using (ZipArchive archive = new ZipArchive(compressedStream, ZipArchiveMode.Create, true))
{
ZipArchiveEntry entry = archive.CreateEntry(Path.GetFileName(_fileObject.FileUri));
using (Stream archiveStream = entry.Open())
{
_memoryStream.CopyTo(archiveStream);
}
}
compressedStream.Position = 0;
_fileObject.FileUri += ZipExtension;
_fileObject.Stream.Set(compressedStream);
Log.Debug($"compressing memoryStream complete. size: {compressedStream.Length} position: {compressedStream.Position}");
return(compressedStream);
}
public CsvExceptionRecord(string traceRecord, FileObject fileObject, string resourceUri = null)
{
Populate(fileObject, traceRecord, resourceUri);
}
private FileObjectCollection FormatSetupFile(FileObject fileObject)
{
return(FormatTraceFile <CsvSetupRecord>(fileObject));
}
private FileObjectCollection FormatDtrFile(FileObject fileObject)
{
return(FormatTraceFile <DtrTraceRecord>(fileObject));
}
public StreamManager(FileObject fileObject = null)
{
_fileObject = fileObject;
}
public void PostMessageToQueue(string queueUriWithSas, KustoIngestionMessage message, FileObject fileObject)
{
Log.Info($"post: {queueUriWithSas}", ConsoleColor.Magenta);
_totalBlobIngestQueued++;
CloudQueue queue = new CloudQueue(new Uri(queueUriWithSas));
CloudQueueMessage queueMessage = new CloudQueueMessage(JsonConvert.SerializeObject(message));
OperationContext context = new OperationContext()
{
ClientRequestID = message.Id,
};
queue.AddMessage(queueMessage, _messageTimeToLive, null, null, context);
if (Config.KustoUseIngestMessage)
{
_messageList.Add(message.Id);
}
else
{
_messageList.Add(fileObject.RelativeUri);
}
Log.Info($"queue message id: {message.Id}");
}
public KustoIngestionMappings(FileObject fileObject)
{
_fileObject = fileObject;
}
private void QueueBlobSegmentDownload(BlobResultSegment blobResultSegment)
{
int parentId = Thread.CurrentThread.ManagedThreadId;
Log.Debug($"enter. current id:{parentId}. results count: {blobResultSegment.Results.Count()}");
foreach (var blob in blobResultSegment.Results)
{
ICloudBlob blobRef = null;
Log.Debug($"parent id:{parentId} current Id:{Thread.CurrentThread.ManagedThreadId}");
if (blob is CloudBlobDirectory)
{
if (!string.IsNullOrEmpty(Config.NodeFilter) && !Regex.IsMatch(blob.Uri.ToString(), Config.NodeFilter, RegexOptions.IgnoreCase))
{
Log.Warning($"blob:{blob.Uri} does not match nodeFilter pattern:{Config.NodeFilter}, skipping...", ConsoleColor.Yellow);
continue;
}
DownloadBlobsFromDirectory(blob as CloudBlobDirectory);
Log.Debug("blob is directory.");
continue;
}
Interlocked.Increment(ref TotalFilesEnumerated);
if (!string.IsNullOrEmpty(Config.UriFilter) && !Regex.IsMatch(blob.Uri.ToString(), Config.UriFilter, RegexOptions.IgnoreCase))
{
Interlocked.Increment(ref TotalFilesSkipped);
Log.Warning($"blob:{blob.Uri} does not match uriFilter pattern:{Config.UriFilter}, skipping...", ConsoleColor.Yellow);
continue;
}
if (Regex.IsMatch(blob.Uri.ToString(), FileFilterPattern, RegexOptions.IgnoreCase))
{
long ticks = Convert.ToInt64(Regex.Match(blob.Uri.ToString(), FileFilterPattern, RegexOptions.IgnoreCase).Groups[1].Value);
if (ticks <Config.StartTimeUtc.Ticks | ticks> Config.EndTimeUtc.Ticks)
{
Interlocked.Increment(ref TotalFilesSkipped);
Log.Debug($"exclude:bloburi ticks outside of time range:{blob.Uri}");
continue;
}
}
try
{
Log.Debug($"file Blob: {blob.Uri}");
blobRef = blob.Container.ServiceClient.GetBlobReferenceFromServerAsync(blob.Uri).Result;
}
catch (StorageException se)
{
Interlocked.Increment(ref TotalErrors);
Log.Exception($"getting ref for {blob.Uri}, skipping. {se.Message}");
continue;
}
if (blobRef.Properties.LastModified.HasValue)
{
DateTimeOffset lastModified = blobRef.Properties.LastModified.Value;
if (Config.FileType != FileTypesEnum.any && !FileTypes.MapFileTypeUri(blob.Uri.AbsolutePath).Equals(Config.FileType))
{
Interlocked.Increment(ref TotalFilesSkipped);
Log.Debug($"skipping uri with incorrect file type: {FileTypes.MapFileTypeUri(blob.Uri.AbsolutePath)}");
continue;
}
if (lastModified >= Config.StartTimeUtc && lastModified <= Config.EndTimeUtc)
{
Interlocked.Increment(ref TotalFilesMatched);
if (Config.List)
{
Log.Info($"listing file with timestamp: {lastModified}\r\n file: {blob.Uri.AbsolutePath}");
continue;
}
if (ReturnSourceFileLink)
{
IngestCallback?.Invoke(new FileObject(blob.Uri.AbsolutePath, Config.SasEndpointInfo.BlobEndpoint)
{
Length = blobRef.Properties.Length
});
continue;
}
FileObject fileObject = new FileObject(blob.Uri.AbsolutePath, Config.CacheLocation);
Log.Info($"queueing blob with timestamp: {lastModified}\r\n file: {blob.Uri.AbsolutePath}");
if (!fileObject.Exists)
{
fileObject.DownloadAction = () =>
{
((CloudBlockBlob)blob).DownloadToStreamAsync(fileObject.Stream.Get(), null,
new BlobRequestOptions()
{
RetryPolicy = new IngestRetryPolicy(),
ParallelOperationThreadCount = Config.Threads
}, null).Wait();
};
IngestCallback?.Invoke(fileObject);
Interlocked.Increment(ref TotalFilesDownloaded);
}
else
{
Log.Warning($"destination file exists. skipping download:\r\n file: {fileObject}");
IngestCallback?.Invoke(fileObject);
}
}
}
else
{
Log.Error("unable to read blob modified date", blobRef);
TotalErrors++;
}
}
}
public DtrTraceRecord(string traceRecord, FileObject fileObject, string resourceUri = null)
{
Populate(fileObject, traceRecord, resourceUri);
}
private void UploadCacheData(List <string> uris)
{
Log.Info("enter");
List <string> files = new List <string>();
if (uris.Count > 0)
{
foreach (string file in uris)
{
if (File.Exists(file))
{
Log.Info($"adding file to list: {file}");
files.Add(file);
}
else
{
Log.Warning($"file does not exist: {file}");
}
}
}
else
{
switch (Config.FileType)
{
case FileTypesEnum.counter:
files = Directory.GetFiles(Config.CacheLocation, $"*{PerfCtrExtension}", SearchOption.AllDirectories).ToList();
if (files.Count < 1)
{
files = Directory.GetFiles(Config.CacheLocation, $"*{PerfCsvExtension}", SearchOption.AllDirectories).ToList();
}
break;
case FileTypesEnum.setup:
files = Directory.GetFiles(Config.CacheLocation, $"*{SetupExtension}", SearchOption.AllDirectories).ToList();
break;
case FileTypesEnum.table:
files = Directory.GetFiles(Config.CacheLocation, $"*{TableExtension}", SearchOption.AllDirectories).ToList();
break;
case FileTypesEnum.trace:
files = Directory.GetFiles(Config.CacheLocation, $"*{TraceFileExtension}{ZipExtension}", SearchOption.AllDirectories).ToList();
if (files.Count < 1)
{
files = Directory.GetFiles(Config.CacheLocation, $"*{TraceFileExtension}", SearchOption.AllDirectories).ToList();
}
break;
default:
Log.Warning($"invalid filetype for cache upload. returning {Config.FileType}");
return;
}
}
if (files.Count < 1)
{
Log.Error($"configuration set to upload cache files from 'cachelocation' {Config.CacheLocation} but no files found");
}
foreach (string file in files)
{
FileObject fileObject = new FileObject(file, Config.CacheLocation);
Log.Info($"adding file: {fileObject.FileUri}", ConsoleColor.Green);
if (!Config.List)
{
QueueForIngest(fileObject);
}
}
}
public FileObjectCollection ProcessFile(FileObject fileObject)
{
Log.Debug($"enter:{fileObject.FileUri}");
if (fileObject.DownloadAction != null)
{
Log.Info($"downloading:{fileObject.FileUri}", ConsoleColor.Cyan, ConsoleColor.DarkBlue);
_fileTasks.TaskAction(fileObject.DownloadAction).Wait();
Log.Info($"downloaded:{fileObject.FileUri}", ConsoleColor.DarkCyan, ConsoleColor.DarkBlue);
}
if (fileObject.DownloadAction != null && fileObject.Stream.Get().Length < 1 && !fileObject.Exists)
{
string error = $"memoryStream does not exist and file does not exist {fileObject.FileUri}";
Log.Error(error);
throw new ArgumentException(error);
}
if (!fileObject.FileType.Equals(FileTypesEnum.any))
{
if (fileObject.Stream.Get().Length < 1 & fileObject.Exists)
{
// for cached directory uploads
fileObject.Stream.ReadFromFile();
}
if (fileObject.FileExtensionType.Equals(FileExtensionTypesEnum.zip))
{
fileObject.Stream.Decompress();
SaveToCache(fileObject);
}
}
switch (fileObject.FileDataType)
{
case FileDataTypesEnum.unknown:
{
if (fileObject.FileType.Equals(FileTypesEnum.any))
{
SaveToCache(fileObject);
return(new FileObjectCollection());
}
Log.Warning("unknown file", fileObject);
break;
}
case FileDataTypesEnum.counter:
{
if (fileObject.FileExtensionType.Equals(FileExtensionTypesEnum.blg))
{
return(FormatCounterFile(fileObject));
}
break;
}
case FileDataTypesEnum.data:
case FileDataTypesEnum.fabriccrashdumps:
{
if (fileObject.FileExtensionType.Equals(FileExtensionTypesEnum.dmp))
{
return(FormatExceptionFile(fileObject));
}
SaveToCache(fileObject);
return(new FileObjectCollection());
}
case FileDataTypesEnum.fabric:
case FileDataTypesEnum.lease:
{
if (fileObject.FileExtensionType.Equals(FileExtensionTypesEnum.dtr))
{
return(FormatDtrFile(fileObject));
}
break;
}
case FileDataTypesEnum.bootstrap:
case FileDataTypesEnum.fabricsetup:
case FileDataTypesEnum.fabricdeployer:
{
if (fileObject.FileExtensionType.Equals(FileExtensionTypesEnum.trace))
{
return(FormatSetupFile(fileObject));
}
break;
}
case FileDataTypesEnum.table:
{
if (fileObject.FileExtensionType.Equals(FileExtensionTypesEnum.csv))
{
return(FormatTableFile(fileObject));
}
break;
}
default:
{
Log.Warning("unsupported file data type", fileObject.FileDataType);
break;
}
}
Log.Warning($"returning: empty fileObjectCollection for file: {fileObject.FileUri}");
return(new FileObjectCollection());
}
public IRecord Populate(FileObject fileObject, string dtrRecord, string resourceUri = null)
{
return(this as ITraceRecord);
}
private void QueueBlobSegmentDownload(BlobResultSegment blobResultSegment)
{
int parentId = Thread.CurrentThread.ManagedThreadId;
Log.Debug($"enter. current id:{parentId}. results count: {blobResultSegment.Results.Count()}");
long segmentMinDateTicks = Interlocked.Read(ref DiscoveredMinDateTicks);
long segmentMaxDateTicks = Interlocked.Read(ref DiscoveredMaxDateTicks);
foreach (IListBlobItem blob in blobResultSegment.Results)
{
ICloudBlob blobRef = null;
Log.Debug($"parent id:{parentId} current Id:{Thread.CurrentThread.ManagedThreadId}");
if (blob is CloudBlobDirectory)
{
if (!string.IsNullOrEmpty(Config.NodeFilter) && !Regex.IsMatch(blob.Uri.ToString(), Config.NodeFilter, RegexOptions.IgnoreCase))
{
Log.Debug($"blob:{blob.Uri} does not match nodeFilter pattern:{Config.NodeFilter}, skipping...");
continue;
}
DownloadBlobsFromDirectory(blob as CloudBlobDirectory);
Log.Debug("blob is directory.");
continue;
}
Interlocked.Increment(ref TotalFilesEnumerated);
if (Regex.IsMatch(blob.Uri.ToString(), FileFilterPattern, RegexOptions.IgnoreCase))
{
long ticks = Convert.ToInt64(Regex.Match(blob.Uri.ToString(), FileFilterPattern, RegexOptions.IgnoreCase).Groups[1].Value);
if (ticks <Config.StartTimeUtc.Ticks | ticks> Config.EndTimeUtc.Ticks)
{
Interlocked.Increment(ref TotalFilesSkipped);
Log.Debug($"exclude:bloburi file ticks {new DateTime(ticks).ToString("o")} outside of time range:{blob.Uri}");
SetMinMaxDate(ref segmentMinDateTicks, ref segmentMaxDateTicks, ticks);
continue;
}
}
else
{
Log.Debug($"regex not matched: {blob.Uri.ToString()} pattern: {FileFilterPattern}");
}
try
{
Log.Debug($"file Blob: {blob.Uri}");
blobRef = blob.Container.ServiceClient.GetBlobReferenceFromServerAsync(blob.Uri).Result;
}
catch (StorageException se)
{
Interlocked.Increment(ref TotalErrors);
Log.Exception($"getting ref for {blob.Uri}, skipping. {se.Message}");
continue;
}
if (blobRef.Properties.LastModified.HasValue)
{
DateTimeOffset lastModified = blobRef.Properties.LastModified.Value;
SetMinMaxDate(ref segmentMinDateTicks, ref segmentMaxDateTicks, lastModified.Ticks);
if (!string.IsNullOrEmpty(Config.UriFilter) && !Regex.IsMatch(blob.Uri.ToString(), Config.UriFilter, RegexOptions.IgnoreCase))
{
Interlocked.Increment(ref TotalFilesSkipped);
Log.Debug($"blob:{blob.Uri} does not match uriFilter pattern:{Config.UriFilter}, skipping...");
continue;
}
if (Config.FileType != FileTypesEnum.any &&
!FileTypes.MapFileTypeUri(blob.Uri.AbsolutePath).Equals(Config.FileType))
{
Interlocked.Increment(ref TotalFilesSkipped);
Log.Debug($"skipping uri with incorrect file type: {FileTypes.MapFileTypeUri(blob.Uri.AbsolutePath)}");
continue;
}
if (lastModified >= Config.StartTimeUtc && lastModified <= Config.EndTimeUtc)
{
Interlocked.Increment(ref TotalFilesMatched);
if (Config.List)
{
Log.Info($"listing file with timestamp: {lastModified}\r\n file: {blob.Uri.AbsolutePath}");
continue;
}
if (ReturnSourceFileLink)
{
IngestCallback?.Invoke(new FileObject(blob.Uri.AbsolutePath, Config.SasEndpointInfo.BlobEndpoint)
{
Length = blobRef.Properties.Length,
LastModified = lastModified
});
continue;
}
FileObject fileObject = new FileObject(blob.Uri.AbsolutePath, Config.CacheLocation)
{
Length = blobRef.Properties.Length,
LastModified = lastModified
};
Log.Info($"queueing blob with timestamp: {lastModified}\r\n file: {blob.Uri.AbsolutePath}");
InvokeCallback(blob, fileObject);
}
else
{
Interlocked.Increment(ref TotalFilesSkipped);
Log.Debug($"exclude:bloburi {lastModified.ToString("o")} outside of time range:{blob.Uri}");
SetMinMaxDate(ref segmentMinDateTicks, ref segmentMaxDateTicks, lastModified.Ticks);
continue;
}
}
else
{
Log.Error("unable to read blob modified date", blobRef);
TotalErrors++;
}
}
}
