private IList <CsvCounterRecord> ExtractPerfCsvData(FileObject fileObject) { List <CsvCounterRecord> csvRecords = new List <CsvCounterRecord>(); try { IList <string> allLines = fileObject.Stream.Read(); MatchCollection matchList = Regex.Matches(allLines[0], "\"(.+?)\""); string[] headers = matchList.Cast <Match>().Select(match => match.Value).ToArray(); for (int csvRecordIndex = 1; csvRecordIndex < allLines.Count; csvRecordIndex++) { string[] counterValues = allLines[csvRecordIndex].Split(','); for (int headerIndex = 1; headerIndex < headers.Length; headerIndex++) { if (counterValues.Length > headerIndex) { string stringValue = counterValues[headerIndex].Trim('"').Trim(' '); if (!string.IsNullOrEmpty(stringValue)) { try { csvRecords.Add(new CsvCounterRecord() { Timestamp = Convert.ToDateTime(counterValues[0].Trim('"').Trim(' ')), CounterName = headers[headerIndex], CounterValue = Decimal.Parse(stringValue, NumberStyles.AllowExponent | NumberStyles.AllowDecimalPoint), NodeName = fileObject.NodeName, FileType = fileObject.FileDataType.ToString(), RelativeUri = fileObject.RelativeUri }); } catch (Exception ex) { Log.Exception($"stringValue:{stringValue} exception:{ex}"); } } } } } return(csvRecords); } catch (Exception e) { Log.Exception($"{e}", csvRecords); return(csvRecords); } }
public void QueueForIngest(FileObject fileObject) { Log.Debug("enter"); if (Config.IsKustoConfigured()) { TaskManager.QueueTaskAction(() => Kusto.AddFile(fileObject)); } if (Config.IsLogAnalyticsConfigured()) { TaskManager.QueueTaskAction(() => LogAnalytics.AddFile(fileObject)); } }
private void SaveToCache(FileObject fileObject, bool force = false) { try { if (force || (!Config.UseMemoryStream & !fileObject.Exists)) { fileObject.Stream.SaveToFile(); } } catch (Exception e) { Log.Exception($"{e}", fileObject); } }
private bool PostData(FileObject fileObject, bool connectivityCheck = false) { Log.Debug("enter"); string jsonBody = Config.UseMemoryStream || connectivityCheck ? new StreamReader(fileObject.Stream.Get()).ReadToEnd() : File.ReadAllText(fileObject.FileUri); fileObject.Stream.Close(); byte[] jsonBytes = Encoding.UTF8.GetBytes(jsonBody); string date = DateTime.UtcNow.ToString("r"); string signature = BuildSignature(date, jsonBytes); Log.Info($"signature length: {signature.Length} date: {date} json length: {jsonBytes.Length}\r\n file: {Path.GetFileName(fileObject.FileUri)}", ConsoleColor.Magenta); try { string uri = "https://" + Config.LogAnalyticsId + _logAnalyticsCustomLogSuffix; Log.Debug($"post uri:{uri}"); Dictionary <string, string> headers = new Dictionary <string, string>(); // send empty log name for connectivity check // successful connect will respond with badrequest headers.Add("Log-Type", connectivityCheck ? string.Empty : Config.LogAnalyticsName); headers.Add("Authorization", signature); headers.Add("x-ms-date", date); headers.Add("time-generated-field", _timeStampField); _httpClient.SendRequest( uri: uri, jsonBody: jsonBody, httpMethod: HttpMethod.Post, headers: headers, okStatus: connectivityCheck ? HttpStatusCode.BadRequest : HttpStatusCode.OK); Log.Info($"{_httpClient.Response?.ReasonPhrase}"); if (_httpClient.Success || (connectivityCheck && _httpClient.StatusCode == HttpStatusCode.BadRequest)) { return(true); } Log.Error("unsuccessful response:", _httpClient.Response); return(false); } catch (Exception e) { Log.Exception($"post exception:{e}"); return(false); } }
public IRecord Populate(FileObject fileObject, string traceRecord, string resourceUri = null) { Match match = Regex.Match(traceRecord, @"/(?<Type>\w+?)(\.\w+?)?\.(?<PID>\d+?)\.dmp"); Timestamp = fileObject.LastModified.UtcDateTime; PID = Convert.ToInt32(match.Groups["PID"].Value); Type = match.Groups["Type"].Value; Text = traceRecord; NodeName = fileObject.NodeName; FileType = fileObject.FileDataType.ToString(); RelativeUri = fileObject.RelativeUri; ResourceUri = resourceUri; return(this); }
public IRecord Populate(FileObject fileObject, string traceRecord, string resourceUri = null) { string[] fields = ParseRecord(traceRecord); Timestamp = Convert.ToDateTime(fields[0].Replace("-", " ")); Level = fields[1]; PID = Convert.ToInt32(fields[2]); Type = fields[3]; Text = fields[4]; NodeName = fileObject.NodeName; FileType = fileObject.FileDataType.ToString(); RelativeUri = fileObject.RelativeUri; ResourceUri = resourceUri; return(this); }
private FileObjectCollection FormatTraceFile <T>(FileObject fileObject) where T : IRecord, new() { Log.Debug($"enter:{fileObject.FileUri}"); IList <IRecord> records = new List <IRecord>(); // handles dtr, setup, and deployer file timestamp formats string newEventPattern = @"^[0-9]{2,4}(-|/)[0-9]{1,2}(-|/)[0-9]{1,2}(-| )[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}"; Regex regex = new Regex(newEventPattern, RegexOptions.Compiled); string record = string.Empty; try { foreach (string tempLine in fileObject.Stream.Read()) { if (regex.IsMatch(tempLine)) { // new record, write old record if (record.Length > 0) { records.Add(new T().Populate(fileObject, record, Config.ResourceUri)); } record = string.Empty; } record += tempLine; } // last record if (record.Length > 0) { records.Add(new T().Populate(fileObject, record, Config.ResourceUri)); } Log.Debug($"finished format:{fileObject.FileUri}"); TotalFilesFormatted++; TotalRecords += records.Count; return(PopulateCollection(fileObject, records.Cast <IRecord>().ToList())); } catch (Exception e) { Log.Exception($"file:{fileObject.FileUri} exception:{e}"); return(new FileObjectCollection() { fileObject }); } }
private string SetIngestionMapping(FileObject fileObject) { string ingestionJsonString = null; switch (Config.FileType) { case FileTypesEnum.counter: ingestionJsonString = JsonConvert.SerializeObject(new KustoIngestionMappings(fileObject) { ResourceUri = Config.ResourceUri, SetConstants = Config.KustoUseBlobAsSource, }.CounterSchema()); break; case FileTypesEnum.setup: ingestionJsonString = JsonConvert.SerializeObject(new KustoIngestionMappings(fileObject) { ResourceUri = Config.ResourceUri, SetConstants = Config.KustoUseBlobAsSource, }.SetupSchema()); break; case FileTypesEnum.table: ingestionJsonString = JsonConvert.SerializeObject(new KustoIngestionMappings(fileObject) { ResourceUri = Config.ResourceUri }.TableSchema()); break; case FileTypesEnum.trace: ingestionJsonString = JsonConvert.SerializeObject(new KustoIngestionMappings(fileObject) { ResourceUri = Config.ResourceUri, SetConstants = Config.KustoUseBlobAsSource, }.TraceSchema()); break; } Log.Debug($"Ingestion Mapping: {ingestionJsonString}"); return(ingestionJsonString); }
public bool Connect() { if (Config.LogAnalyticsCreate | Config.LogAnalyticsRecreate | Config.Unique) { _arm.Authenticate(); GetCurrentWorkspace(); if (Config.LogAnalyticsRecreate) { return(RecreateWorkspace()); } if (Config.LogAnalyticsCreate) { return(CreateWorkspace()); } if (Config.Unique) { _ingestedUris.AddRange(PostQueryList($"['{Config.LogAnalyticsName}_CL']|distinct RelativeUri_s")); Log.Info($"listResults:", _ingestedUris); } } // send empty object to check connection FileObject fileObject = new FileObject(); fileObject.Stream.Set(Encoding.UTF8.GetBytes("{}")); Log.Info($"Checking connection to log analytics workspace {Config.LogAnalyticsId} ", ConsoleColor.Green); if (!PostData(fileObject, true)) { return(false); } if (Config.IsLogAnalyticsPurgeRequested()) { _arm.Authenticate(); Purge(); return(false); } return(true); }
public void AddFile(FileObject fileObject) { Log.Debug("enter"); if (!CanIngest(fileObject.RelativeUri)) { Log.Warning($"file already ingested. skipping: {fileObject.RelativeUri}"); return; } if (Config.KustoUseBlobAsSource) { IngestSingleFile(fileObject); } else { IngestMultipleFiles(FileMgr.ProcessFile(fileObject)); } }
public void QueueForIngest(FileObject fileObject) { Log.Debug("enter"); if (Config.IsKustoConfigured() | Config.IsLogAnalyticsConfigured()) { if (Config.IsKustoConfigured()) { _taskManager.QueueTaskAction(() => _kusto.AddFile(fileObject)); } if (Config.IsLogAnalyticsConfigured()) { _taskManager.QueueTaskAction(() => _logAnalytics.AddFile(fileObject)); } } else { _taskManager.QueueTaskAction(() => FileMgr.ProcessFile(fileObject)); } }
private FileObjectCollection SerializeCsv <T>(FileObject fileObject, IList <T> records) { Log.Debug("enter"); FileObjectCollection collection = new FileObjectCollection() { fileObject }; int counter = 0; string sourceFile = fileObject.FileUri.ToLower().Replace(CsvExtension, ""); fileObject.FileUri = $"{sourceFile}{CsvExtension}"; List <byte> csvSerializedBytes = new List <byte>(); foreach (T record in records) { byte[] recordBytes = Encoding.UTF8.GetBytes(record.ToString()); if (csvSerializedBytes.Count + recordBytes.Length > MaxCsvTransmitBytes) { fileObject.Stream.Set(csvSerializedBytes.ToArray()); fileObject.Length = fileObject.Stream.Get().Length; csvSerializedBytes.Clear(); fileObject = new FileObject($"{sourceFile}.{counter}{CsvExtension}", fileObject.BaseUri); Log.Debug($"csv serialized size: {csvSerializedBytes.Count} file: {fileObject.FileUri}"); collection.Add(fileObject); } csvSerializedBytes.AddRange(recordBytes); counter++; } fileObject.Stream.Set(csvSerializedBytes.ToArray()); fileObject.Length = fileObject.Stream.Get().Length; Log.Debug($"csv serialized size: {csvSerializedBytes.Count} file: {fileObject.FileUri}"); return(collection); }
private void InvokeCallback(IListBlobItem blob, FileObject fileObject) { if (!fileObject.Exists) { BlobRequestOptions blobRequestOptions = new BlobRequestOptions() { RetryPolicy = new IngestRetryPolicy(), ParallelOperationThreadCount = Config.Threads }; if (fileObject.Length > MaxStreamTransmitBytes) { fileObject.DownloadAction = () => { if (!Directory.Exists(Path.GetDirectoryName(fileObject.FileUri))) { Directory.CreateDirectory(Path.GetDirectoryName(fileObject.FileUri)); } ((CloudBlockBlob)blob).DownloadToFileAsync(fileObject.FileUri, FileMode.Create, null, blobRequestOptions, null).Wait(); }; } else { fileObject.DownloadAction = () => { ((CloudBlockBlob)blob).DownloadToStreamAsync(fileObject.Stream.Get(), null, blobRequestOptions, null).Wait(); }; } IngestCallback?.Invoke(fileObject); Interlocked.Increment(ref TotalFilesDownloaded); } else { Log.Warning($"destination file exists. skipping download:\r\n file: {fileObject}"); Interlocked.Increment(ref TotalFilesSkipped); } }
public void ImportJson(FileObject fileObject) { int retry = 0; if (fileObject.Stream.Get().Length < 1 && (!fileObject.FileUri.ToLower().EndsWith(JsonExtension) | !fileObject.Exists)) { Log.Warning($"no json data to send: {fileObject.FileUri}"); return; } while (!PostData(fileObject) & retry++ < RetryCount) { Log.Error($"error importing: {fileObject.FileUri} retry:{retry}"); if (retry == 1 && Config.LogDebug) { File.WriteAllBytes(fileObject.FileUri, fileObject.Stream.Get().ToArray()); Log.Error($"json saved to {fileObject.FileUri}"); } TotalErrors++; } }
public MemoryStream Compress(FileObject fileObject = null) { fileObject = fileObject ?? _fileObject; if (_fileObject.FileExtensionType.Equals(FileExtensionTypesEnum.zip)) { string error = $"{_fileObject.FileUri} is already zip file"; Log.Error(error); throw new ArgumentException(); } Open(); _memoryStream.Position = 0; Log.Debug($"compressing memoryStream start. start size: {fileObject.Length} dest size: {_memoryStream.Length} position: {_memoryStream.Position}"); fileObject.Length = _memoryStream.Length; MemoryStream compressedStream = new MemoryStream(); using (ZipArchive archive = new ZipArchive(compressedStream, ZipArchiveMode.Create, true)) { ZipArchiveEntry entry = archive.CreateEntry(Path.GetFileName(_fileObject.FileUri)); using (Stream archiveStream = entry.Open()) { _memoryStream.CopyTo(archiveStream); } } compressedStream.Position = 0; _fileObject.FileUri += ZipExtension; _fileObject.Stream.Set(compressedStream); Log.Debug($"compressing memoryStream complete. size: {compressedStream.Length} position: {compressedStream.Position}"); return(compressedStream); }
public CsvExceptionRecord(string traceRecord, FileObject fileObject, string resourceUri = null) { Populate(fileObject, traceRecord, resourceUri); }
private FileObjectCollection FormatSetupFile(FileObject fileObject) { return(FormatTraceFile <CsvSetupRecord>(fileObject)); }
private FileObjectCollection FormatDtrFile(FileObject fileObject) { return(FormatTraceFile <DtrTraceRecord>(fileObject)); }
public StreamManager(FileObject fileObject = null) { _fileObject = fileObject; }
public void PostMessageToQueue(string queueUriWithSas, KustoIngestionMessage message, FileObject fileObject) { Log.Info($"post: {queueUriWithSas}", ConsoleColor.Magenta); _totalBlobIngestQueued++; CloudQueue queue = new CloudQueue(new Uri(queueUriWithSas)); CloudQueueMessage queueMessage = new CloudQueueMessage(JsonConvert.SerializeObject(message)); OperationContext context = new OperationContext() { ClientRequestID = message.Id, }; queue.AddMessage(queueMessage, _messageTimeToLive, null, null, context); if (Config.KustoUseIngestMessage) { _messageList.Add(message.Id); } else { _messageList.Add(fileObject.RelativeUri); } Log.Info($"queue message id: {message.Id}"); }
public KustoIngestionMappings(FileObject fileObject) { _fileObject = fileObject; }
private void QueueBlobSegmentDownload(BlobResultSegment blobResultSegment) { int parentId = Thread.CurrentThread.ManagedThreadId; Log.Debug($"enter. current id:{parentId}. results count: {blobResultSegment.Results.Count()}"); foreach (var blob in blobResultSegment.Results) { ICloudBlob blobRef = null; Log.Debug($"parent id:{parentId} current Id:{Thread.CurrentThread.ManagedThreadId}"); if (blob is CloudBlobDirectory) { if (!string.IsNullOrEmpty(Config.NodeFilter) && !Regex.IsMatch(blob.Uri.ToString(), Config.NodeFilter, RegexOptions.IgnoreCase)) { Log.Warning($"blob:{blob.Uri} does not match nodeFilter pattern:{Config.NodeFilter}, skipping...", ConsoleColor.Yellow); continue; } DownloadBlobsFromDirectory(blob as CloudBlobDirectory); Log.Debug("blob is directory."); continue; } Interlocked.Increment(ref TotalFilesEnumerated); if (!string.IsNullOrEmpty(Config.UriFilter) && !Regex.IsMatch(blob.Uri.ToString(), Config.UriFilter, RegexOptions.IgnoreCase)) { Interlocked.Increment(ref TotalFilesSkipped); Log.Warning($"blob:{blob.Uri} does not match uriFilter pattern:{Config.UriFilter}, skipping...", ConsoleColor.Yellow); continue; } if (Regex.IsMatch(blob.Uri.ToString(), FileFilterPattern, RegexOptions.IgnoreCase)) { long ticks = Convert.ToInt64(Regex.Match(blob.Uri.ToString(), FileFilterPattern, RegexOptions.IgnoreCase).Groups[1].Value); if (ticks <Config.StartTimeUtc.Ticks | ticks> Config.EndTimeUtc.Ticks) { Interlocked.Increment(ref TotalFilesSkipped); Log.Debug($"exclude:bloburi ticks outside of time range:{blob.Uri}"); continue; } } try { Log.Debug($"file Blob: {blob.Uri}"); blobRef = blob.Container.ServiceClient.GetBlobReferenceFromServerAsync(blob.Uri).Result; } catch (StorageException se) { Interlocked.Increment(ref TotalErrors); Log.Exception($"getting ref for {blob.Uri}, skipping. {se.Message}"); continue; } if (blobRef.Properties.LastModified.HasValue) { DateTimeOffset lastModified = blobRef.Properties.LastModified.Value; if (Config.FileType != FileTypesEnum.any && !FileTypes.MapFileTypeUri(blob.Uri.AbsolutePath).Equals(Config.FileType)) { Interlocked.Increment(ref TotalFilesSkipped); Log.Debug($"skipping uri with incorrect file type: {FileTypes.MapFileTypeUri(blob.Uri.AbsolutePath)}"); continue; } if (lastModified >= Config.StartTimeUtc && lastModified <= Config.EndTimeUtc) { Interlocked.Increment(ref TotalFilesMatched); if (Config.List) { Log.Info($"listing file with timestamp: {lastModified}\r\n file: {blob.Uri.AbsolutePath}"); continue; } if (ReturnSourceFileLink) { IngestCallback?.Invoke(new FileObject(blob.Uri.AbsolutePath, Config.SasEndpointInfo.BlobEndpoint) { Length = blobRef.Properties.Length }); continue; } FileObject fileObject = new FileObject(blob.Uri.AbsolutePath, Config.CacheLocation); Log.Info($"queueing blob with timestamp: {lastModified}\r\n file: {blob.Uri.AbsolutePath}"); if (!fileObject.Exists) { fileObject.DownloadAction = () => { ((CloudBlockBlob)blob).DownloadToStreamAsync(fileObject.Stream.Get(), null, new BlobRequestOptions() { RetryPolicy = new IngestRetryPolicy(), ParallelOperationThreadCount = Config.Threads }, null).Wait(); }; IngestCallback?.Invoke(fileObject); Interlocked.Increment(ref TotalFilesDownloaded); } else { Log.Warning($"destination file exists. skipping download:\r\n file: {fileObject}"); IngestCallback?.Invoke(fileObject); } } } else { Log.Error("unable to read blob modified date", blobRef); TotalErrors++; } } }
public DtrTraceRecord(string traceRecord, FileObject fileObject, string resourceUri = null) { Populate(fileObject, traceRecord, resourceUri); }
private void UploadCacheData(List <string> uris) { Log.Info("enter"); List <string> files = new List <string>(); if (uris.Count > 0) { foreach (string file in uris) { if (File.Exists(file)) { Log.Info($"adding file to list: {file}"); files.Add(file); } else { Log.Warning($"file does not exist: {file}"); } } } else { switch (Config.FileType) { case FileTypesEnum.counter: files = Directory.GetFiles(Config.CacheLocation, $"*{PerfCtrExtension}", SearchOption.AllDirectories).ToList(); if (files.Count < 1) { files = Directory.GetFiles(Config.CacheLocation, $"*{PerfCsvExtension}", SearchOption.AllDirectories).ToList(); } break; case FileTypesEnum.setup: files = Directory.GetFiles(Config.CacheLocation, $"*{SetupExtension}", SearchOption.AllDirectories).ToList(); break; case FileTypesEnum.table: files = Directory.GetFiles(Config.CacheLocation, $"*{TableExtension}", SearchOption.AllDirectories).ToList(); break; case FileTypesEnum.trace: files = Directory.GetFiles(Config.CacheLocation, $"*{TraceFileExtension}{ZipExtension}", SearchOption.AllDirectories).ToList(); if (files.Count < 1) { files = Directory.GetFiles(Config.CacheLocation, $"*{TraceFileExtension}", SearchOption.AllDirectories).ToList(); } break; default: Log.Warning($"invalid filetype for cache upload. returning {Config.FileType}"); return; } } if (files.Count < 1) { Log.Error($"configuration set to upload cache files from 'cachelocation' {Config.CacheLocation} but no files found"); } foreach (string file in files) { FileObject fileObject = new FileObject(file, Config.CacheLocation); Log.Info($"adding file: {fileObject.FileUri}", ConsoleColor.Green); if (!Config.List) { QueueForIngest(fileObject); } } }
public FileObjectCollection ProcessFile(FileObject fileObject) { Log.Debug($"enter:{fileObject.FileUri}"); if (fileObject.DownloadAction != null) { Log.Info($"downloading:{fileObject.FileUri}", ConsoleColor.Cyan, ConsoleColor.DarkBlue); _fileTasks.TaskAction(fileObject.DownloadAction).Wait(); Log.Info($"downloaded:{fileObject.FileUri}", ConsoleColor.DarkCyan, ConsoleColor.DarkBlue); } if (fileObject.DownloadAction != null && fileObject.Stream.Get().Length < 1 && !fileObject.Exists) { string error = $"memoryStream does not exist and file does not exist {fileObject.FileUri}"; Log.Error(error); throw new ArgumentException(error); } if (!fileObject.FileType.Equals(FileTypesEnum.any)) { if (fileObject.Stream.Get().Length < 1 & fileObject.Exists) { // for cached directory uploads fileObject.Stream.ReadFromFile(); } if (fileObject.FileExtensionType.Equals(FileExtensionTypesEnum.zip)) { fileObject.Stream.Decompress(); SaveToCache(fileObject); } } switch (fileObject.FileDataType) { case FileDataTypesEnum.unknown: { if (fileObject.FileType.Equals(FileTypesEnum.any)) { SaveToCache(fileObject); return(new FileObjectCollection()); } Log.Warning("unknown file", fileObject); break; } case FileDataTypesEnum.counter: { if (fileObject.FileExtensionType.Equals(FileExtensionTypesEnum.blg)) { return(FormatCounterFile(fileObject)); } break; } case FileDataTypesEnum.data: case FileDataTypesEnum.fabriccrashdumps: { if (fileObject.FileExtensionType.Equals(FileExtensionTypesEnum.dmp)) { return(FormatExceptionFile(fileObject)); } SaveToCache(fileObject); return(new FileObjectCollection()); } case FileDataTypesEnum.fabric: case FileDataTypesEnum.lease: { if (fileObject.FileExtensionType.Equals(FileExtensionTypesEnum.dtr)) { return(FormatDtrFile(fileObject)); } break; } case FileDataTypesEnum.bootstrap: case FileDataTypesEnum.fabricsetup: case FileDataTypesEnum.fabricdeployer: { if (fileObject.FileExtensionType.Equals(FileExtensionTypesEnum.trace)) { return(FormatSetupFile(fileObject)); } break; } case FileDataTypesEnum.table: { if (fileObject.FileExtensionType.Equals(FileExtensionTypesEnum.csv)) { return(FormatTableFile(fileObject)); } break; } default: { Log.Warning("unsupported file data type", fileObject.FileDataType); break; } } Log.Warning($"returning: empty fileObjectCollection for file: {fileObject.FileUri}"); return(new FileObjectCollection()); }
public IRecord Populate(FileObject fileObject, string dtrRecord, string resourceUri = null) { return(this as ITraceRecord); }
private void QueueBlobSegmentDownload(BlobResultSegment blobResultSegment) { int parentId = Thread.CurrentThread.ManagedThreadId; Log.Debug($"enter. current id:{parentId}. results count: {blobResultSegment.Results.Count()}"); long segmentMinDateTicks = Interlocked.Read(ref DiscoveredMinDateTicks); long segmentMaxDateTicks = Interlocked.Read(ref DiscoveredMaxDateTicks); foreach (IListBlobItem blob in blobResultSegment.Results) { ICloudBlob blobRef = null; Log.Debug($"parent id:{parentId} current Id:{Thread.CurrentThread.ManagedThreadId}"); if (blob is CloudBlobDirectory) { if (!string.IsNullOrEmpty(Config.NodeFilter) && !Regex.IsMatch(blob.Uri.ToString(), Config.NodeFilter, RegexOptions.IgnoreCase)) { Log.Debug($"blob:{blob.Uri} does not match nodeFilter pattern:{Config.NodeFilter}, skipping..."); continue; } DownloadBlobsFromDirectory(blob as CloudBlobDirectory); Log.Debug("blob is directory."); continue; } Interlocked.Increment(ref TotalFilesEnumerated); if (Regex.IsMatch(blob.Uri.ToString(), FileFilterPattern, RegexOptions.IgnoreCase)) { long ticks = Convert.ToInt64(Regex.Match(blob.Uri.ToString(), FileFilterPattern, RegexOptions.IgnoreCase).Groups[1].Value); if (ticks <Config.StartTimeUtc.Ticks | ticks> Config.EndTimeUtc.Ticks) { Interlocked.Increment(ref TotalFilesSkipped); Log.Debug($"exclude:bloburi file ticks {new DateTime(ticks).ToString("o")} outside of time range:{blob.Uri}"); SetMinMaxDate(ref segmentMinDateTicks, ref segmentMaxDateTicks, ticks); continue; } } else { Log.Debug($"regex not matched: {blob.Uri.ToString()} pattern: {FileFilterPattern}"); } try { Log.Debug($"file Blob: {blob.Uri}"); blobRef = blob.Container.ServiceClient.GetBlobReferenceFromServerAsync(blob.Uri).Result; } catch (StorageException se) { Interlocked.Increment(ref TotalErrors); Log.Exception($"getting ref for {blob.Uri}, skipping. {se.Message}"); continue; } if (blobRef.Properties.LastModified.HasValue) { DateTimeOffset lastModified = blobRef.Properties.LastModified.Value; SetMinMaxDate(ref segmentMinDateTicks, ref segmentMaxDateTicks, lastModified.Ticks); if (!string.IsNullOrEmpty(Config.UriFilter) && !Regex.IsMatch(blob.Uri.ToString(), Config.UriFilter, RegexOptions.IgnoreCase)) { Interlocked.Increment(ref TotalFilesSkipped); Log.Debug($"blob:{blob.Uri} does not match uriFilter pattern:{Config.UriFilter}, skipping..."); continue; } if (Config.FileType != FileTypesEnum.any && !FileTypes.MapFileTypeUri(blob.Uri.AbsolutePath).Equals(Config.FileType)) { Interlocked.Increment(ref TotalFilesSkipped); Log.Debug($"skipping uri with incorrect file type: {FileTypes.MapFileTypeUri(blob.Uri.AbsolutePath)}"); continue; } if (lastModified >= Config.StartTimeUtc && lastModified <= Config.EndTimeUtc) { Interlocked.Increment(ref TotalFilesMatched); if (Config.List) { Log.Info($"listing file with timestamp: {lastModified}\r\n file: {blob.Uri.AbsolutePath}"); continue; } if (ReturnSourceFileLink) { IngestCallback?.Invoke(new FileObject(blob.Uri.AbsolutePath, Config.SasEndpointInfo.BlobEndpoint) { Length = blobRef.Properties.Length, LastModified = lastModified }); continue; } FileObject fileObject = new FileObject(blob.Uri.AbsolutePath, Config.CacheLocation) { Length = blobRef.Properties.Length, LastModified = lastModified }; Log.Info($"queueing blob with timestamp: {lastModified}\r\n file: {blob.Uri.AbsolutePath}"); InvokeCallback(blob, fileObject); } else { Interlocked.Increment(ref TotalFilesSkipped); Log.Debug($"exclude:bloburi {lastModified.ToString("o")} outside of time range:{blob.Uri}"); SetMinMaxDate(ref segmentMinDateTicks, ref segmentMaxDateTicks, lastModified.Ticks); continue; } } else { Log.Error("unable to read blob modified date", blobRef); TotalErrors++; } } }