private string DetermineClusterId()
{
string clusterId = string.Empty;
if (!string.IsNullOrEmpty(Config.SasEndpointInfo.AbsolutePath))
{
//fabriclogs-e2fd6f05-921f-4e81-92d5-f70a648be762
string pattern = ".+-([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})";
if (Regex.IsMatch(Config.SasEndpointInfo.AbsolutePath, pattern))
{
clusterId = Regex.Match(Config.SasEndpointInfo.AbsolutePath, pattern).Groups[1].Value;
}
}
if (string.IsNullOrEmpty(clusterId))
{
TableManager tableMgr = new TableManager();
if (tableMgr.Connect())
{
clusterId = tableMgr.QueryTablesForClusterId();
}
}
Log.Info($"cluster id:{clusterId}");
return(clusterId);
}
private void DownloadAzureData(List <string> uris = null)
{
string containerPrefix = null;
string tablePrefix = null;
string clusterId = DetermineClusterId();
if (!Config.FileType.Equals(FileTypesEnum.any) && !Config.FileType.Equals(FileTypesEnum.table))
{
containerPrefix = FileTypes.MapFileTypeUriPrefix(Config.FileType);
if (!string.IsNullOrEmpty(clusterId))
{
// 's-' in prefix may not always be correct
containerPrefix += "s-" + clusterId;
}
tablePrefix = containerPrefix + clusterId?.Replace("-", "");
}
if (Config.FileType == FileTypesEnum.table)
{
TableManager tableMgr = new TableManager()
{
IngestCallback = (exportedFile) => { QueueForIngest(exportedFile); }
};
if (tableMgr.Connect())
{
tableMgr.DownloadTables(tablePrefix);
}
}
else
{
BlobManager blobMgr = new BlobManager()
{
IngestCallback = (sourceFileUri) => { QueueForIngest(sourceFileUri); },
ReturnSourceFileLink = (Config.IsKustoConfigured() & Config.KustoUseBlobAsSource) | Config.FileType == FileTypesEnum.exception
};
if (blobMgr.Connect())
{
if (uris?.Count > 0)
{
blobMgr.DownloadFiles(uris);
}
else
{
blobMgr.DownloadContainers(containerPrefix);
}
}
}
}
