public string UploadFileToBlobContainer(FileObject fileObject, string blobContainerUri, string containerName, string blobName)
{
Log.Info($"uploading: {fileObject.Stream.Get().Length} bytes to {fileObject.FileUri} to {blobContainerUri}", ConsoleColor.Magenta);
Uri blobUri = new Uri(blobContainerUri);
BlobRequestOptions blobRequestOptions = new BlobRequestOptions()
{
RetryPolicy = new IngestRetryPolicy(),
ParallelOperationThreadCount = Config.Threads,
};
CloudBlobContainer blobContainer = new CloudBlobContainer(blobUri);
CloudBlockBlob blockBlob = blobContainer.GetBlockBlobReference(blobName);
if (Config.UseMemoryStream)
{
_kustoTasks.TaskAction(() => blockBlob.UploadFromStreamAsync(fileObject.Stream.Get(), null, blobRequestOptions, null).Wait()).Wait();
fileObject.Stream.Close();
}
else
{
_kustoTasks.TaskAction(() => blockBlob.UploadFromFileAsync(fileObject.FileUri, null, blobRequestOptions, null).Wait()).Wait();
}
Log.Info($"uploaded: {fileObject.FileUri} to {blobContainerUri}", ConsoleColor.DarkMagenta);
return($"{blockBlob.Uri.AbsoluteUri}{blobUri.Query}");
}
private void DownloadBlobsFromDirectory(CloudBlobDirectory directory)
{
Log.Info($"enumerating:{directory.Uri}", ConsoleColor.Cyan);
foreach (BlobResultSegment segment in EnumerateDirectoryBlobs(directory))
{
_blobChildTasks.TaskAction(() => QueueBlobSegmentDownload(segment));
}
}
private void DownloadBlobsFromContainer(CloudBlobContainer container)
{
Log.Info($"enumerating:{container.Name}", ConsoleColor.Black, ConsoleColor.Cyan);
foreach (BlobResultSegment segment in EnumerateContainerBlobs(container))
{
_blobTasks.TaskAction(() => QueueBlobSegmentDownload(segment));
}
}
public FileObjectCollection ProcessFile(FileObject fileObject)
{
Log.Debug($"enter:{fileObject.FileUri}");
if (fileObject.DownloadAction != null)
{
Log.Info($"downloading:{fileObject.FileUri}", ConsoleColor.Cyan, ConsoleColor.DarkBlue);
_fileTasks.TaskAction(fileObject.DownloadAction).Wait();
Log.Info($"downloaded:{fileObject.FileUri}", ConsoleColor.DarkCyan, ConsoleColor.DarkBlue);
}
if (fileObject.DownloadAction != null && fileObject.Stream.Get().Length < 1 && !fileObject.Exists)
{
string error = $"memoryStream does not exist and file does not exist {fileObject.FileUri}";
Log.Error(error);
throw new ArgumentException(error);
}
if (!fileObject.FileType.Equals(FileTypesEnum.any))
{
if (fileObject.Stream.Get().Length < 1 & fileObject.Exists)
{
// for cached directory uploads
fileObject.Stream.ReadFromFile();
}
if (fileObject.FileExtensionType.Equals(FileExtensionTypesEnum.zip))
{
fileObject.Stream.Decompress();
SaveToCache(fileObject);
}
}
switch (fileObject.FileDataType)
{
case FileDataTypesEnum.unknown:
{
if (fileObject.FileType.Equals(FileTypesEnum.any))
{
SaveToCache(fileObject);
return(new FileObjectCollection());
}
Log.Warning("unknown file", fileObject);
break;
}
case FileDataTypesEnum.counter:
{
if (fileObject.FileExtensionType.Equals(FileExtensionTypesEnum.blg))
{
return(FormatCounterFile(fileObject));
}
break;
}
case FileDataTypesEnum.data:
case FileDataTypesEnum.fabriccrashdumps:
{
if (fileObject.FileExtensionType.Equals(FileExtensionTypesEnum.dmp))
{
return(FormatExceptionFile(fileObject));
}
SaveToCache(fileObject);
return(new FileObjectCollection());
}
case FileDataTypesEnum.fabric:
case FileDataTypesEnum.lease:
{
if (fileObject.FileExtensionType.Equals(FileExtensionTypesEnum.dtr))
{
return(FormatDtrFile(fileObject));
}
break;
}
case FileDataTypesEnum.bootstrap:
case FileDataTypesEnum.fabricsetup:
case FileDataTypesEnum.fabricdeployer:
{
if (fileObject.FileExtensionType.Equals(FileExtensionTypesEnum.trace))
{
return(FormatSetupFile(fileObject));
}
break;
}
case FileDataTypesEnum.table:
{
if (fileObject.FileExtensionType.Equals(FileExtensionTypesEnum.csv))
{
return(FormatTableFile(fileObject));
}
break;
}
default:
{
Log.Warning("unsupported file data type", fileObject.FileDataType);
break;
}
}
Log.Warning($"returning: empty fileObjectCollection for file: {fileObject.FileUri}");
return(new FileObjectCollection());
}