public static void EmptyContainer(CloudBlobContainer container)
{
var blobInfos = container.ListBlobs(null, true, BlobListingDetails.None);
CATFunctions.Print("Deleting old files (" + blobInfos.Count() + ")", true, false);
CATFunctions.ShowProgress(blobInfos.Count());
CATFunctions.StartSubProcess("Deleting old files...");
Parallel.ForEach(blobInfos, (blobInfo) =>
{
CloudBlob blob = (CloudBlob)blobInfo;
blob.Delete();
CATFunctions.Progress();
CATFunctions.Print("Deleting blob " + blob.Name);
});
CATFunctions.Print("Container is empty", true, true);
CATFunctions.EndSubProcess();
CATFunctions.FinishProgress();
}
static void RunOptions(Options opts)
{
//if (string.IsNullOrEmpty(opts.Extension) && !opts.Replace)
//{
// return;
//}
CATFunctions.StartProcess("");
CATFunctions.DisplayProcessnigAnimation(true);
BlobServiceClient service = new BlobServiceClient(opts.ConnectionString);
BlobContainerClient blobContainer = service.GetBlobContainerClient(opts.Container);
Utility.Process(opts.SourceDirectory,
blobContainer,
opts.Extensions,
opts.MaxAgeSeconds,
opts.DefaultContenType,
opts.ClearContainer.GetValueOrDefault(),
opts.SyncContainer.GetValueOrDefault(),
opts.ExcludeDirs);
CATFunctions.Print("Process is done.", true, false);
CATFunctions.DisplayProcessnigAnimation(false);
CATFunctions.EndProcess();
}
static void HandleParseError(IEnumerable <Error> errs)
{
CATFunctions.Print(errs.Count() + " errors occured");
foreach (Error item in errs)
{
CATFunctions.Print(item.ToString());
}
}
public static void Process(string SourceDir,
BlobContainerClient container,
IEnumerable <string> extensions,
int cacheControlMaxAgeSeconds,
string defaultContenType,
bool clearContainer,
bool syncContainer,
IEnumerable <string> excludeDirs)
{
DirectoryInfo dirInfo = new DirectoryInfo(SourceDir);
//step 1
DeleteUnnecessaryFilesFromSourceDir(dirInfo, ".gz");
DeleteUnnecessaryFilesFromSourceDir(dirInfo, ".br");
List <FileInfo> fileslist = dirInfo.GetFiles("*", SearchOption.AllDirectories).ToList();
//step 2
ReportFilesWithNoExtension(fileslist);
//step 3
EnsureContentTypes(fileslist, defaultContenType);
//step 4
if (clearContainer)
{
EmptyContainer(container);
}
//step 5
if (excludeDirs.Any())
{
fileslist.RemoveAll(x => excludeDirs.Any(y => y.ToLower().Equals(x.Directory.FullName.ToLower())));
}
//step 6 sync or upload
if (syncContainer)
{
SyncFiles(SourceDir, container, fileslist, extensions, cacheControlMaxAgeSeconds);
}
else
{
UploadFiles(SourceDir, container, fileslist, extensions, cacheControlMaxAgeSeconds);
}
//step 7
List <string> allExtensions = fileslist.Select(x => x.Extension.ToLower()).Distinct().ToList();
CATFunctions.Print(string.Empty, true);
foreach (var item in allExtensions.Where(x => !extensions.Any(y => y.ToLower().Equals(x))))
{
CATFunctions.Print(item + " extension was not compressed, is this ok?????????????????????");
}
CATFunctions.Print(string.Empty, true);
}
public static void UploadFiles(string SourceDir,
BlobContainerClient container,
IEnumerable <FileInfo> fileslist,
IEnumerable <string> extensions,
int cacheControlMaxAgeSeconds)
{
string cacheControlHeader = "public, max-age=" + cacheControlMaxAgeSeconds.ToString();
CATFunctions.Print("Processing files...", true, false);
CATFunctions.ShowProgress(fileslist.Count());
CATFunctions.StartSubProcess("Uploading files...");
Parallel.ForEach(fileslist, (fileInfo) =>
{
CATFunctions.Progress();
ContentTypeHelper.CurrentContentTypes.TryGetValue(fileInfo.Extension, out string contentType);
string filePath = GetFilePath(SourceDir, fileInfo, true);
BlobClient blob = container.GetBlobClient(filePath);
if (extensions.Contains(fileInfo.Extension, StringComparer.OrdinalIgnoreCase))
{
CATFunctions.Print("Compress file - " + filePath);
using (MemoryStream memoryStream = new MemoryStream())
{
using (var brotliStream = new BrotliStream(memoryStream, CompressionMode.Compress, true))
using (var blobStream = fileInfo.OpenRead())
{
blobStream.CopyTo(brotliStream);
}
memoryStream.Position = 0;
blob.Upload(memoryStream);
CATFunctions.Print("uploading file - " + filePath);
BlobHttpHeaders headers = new()
{
ContentType = contentType,
ContentEncoding = "br",
CacheControl = cacheControlHeader,
};
blob.SetHttpHeadersAsync(headers);
}
public static void ReportFilesWithNoExtension(IEnumerable <FileInfo> fileslist)
{
if (fileslist.Any(x => string.IsNullOrEmpty(x.Extension)))
{
CATFunctions.Print("======!!!!!!! Warning !!!!!!=======", true, false);
CATFunctions.Print("Found " + fileslist.Where(x => string.IsNullOrEmpty(x.Extension)).Count() + " files with no extension", false, true);
foreach (var item in fileslist.Where(x => string.IsNullOrEmpty(x.Extension)))
{
CATFunctions.Print("Warning - No file extension - " + item.FullName);
}
}
}
static void RunOptions(Options opts)
{
//if (string.IsNullOrEmpty(opts.Extension) && !opts.Replace)
//{
// return;
//}
CATFunctions.StartProcess("");
CATFunctions.DisplayProcessnigAnimation(true);
CloudStorageAccount storageAccount;
if (!string.IsNullOrEmpty(opts.StorageAccount) && !string.IsNullOrEmpty(opts.StorageKey))
{
storageAccount = new CloudStorageAccount(new StorageCredentials(opts.StorageAccount, opts.StorageKey), true);
}
else
{
return;
}
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer blobContainer = blobClient.GetContainerReference(opts.Container);
Utility.Process(opts.SourceDirectory,
blobContainer,
opts.Extensions,
opts.MaxAgeSeconds,
opts.DefaultContenType,
opts.ClearContainer);
if (opts.Wildcard)
{
Utility.SetWildcardCorsOnBlobService(storageAccount);
}
CATFunctions.Print("Process is done.", true, false);
CATFunctions.DisplayProcessnigAnimation(false);
CATFunctions.EndProcess();
}
public static void EnsureContentTypes(IEnumerable <FileInfo> fileslist, string defaultContenType)
{
bool b = true;
CATFunctions.Print("Checking content types...", true, true);
List <string> fileExtensions = fileslist.Select(o => o.Extension).Distinct().ToList();
ContentTypeHelper.CurrentContentTypes = new Dictionary <string, string>();
string contentType;
foreach (var item in fileExtensions)
{
if (string.IsNullOrEmpty(item))
{
contentType = defaultContenType;
}
else
{
if (ContentTypeHelper.contentTypes.TryGetValue(item, out contentType))
{
ContentTypeHelper.CurrentContentTypes.Add(item, contentType);
}
else
{
CATFunctions.Print("Warning - Content type is missing for " + item);
ContentTypeHelper.CurrentContentTypes.Add(item, defaultContenType);
b = false;
}
}
}
if (!b)
{
CATFunctions.Print("======!!!!!!! Warning !!!!!!=======", true, false);
CATFunctions.Print("For all missing cointent type extensions will be set to default or provided value - " + defaultContenType, false, true);
}
CATFunctions.Print("Content types are ready.", true, true);
}
public static void DeleteUnnecessaryFilesFromSourceDir(DirectoryInfo dirInfo)
{
IEnumerable <FileInfo> fileslist = dirInfo.GetFiles("*", SearchOption.AllDirectories).Where(x => x.Extension.Equals(".gz"));
if (fileslist.Any())
{
CATFunctions.Print("======!!!!!!! Warning !!!!!!=======", true, false);
CATFunctions.Print("Found " + fileslist.Count() + " files with extension .gz");
CATFunctions.Print(".gz files are not necessary for deploy.", false, true);
foreach (var item in fileslist)
{
File.Delete(item.FullName);
CATFunctions.Print("Deleting - " + item.FullName);
}
}
}
public static void SetCORSPropertiesOnBlobService(this CloudStorageAccount storageAccount,
Func <CorsProperties, CorsProperties> alterCorsRules)
{
CATFunctions.Print("Configuring CORS.", true, true);
if (storageAccount == null || alterCorsRules == null)
{
throw new ArgumentNullException();
}
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
ServiceProperties serviceProperties = blobClient.GetServiceProperties();
serviceProperties.Cors = alterCorsRules(serviceProperties.Cors) ?? new CorsProperties();
blobClient.SetServiceProperties(serviceProperties);
}
public static void UploadFiles(string SourceDir,
CloudBlobContainer container,
IEnumerable <FileInfo> fileslist,
IEnumerable <string> extensions,
int cacheControlMaxAgeSeconds)
{
int SourceDirLenght = SourceDir.Length + 1;
string cacheControlHeader = "public, max-age=" + cacheControlMaxAgeSeconds.ToString();
CATFunctions.Print("Processing files...", true, false);
CATFunctions.ShowProgress(fileslist.Count());
CATFunctions.StartSubProcess("Uploading files...");
Parallel.ForEach(fileslist, (fileInfo) =>
{
CATFunctions.Progress();
string contentType;
ContentTypeHelper.CurrentContentTypes.TryGetValue(fileInfo.Extension, out contentType);
string filePath = GetFilePath(SourceDirLenght, fileInfo);
CloudBlockBlob blob = container.GetBlockBlobReference(filePath);
if (extensions.Contains(fileInfo.Extension, StringComparer.OrdinalIgnoreCase))
{
byte[] compressedBytes;
CATFunctions.Print("Gzip file - " + filePath);
using (MemoryStream memoryStream = new MemoryStream())
{
using (var gzipStream = new GZipStream(memoryStream, CompressionMode.Compress))
using (var blobStream = fileInfo.OpenRead())
{
blobStream.CopyTo(gzipStream);
}
compressedBytes = memoryStream.ToArray();
CATFunctions.Print("uploading file - " + filePath);
blob.UploadFromByteArray(compressedBytes, 0, compressedBytes.Length);
blob.Properties.CacheControl = cacheControlHeader;
blob.Properties.ContentEncoding = "gzip";
blob.Properties.ContentType = contentType;
blob.SetProperties();
}
}
else
{
CATFunctions.Print("uploading file - " + filePath);
blob.UploadFromFile(fileInfo.FullName);
blob.Properties.CacheControl = cacheControlHeader;
blob.Properties.ContentType = contentType;
blob.SetProperties();
}
});
CATFunctions.EndSubProcess();
CATFunctions.FinishProgress();
}
