private static async Task UploadAndTransform(string rootDir, string transformDir)
{
var uploader = new BlobUploader(await AzureUtilities.GetImagesBlobContainerAsync(ImageDataConnectionEnv));
var imgSetTable = await AzureUtilities.GetImageSetTable(ImageDataConnectionEnv);
var imgTransformTable = await AzureUtilities.GetImageTransformTable(ImageDataConnectionEnv);
var crawler = new ImageDirectoryCrawler()
{
TagExtractor = x => x.Split('\\'),
BlobUploader = (f, b) =>
{
var fileinfo = new FileInfo(f);
var blobPath = "noodlefrenzy/" + (b.Length > 0 ? (b + "/") : "") + fileinfo.Name;
Trace.TraceInformation("Uploading '{0}' to '{1}'", f, blobPath);
return uploader.Upload(f, blobPath);
},
ImageSetUpserter = imgSet =>
{
var op = TableOperation.InsertOrReplace(imgSet);
return imgSetTable.ExecuteAsync(op);
}
};
var imageMagickPath = Environment.GetEnvironmentVariable(ImageMagickPathEnv);
try
{
// Transform to charcoal...
var transform = new ImageTransform("Charcoal", "0")
{
CommandLineArguments = "-charcoal 2 {infile} {outfile}"
};
var upsert = TableOperation.InsertOrReplace(transform);
await imgTransformTable.ExecuteAsync(upsert);
await crawler.TransformTree(rootDir, "0", transform, imageMagickPath, transformDir);
// Upload initial original images.
await crawler.WalkTree(rootDir, "0");
}
catch (AggregateException e)
{
Console.WriteLine("Crawling failed:");
foreach (var ex in e.InnerExceptions)
{
Console.WriteLine(ex.Message);
}
}
}
/// <summary>
/// Set of (possibly transformed) images.
/// </summary>
/// <remarks>
/// Blob path is original/_dir_/_image version_, or if transformed it's transform/_transform name_/_transform version_/_dir_/_image version_
/// </remarks>
public ImageSet(string pathSuffix, string version, ImageTransform transform = null)
{
if (pathSuffix == null) throw new ArgumentNullException("pathSuffix");
var prefix = transform == null ? "original/" : string.Format("transform/{0}/{1}/", transform.Name, transform.Version);
this.BlobPath = prefix + pathSuffix.Replace('\\', '/') + "/" + version;
this.Path = pathSuffix == "" ? "<root>" : pathSuffix;
this.Version = version;
this.Tags = new List<string>();
this.PartitionKey = this.CleanPartitionKey(this.Path);
this.RowKey = this.CleanRowKey(this.Version);
if (transform != null)
{
this.TransformPartitionKey = transform.PartitionKey;
this.TransformRowKey = transform.RowKey;
}
}
/// <summary>
/// Walk the given directory (and all children) looking for images, transform, upload to blob storage, and store metadata in table storage.
/// </summary>
/// <param name="rootDirectory">Directory to walk (recursively).</param>
/// <param name="imagesVersion">Version to tag to uploaded image sets.</param>
/// <returns></returns>
/// <remarks>
/// Side-effect: Transformed images are left on disk in transformedRoot. For us, this is a feature :)
/// NOTE: Does NOT upsert ImageTransform - it's assumed this is a known transform that already exists.
/// </remarks>
public async Task TransformTree(string rootDirectory, string imagesVersion, ImageTransform transform, string imageMagickPath, string transformedRoot)
{
TestPreconditions(rootDirectory);
var images = from file in Directory.EnumerateFiles(rootDirectory, "*.*", SearchOption.AllDirectories).Select(x => new FileInfo(x))
where this.Extensions.Contains(file.Extension)
select file;
var byDirectory = from img in images
group img by img.DirectoryName;
var uploadTasks = Enumerable.Empty<Tuple<FileInfo, Task>>();
var imgSets = new List<ImageSet>();
foreach (var dir in byDirectory)
{
var suffix = dir.Key.Length == rootDirectory.Length ? "" : dir.Key.Substring(rootDirectory.Length + 1);
suffix = suffix.Trim();
var transformDir = Path.Combine(transformedRoot, suffix);
Directory.CreateDirectory(transformDir);
var imgSet = new ImageSet(suffix, imagesVersion, transform)
{
Tags = this.TagExtractor(suffix).Select(x => x.Trim()).Where(x => !string.IsNullOrEmpty(x)).ToList()
};
foreach (var file in dir)
{
var infile = file.FullName;
var outfile = Path.Combine(transformDir, file.Name);
var cmdline = transform.GetCommandLineArguments(infile, outfile);
Trace.TraceInformation("Transforming: '{0} {1}'", imageMagickPath, cmdline);
var proc = Process.Start(new ProcessStartInfo()
{
FileName = imageMagickPath,
Arguments = cmdline,
UseShellExecute = false,
RedirectStandardError = true
});
proc.WaitForExit();
var exitCode = proc.ExitCode;
proc.Close();
if (exitCode != 0)
{
Trace.TraceWarning("Failed to execute '{0} {1}': Code {2}", imageMagickPath, cmdline, exitCode);
}
}
var transformedImages = from file in Directory.EnumerateFiles(transformDir, "*.*", SearchOption.TopDirectoryOnly).Select(x => new FileInfo(x))
where this.Extensions.Contains(file.Extension)
select file;
if (transformedImages.Any())
{
Trace.TraceInformation("New Image Set {0} w/ tags ('{1}')", imgSet.PartitionKey, string.Join("', '", imgSet.Tags));
uploadTasks = uploadTasks.Concat(transformedImages.Select(file =>
Tuple.Create(file, this.BlobUploader(file.FullName, imgSet.BlobPath))));
imgSets.Add(imgSet);
}
else
{
Trace.TraceWarning("No transformed images found for '{0}'", transformDir);
}
}
var failedUpserts = await Utilities.ThrottleWork(MaxParallelUpserts, imgSets.Select(imgSet => this.ImageSetUpserter(imgSet)));
var failedUploads = await Utilities.ThrottleWork(MaxParallelUploads, uploadTasks.Select(x => x.Item2));
if (failedUpserts.Any() || failedUploads.Any())
{
var filesByTask = uploadTasks.ToDictionary(x => x.Item2, x => x.Item1);
throw Utilities.AsAggregateException(failedUploads.Concat(failedUpserts),
t => filesByTask.ContainsKey(t) ? string.Format("Failed upload for '{0}'", filesByTask[t]) : null);
}
}