private static async Task UploadAndTransform(string rootDir, string transformDir) { var uploader = new BlobUploader(await AzureUtilities.GetImagesBlobContainerAsync(ImageDataConnectionEnv)); var imgSetTable = await AzureUtilities.GetImageSetTable(ImageDataConnectionEnv); var imgTransformTable = await AzureUtilities.GetImageTransformTable(ImageDataConnectionEnv); var crawler = new ImageDirectoryCrawler() { TagExtractor = x => x.Split('\\'), BlobUploader = (f, b) => { var fileinfo = new FileInfo(f); var blobPath = "noodlefrenzy/" + (b.Length > 0 ? (b + "/") : "") + fileinfo.Name; Trace.TraceInformation("Uploading '{0}' to '{1}'", f, blobPath); return uploader.Upload(f, blobPath); }, ImageSetUpserter = imgSet => { var op = TableOperation.InsertOrReplace(imgSet); return imgSetTable.ExecuteAsync(op); } }; var imageMagickPath = Environment.GetEnvironmentVariable(ImageMagickPathEnv); try { // Transform to charcoal... var transform = new ImageTransform("Charcoal", "0") { CommandLineArguments = "-charcoal 2 {infile} {outfile}" }; var upsert = TableOperation.InsertOrReplace(transform); await imgTransformTable.ExecuteAsync(upsert); await crawler.TransformTree(rootDir, "0", transform, imageMagickPath, transformDir); // Upload initial original images. await crawler.WalkTree(rootDir, "0"); } catch (AggregateException e) { Console.WriteLine("Crawling failed:"); foreach (var ex in e.InnerExceptions) { Console.WriteLine(ex.Message); } } }
/// <summary> /// Set of (possibly transformed) images. /// </summary> /// <remarks> /// Blob path is original/_dir_/_image version_, or if transformed it's transform/_transform name_/_transform version_/_dir_/_image version_ /// </remarks> public ImageSet(string pathSuffix, string version, ImageTransform transform = null) { if (pathSuffix == null) throw new ArgumentNullException("pathSuffix"); var prefix = transform == null ? "original/" : string.Format("transform/{0}/{1}/", transform.Name, transform.Version); this.BlobPath = prefix + pathSuffix.Replace('\\', '/') + "/" + version; this.Path = pathSuffix == "" ? "<root>" : pathSuffix; this.Version = version; this.Tags = new List<string>(); this.PartitionKey = this.CleanPartitionKey(this.Path); this.RowKey = this.CleanRowKey(this.Version); if (transform != null) { this.TransformPartitionKey = transform.PartitionKey; this.TransformRowKey = transform.RowKey; } }
/// <summary> /// Walk the given directory (and all children) looking for images, transform, upload to blob storage, and store metadata in table storage. /// </summary> /// <param name="rootDirectory">Directory to walk (recursively).</param> /// <param name="imagesVersion">Version to tag to uploaded image sets.</param> /// <returns></returns> /// <remarks> /// Side-effect: Transformed images are left on disk in transformedRoot. For us, this is a feature :) /// NOTE: Does NOT upsert ImageTransform - it's assumed this is a known transform that already exists. /// </remarks> public async Task TransformTree(string rootDirectory, string imagesVersion, ImageTransform transform, string imageMagickPath, string transformedRoot) { TestPreconditions(rootDirectory); var images = from file in Directory.EnumerateFiles(rootDirectory, "*.*", SearchOption.AllDirectories).Select(x => new FileInfo(x)) where this.Extensions.Contains(file.Extension) select file; var byDirectory = from img in images group img by img.DirectoryName; var uploadTasks = Enumerable.Empty<Tuple<FileInfo, Task>>(); var imgSets = new List<ImageSet>(); foreach (var dir in byDirectory) { var suffix = dir.Key.Length == rootDirectory.Length ? "" : dir.Key.Substring(rootDirectory.Length + 1); suffix = suffix.Trim(); var transformDir = Path.Combine(transformedRoot, suffix); Directory.CreateDirectory(transformDir); var imgSet = new ImageSet(suffix, imagesVersion, transform) { Tags = this.TagExtractor(suffix).Select(x => x.Trim()).Where(x => !string.IsNullOrEmpty(x)).ToList() }; foreach (var file in dir) { var infile = file.FullName; var outfile = Path.Combine(transformDir, file.Name); var cmdline = transform.GetCommandLineArguments(infile, outfile); Trace.TraceInformation("Transforming: '{0} {1}'", imageMagickPath, cmdline); var proc = Process.Start(new ProcessStartInfo() { FileName = imageMagickPath, Arguments = cmdline, UseShellExecute = false, RedirectStandardError = true }); proc.WaitForExit(); var exitCode = proc.ExitCode; proc.Close(); if (exitCode != 0) { Trace.TraceWarning("Failed to execute '{0} {1}': Code {2}", imageMagickPath, cmdline, exitCode); } } var transformedImages = from file in Directory.EnumerateFiles(transformDir, "*.*", SearchOption.TopDirectoryOnly).Select(x => new FileInfo(x)) where this.Extensions.Contains(file.Extension) select file; if (transformedImages.Any()) { Trace.TraceInformation("New Image Set {0} w/ tags ('{1}')", imgSet.PartitionKey, string.Join("', '", imgSet.Tags)); uploadTasks = uploadTasks.Concat(transformedImages.Select(file => Tuple.Create(file, this.BlobUploader(file.FullName, imgSet.BlobPath)))); imgSets.Add(imgSet); } else { Trace.TraceWarning("No transformed images found for '{0}'", transformDir); } } var failedUpserts = await Utilities.ThrottleWork(MaxParallelUpserts, imgSets.Select(imgSet => this.ImageSetUpserter(imgSet))); var failedUploads = await Utilities.ThrottleWork(MaxParallelUploads, uploadTasks.Select(x => x.Item2)); if (failedUpserts.Any() || failedUploads.Any()) { var filesByTask = uploadTasks.ToDictionary(x => x.Item2, x => x.Item1); throw Utilities.AsAggregateException(failedUploads.Concat(failedUpserts), t => filesByTask.ContainsKey(t) ? string.Format("Failed upload for '{0}'", filesByTask[t]) : null); } }