/// <summary> /// Gets all VOD Asset Id to Poster Maps for all VHO's /// </summary> /// <param name="config">Configuration that contains all VHO data</param> /// <returns>Enumerable with a tuple that contains the vod asset id and poster file (full path)</returns> public static IEnumerable <Tuple <int, string> > GetAllPosterSourceMaps(NGVodPosterConfig config, CancellationToken CancelToken) { StringBuilder sbCmd = new StringBuilder(); sbCmd.AppendFormat("SELECT * FROM {0} WHERE {0}.strPosterFile IS NOT NULL", _posterSourceMapTbl); foreach (var vho in config.Vhos.Keys) { CancelToken.ThrowIfCancellationRequested(); NGVodVHO vodvho = config.Vhos[vho]; foreach (var dr in DBFactory.SQL_ExecuteReader(vodvho.IMGDs.CreateConnectionString(vodvho.IMGDb), sbCmd.ToString(), CommandType.Text, null)) { CancelToken.ThrowIfCancellationRequested(); yield return(new Tuple <int, string>(int.Parse(dr.GetString(0)), dr.IsDBNull(1) ? string.Empty : Path.Combine(config.SourceDir, dr.GetString(1)))); } } }
public static NGVodPosterConfig GetConfig() { var cfgHelper = new CfgHelper(); //Load xml config var config = cfgHelper.GetConfig("NGVODPoster.xml"); if (config == null) { Trace.TraceError("Failed to gete configuration"); throw new Exception("Failed to load configuration xml file."); } //Get namespace var ns = config.Root.GetDefaultNamespace(); var ngVodConfig = new NGVodPosterConfig(); List <Task> tskList = new List <Task>() { ngVodConfig.setVHO(config, ns), ngVodConfig.setDirs(config, ns), ngVodConfig.setImage(config, ns), ngVodConfig.setLogDir(config, ns), ngVodConfig.setSMTP(config, ns), ngVodConfig.setMaxThreads(config, ns), }; try { Task.WaitAll(tskList.ToArray()); ngVodConfig.setVhoDirs(); } catch (AggregateException aex) { foreach (var ex in aex.InnerExceptions) { Trace.TraceError("Error setting config properties. {0}", ex.Message); } throw aex.Flatten(); } return(ngVodConfig); }
/// <summary> /// Gets all VOD assets from all vho's in the config /// </summary> /// <param name="config">Configuration parameters</param> /// <returns></returns> internal IEnumerable <VODAsset> GetAllVodAssets(NGVodPosterConfig config) { foreach (var vho in config.Vhos) { string conStr = vho.Value.IMGDs.CreateConnectionString(vho.Value.IMGDb); string sproc = "sp_FUI_GetAllVODFolderAssetInfo"; foreach (var dr in DBFactory.SQL_ExecuteReader(conStr, sproc, System.Data.CommandType.StoredProcedure)) { var vAsset = new VODAsset(); vAsset.AssetId = int.Parse(dr.GetString(0)); vAsset.Title = dr.GetString(1); vAsset.PID = dr.GetString(2); vAsset.PAID = dr.GetString(3); vAsset.PosterSource = dr.IsDBNull(4) ? string.Empty : dr.GetString(4); vAsset.PosterDest = GetDestImagePath(vAsset.AssetId, vho.Value.PosterDir); yield return(vAsset); } } }
static void Main(string[] args) { int? maxImages = null; string customAction = null; bool onlyNew = false; string debugLog = Path.Combine(Directory.GetCurrentDirectory(), "Debug.log"); NGVodPosterConfig config = null; CancellationTokenSource tokenSource = new CancellationTokenSource(); CancellationToken token = tokenSource.Token; //Reset the output log try { if (File.Exists(debugLog)) { File.Delete(debugLog); } } catch (Exception ex) { Console.WriteLine("Failed to reset log. {0}", ex.Message); } //Create trace listener for output log using (TextWriterTraceListener twtl = new TextWriterTraceListener(debugLog)) { twtl.TraceOutputOptions = TraceOptions.None; twtl.Filter = new EventTypeFilter(SourceLevels.Information); twtl.Name = "TextWriteTraceListener"; try { Trace.Listeners.Clear(); Trace.Listeners.Add(twtl); Trace.AutoFlush = true; } catch (Exception ex) { Console.Error.WriteLine("Failed to set console output log writer. " + ex); } //Get configuration file try { config = NGVodPosterConfig.GetConfig(); } catch (AggregateException aex) { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("***ERROR GETTING CONFIG PARAMETERS***"); foreach (var ex in aex.Flatten().InnerExceptions) { Console.WriteLine("\t{0}", ex.Message); Trace.TraceError("\t" + ex.Message); } Console.ResetColor(); } catch (Exception ex) { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("***ERROR GETTING CONFIG PARAMETERS***"); Console.WriteLine("\t{0}", ex.Message); Trace.TraceError("\t" + ex.Message); Console.ResetColor(); } //Do not continue if configuration was not found if (config == null) { Environment.Exit(500); } string errorLog = Path.Combine(config.LogErrorDir, "NGVodPoster_Error.log"); if (File.Exists(errorLog)) { File.Delete(errorLog); } //Configure error log using (var twtlError = new TextWriterTraceListener(errorLog)) { twtlError.TraceOutputOptions = TraceOptions.Timestamp; twtlError.Filter = new EventTypeFilter(SourceLevels.Error); Trace.Listeners.Add(twtlError); //Handle manual override parameters from console Trace.TraceInformation("Override Params: "); for (int i = 0; i < args.Length; i++) { try { if (args[i].ToUpper().Equals("-D")) { config.DestinationDir = args[i + 1]; Trace.TraceInformation("Destination: {0}", config.DestinationDir); i++; } else if (args[i].ToUpper().Equals("-S")) { config.SourceDir = args[i + 1]; Trace.TraceInformation("Source: {0}", config.SourceDir); i++; } else if (args[i].ToUpper().Equals("-N")) { maxImages = int.Parse(args[i + 1]); Trace.TraceInformation("Max Images: {0}", maxImages); i++; } else if (args[i].ToUpper().Equals("-T")) { config.MaxThreads = int.Parse(args[i + 1]); Trace.TraceInformation("Max Threads: {0}", config.MaxThreads); i++; } else if (args[i].ToUpper().Equals("-STO")) { try { config.AddEmailTo(args[i + 1]); Trace.TraceInformation("Send Missing Poster Log To: {0}", args[i + 1]); } catch (Exception ex) { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("ERROR: Invalid Email Address Provided."); Trace.TraceError("Invalid Email Address Provided."); Console.ResetColor(); throw ex; } i++; } else if (args[i].ToUpper().Equals("-ONLYNEW")) { onlyNew = true; Trace.TraceInformation("Process Only Deltas"); } else if (args[i].ToUpper().Equals("-OD")) { customAction = "OD"; Trace.TraceInformation("Custom Action: {0}", customAction); } else if (args[i].ToUpper().Equals("-OM")) { customAction = "OM"; Trace.TraceInformation("Custom Action: {0}", customAction); } else if (args[i].Contains("?") || args[i].ToUpper().Equals("-H")) { DisplayHelp(); break; } } catch { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("ERROR: Invalid Parameters"); Console.ResetColor(); DisplayHelp(); } } //Split the max threads by number of vho's being processed to prevent overloading the CPU config.MaxThreads = config.MaxThreads / config.Vhos.Count; IProgress <NgVodPosterProgress> progress = new Progress <NgVodPosterProgress>(ReportProgress); //Create the controller using (var ctrl = new NGVodPosterController(token, progress)) { if (!string.IsNullOrEmpty(customAction)) { ctrl.customAction = customAction; } if (onlyNew) { ctrl.OnlyNew = true; } //Inline cancel key press handler Console.CancelKeyPress += (sender, e) => { if (e.SpecialKey == ConsoleSpecialKey.ControlC) { e.Cancel = true; if (!tokenSource.IsCancellationRequested) { tokenSource.Cancel(); ctrl.ngProgress.IsCanceled = true; progress.Report(ctrl.ngProgress); } } }; try { //Write a menu to the console describing the progress chart Console.ForegroundColor = ConsoleColor.Magenta; Console.WriteLine("\nP: Progress | OK: Successful posters processed | F: Failed |"); Console.WriteLine("Sk: Skipped | T: # of minutes elapsed | R: Remaining assets | ETA\n"); Console.ResetColor(); //Create a separate task for each VHO listed in the config file, and run asyncronously var tskList = new List <Task>(); foreach (var vho in config.Vhos) { tskList.Add(ctrl.BeginProcess(vho.Key, maxImages, config)); } //Used to determine if all vho's processed successfully for source directory cleanup bool allSuccessful = true; try { Task.WaitAll(tskList.ToArray()); } catch (AggregateException aex) { foreach (var ex in aex.Flatten().InnerExceptions) { if (ex is TaskCanceledException || ex is OperationCanceledException) { continue; } else { Trace.TraceError(ex.Message + "\n"); } } } //Create missing poster log try { Console.WriteLine("\nCreating missing poster attachment and sending..."); WriteToMissPosterLog(ctrl.GetAllVodAssets(config).Where(x => string.IsNullOrEmpty(x.PosterSource)), config.EmailTo, config.LogMissPosterDir); Console.WriteLine("Complete\n"); } catch (Exception ex) { if (ex is OperationCanceledException || ex is TaskCanceledException) { Console.WriteLine("Canceled!"); throw; } Console.WriteLine("Failed! {0}\n", ex.Message); Trace.TraceError("Failed to send missing poster log. {0}", ex.Message); } ctrl.Complete(); //If all tasks ran to completion, then begin cleaning up the source folder if (allSuccessful && !maxImages.HasValue && !token.IsCancellationRequested && tskList.All(x => x.Status == TaskStatus.RanToCompletion)) { Console.WriteLine("Cleaning up source directory..."); ctrl.CleanupSource(ref config, token); Console.WriteLine("Complete\n"); } } catch (AggregateException aex) { foreach (var ex in aex.InnerExceptions) { if (ex is TaskCanceledException) { continue; } Trace.TraceError(ex.Message); } } catch (Exception ex) { if (ex is OperationCanceledException || ex is TaskCanceledException) { Trace.TraceInformation("Operation Canceled"); } else { Trace.TraceError("Application Error -- {0}", ex.Message); } } finally { tokenSource.Dispose(); } } //end using ctrl } //end using twtlError } //end using twtl }
/// <summary> /// Begins processing of NGVodPoster image files for the provided VHO /// </summary> /// <param name="vho">Name of the VHO</param> /// <param name="maxImages">Maximum numbere of images to process</param> /// <param name="config">NGVodPoster configuration</param> /// <param name="token">Cancellation Token</param> /// <returns>Task result returns with all assets that do not have posters assigned</returns> internal async Task BeginProcess(string vho, int?maxImages, NGVodPosterConfig config) { if (this.token.IsCancellationRequested) { this.token.ThrowIfCancellationRequested(); } Console.WriteLine("\n\n----Beginning {0}----\n", vho); Trace.TraceInformation(vho); //Get the VHO values from the configuration NGVodVHO ngVho = null; if (!config.Vhos.TryGetValue(vho, out ngVho)) { throw new Exception("Unable to get VHO from config"); } //Get the T-SQL connection string for the IMG front end database string connectionStr = ngVho.IMGConnectString; #if DEBUG Trace.TraceInformation("Connection string for {0} --> {1}", vho, connectionStr); #endif this.token.ThrowIfCancellationRequested(); if (ngProgress.Total != 0) { this.ngProgress.Reset(); } GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect(); bool ranToCompletion = false; if (string.IsNullOrEmpty(this.customAction) || this.customAction.ToUpper().EndsWith("M")) { //Start run task to ensure it ran to completion before attempting cleanup var mainTsk = Task.Factory.StartNew(() => ProcessAsset(config, ngVho.Name, ngVho.PosterDir, connectionStr, this.token)); try { //Allow threads to catch up before changing the console color await Task.Delay(20); Console.ForegroundColor = ConsoleColor.Cyan; this.ngProgress.StopProgress = false; //Wait to finish await mainTsk; } catch (AggregateException aex) { foreach (var ex in aex.InnerExceptions) { Trace.TraceError("Error during processing method. {0}", ex.Message); } } ranToCompletion = mainTsk.Status == TaskStatus.RanToCompletion; if (ngProgress.CompleteCount > config.Vhos.Count && ngProgress.CompleteCount % config.Vhos.Count == 0 && !token.IsCancellationRequested) { await Task.Delay(15000); this.ngProgress.StopProgress = true; this.ngProgress.IsComplete = !this.ngProgress.IsCanceled; } } GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect(); await cleanupDatabase(connectionStr, vho, this.token); //Clean up poster directory based on the vho's active assets if the process had no errors, no max values were specified, //and cancellation was not requested. if ((ranToCompletion || (!string.IsNullOrEmpty(this.customAction) && this.customAction.ToUpper().EndsWith("D"))) && !maxImages.HasValue && !token.IsCancellationRequested) { try { await Task.Factory.StartNew(() => cleanupDestination(GetVODAssets(connectionStr, maxImages, vho, config.SourceDir, ngVho.PosterDir, this.token).Select(x => x.PosterDest).ToList(), ngVho.PosterDir, config.MaxThreads)); } catch (Exception ex) { Trace.TraceError("Error while cleaning up destination directory. {0}", ex.Message); } } }
/// <summary> /// Performs a cleanup of the source directory by removing any posters that are not assigned to any active /// assets across all VHO's. Only perform this operation if all processes have performed successfully. /// </summary> /// <param name="usedSourceFiles">All active assets in all vho's.</param> /// <param name="srcPath">Full path to the source directory.</param> internal void CleanupSource(ref NGVodPosterConfig config, CancellationToken cancelToken) { Trace.TraceInformation("Starting cleanup source..."); //Get all unused assets. //var unusedSrcFiles = Directory.EnumerateFiles(srcPath).Except(usedSourceFiles, new SourceFileComparer()); var unusedSrcFiles = Directory.EnumerateFiles(config.SourceDir) .Except(NGVodPosterDataController.GetAllPosterSourceMaps(config, cancelToken).Select(x => x.Item2).ToList(), new SourceFileComparer()); Trace.TraceInformation("Archiving unused posters. Count => {0}", unusedSrcFiles.Count()); var archiveDir = Path.Combine(config.SourceDir, "Archive"); //create an archive directory to store the newly cleaned up files if (!Directory.Exists(archiveDir)) { Directory.CreateDirectory(archiveDir); } Parallel.ForEach(unusedSrcFiles, (delFile) => { this.token.ThrowIfCancellationRequested(); //if the file exists and the filer is older than the amount of surpassed running days, copy to the archive and delete it if (File.Exists(delFile) && File.GetLastWriteTime(delFile).Date < DateTime.Now.Date.AddDays(-ngProgress.Time.Elapsed.Days)) { var archFileName = Path.Combine(archiveDir, Path.GetFileName(delFile)); try { #if DEBUG Trace.TraceInformation("Copying {0} to {1} and deleting {0}", Path.GetFileName(delFile), Path.GetFileName(archFileName)); #else File.Copy(delFile, archFileName, true); File.Delete(delFile); #endif } catch (Exception ex) { Trace.TraceError("Error while cleaning up source directory file {0}. {1}", delFile, ex.Message); try { if (File.Exists(archFileName) && !File.Exists(delFile)) { #if DEBUG Trace.TraceInformation("Copying {0} back to {1} and overwriting", Path.GetFileName(archFileName), Path.GetFileName(delFile)); #else File.Copy(archFileName, delFile, true); #endif } } catch (Exception exe) { Trace.TraceError("Failed to restore {0} from archive. {1}", archFileName, exe.Message); } } } }); //Cleanup the archive directory of any files older than 90 days Trace.TraceInformation("Cleaning old posters from archive directory..."); Parallel.ForEach(Directory.EnumerateFiles(archiveDir), (archiveFile) => { this.token.ThrowIfCancellationRequested(); if (File.Exists(archiveFile)) { FileInfo fInfo = new FileInfo(archiveFile); if (fInfo.LastWriteTime.CompareTo(DateTime.Now.AddDays(90)) <= 0) { try { #if DEBUG Trace.TraceInformation("Deleting {0}", fInfo.FullName); #else fInfo.Delete(); #endif } catch (Exception ex) { Trace.TraceError("Failed to delete {0} while cleaning up source directory. {1}", fInfo.FullName, ex.Message); } } } }); }
/// <summary> /// Begins matching asset id's to source image files, and if found it will resize and save the file to the destination /// </summary> /// <param name="VAssets">List of the VHO's VOD assets</param> /// <param name="config">The NGVodPosterConfig configuration</param> /// <param name="posterDest">The UNC path to the poster destination directory</param> /// <param name="vhoName">Name of the VHO that is being processed</param> /// <param name="dictSrcPath">Dictionary of asset id to source file mapping</param> /// <param name="indexFile">Index file path</param> /// <param name="cancelToken">Cancellation token</param> private void ProcessAsset(NGVodPosterConfig config, string vhoName, string destDir, string connectionString, CancellationToken cancelToken) { Trace.TraceInformation("INFO({0}): Processing VOD Asset Posters...", vhoName.ToUpper()); //Begin processing each VOD asset obtained from the database asyncronously ParallelOptions po = new ParallelOptions() { MaxDegreeOfParallelism = config.MaxThreads, CancellationToken = cancelToken }; //Add to vod asset count to progress total Interlocked.Add(ref this.ngProgress.Total, GetVODAssets(connectionString, MaxImages, vhoName, config.SourceDir, destDir, cancelToken).Count()); Interlocked.Add(ref this.ngProgress.TotalNoPoster, GetVODAssets(connectionString, MaxImages, vhoName, config.SourceDir, destDir, cancelToken) .Where(x => string.IsNullOrEmpty(x.PosterSource)).Count()); #if DEBUG Trace.TraceInformation("Number of missing posters in {0} ==> {1}", vhoName, GetVODAssets(connectionString, MaxImages, vhoName, config.SourceDir, destDir, cancelToken) .Where(x => string.IsNullOrEmpty(x.PosterSource)).Count()); Trace.TraceInformation("New missing poster total for all vhos ==> {0}", this.ngProgress.TotalNoPoster); #endif try { using (var dataController = new NGVodPosterDataController(connectionString)) { dataController.BeginTransaction(); Parallel.ForEach <VODAsset>(GetVODAssets(config.Vhos[vhoName].IMGConnectString, MaxImages, vhoName, config.SourceDir, destDir, cancelToken) .OrderByDescending(x => !string.IsNullOrEmpty(x.PosterSource)).ThenByDescending(x => x.AssetId), po, (va) => { try { po.CancellationToken.ThrowIfCancellationRequested(); //Make the poster source the full path if (!string.IsNullOrEmpty(va.PosterSource) && !va.PosterSource.Contains(config.SourceDir)) { va.PosterSource = Path.Combine(config.SourceDir, va.PosterSource); } Task insertTsk = null; //Get poster source if it doesn't already exist, or if it doesn't contain the PID/PAID values of the asset if (string.IsNullOrEmpty(va.PosterSource) || !File.Exists(va.PosterSource) || !va.PosterSource.ToLower().Contains(va.PID.ToLower()) || !va.PosterSource.ToLower().Contains(va.PAID.ToLower())) { try { va.PosterSource = GetSourceImagePath(va.PID, va.PAID, config.SourceDir); } catch (Exception ex) { //Increment progress failed value if error was thrown Interlocked.Increment(ref this.ngProgress.Failed); Trace.TraceError("Error getting source image path. {0}", ex.Message); return; } if (string.IsNullOrEmpty(va.PosterSource)) { //If file exists on destination server but does not on the source server, then delete it on the destination. //This prevents incorrect posters from being displayed if the asset ID is changed by the VOD provider. if (File.Exists(va.PosterDest)) { #if DEBUG Trace.TraceInformation("Deleting {0} in {1} because there is no source that matches it", Path.GetFileName(va.PosterDest), vhoName); #else File.Delete(va.PosterDest); #endif Interlocked.Increment(ref this.ngProgress.Deleted); } Interlocked.Increment(ref this.ngProgress.Failed); return; } else { //Insert new source map into database, or update the existing one insertTsk = dataController.InsertAssetAsync(va, cancelToken); } } //Skip if destination file is newer than or the same as the source file if (File.Exists(va.PosterDest) && (File.GetLastWriteTime(va.PosterDest).CompareTo(File.GetLastWriteTime(va.PosterSource)) >= 0 && File.GetCreationTime(va.PosterDest).CompareTo(File.GetCreationTime(va.PosterSource)) >= 0)) { #if DEBUG if (_onlyNew) { Trace.TraceInformation("Skipped: {0} - {1} - {2}", va.AssetId, va.PID, va.PAID); } #endif Interlocked.Increment(ref ngProgress.Skipped); //It is possible that the poster successfully processed but it was never inserted into the database, this will handle this scenario try { if (null != insertTsk) { insertTsk.Wait(); } } catch (Exception ex) { Trace.TraceError("Insert task failed for skipped asset {0} in {1}. {2}", va.AssetId, vhoName, ex.Message); } return; } try { //Resize and save the image to the destination var res = ProcessImage(va, config.ImgHeight, config.ImgWidth, vhoName, po.CancellationToken); switch (res) { case 0: Interlocked.Increment(ref this.ngProgress.Success); break; case 1: Interlocked.Increment(ref this.ngProgress.Skipped); break; default: Interlocked.Increment(ref this.ngProgress.Failed); break; } } catch (OperationCanceledException) { Interlocked.Increment(ref this.ngProgress.Skipped); } catch (Exception ex) { Interlocked.Increment(ref this.ngProgress.Failed); Trace.TraceError("Error processing image for {0} in {1}. {2}", va.AssetId, vhoName, ex.Message); try { sendToBadPosterDir(va.PosterSource); } catch (Exception ex2) { Trace.TraceError("Failed to send to bad poster directory in source folder. {0}", ex2.Message); } //Retry to get the poster source after sending the bad one to the BadImage directory on the source server try { Trace.TraceInformation("Attempting to re-process image for {0} in {1}.", va.AssetId, vhoName); va.PosterSource = GetSourceImagePath(va.PID, va.PID, config.SourceDir); if (!string.IsNullOrEmpty(va.PosterSource)) { var res2 = ProcessImage(va, config.ImgHeight, config.ImgWidth, vhoName, po.CancellationToken); if (res2 == 0) { Interlocked.Decrement(ref this.ngProgress.Failed); Interlocked.Increment(ref this.ngProgress.Success); } else if (null != insertTsk) { //If there is an existing insert task, and the image was not able to be processed //then we have to remove the newly created asset from the database insertTsk.Wait(); insertTsk = dataController.DeleteVodAssetAsync(va, cancelToken); } } } catch (Exception ex3) { Trace.TraceError("Failed re-processing image for {0} in {1}. {2}", va.AssetId, vhoName, ex3.Message); } } //Wait on the async task (will not cause deadlock since it is a console application) if (null != insertTsk) { try { insertTsk.Wait(cancelToken); } catch (AggregateException aex) { foreach (var ex in aex.InnerExceptions) { if (ex is OperationCanceledException || ex is TaskCanceledException) { continue; } Trace.TraceError("Insert task failed in {0} for {1}. {2}", vhoName, va.ToString(), ex.Message); } } finally { insertTsk.Dispose(); } } po.CancellationToken.ThrowIfCancellationRequested(); } catch (OperationCanceledException) { throw; } }); //end parallel foreach statement try { dataController.CommitTransaction(); } catch { dataController.RollbackTransaction(); } }//end using dataConnection } catch (OperationCanceledException) { this.ngProgress.StopProgress = true; this.ngProgress.IsCanceled = true; } }