private async Task cleanupDatabase(string connectionString, string vhoName, CancellationToken cancelToken) { Trace.TraceInformation("Cleaning up database in {0}", vhoName); using (var dataController = new NGVodPosterDataController(connectionString)) { if (!cancelToken.IsCancellationRequested) { try { await dataController.CleanupSourceMapTable(cancelToken); Trace.TraceInformation("Successfully cleaned database in {0}", vhoName); } catch (AggregateException aex) { foreach (var ex in aex.InnerExceptions) { if (ex is OperationCanceledException || ex is TaskCanceledException) { throw ex; } Trace.TraceError("Failed to clean database in {0}. {1}", vhoName, ex.Message); } } } } }
/// <summary> /// Performs a cleanup of the source directory by removing any posters that are not assigned to any active /// assets across all VHO's. Only perform this operation if all processes have performed successfully. /// </summary> /// <param name="usedSourceFiles">All active assets in all vho's.</param> /// <param name="srcPath">Full path to the source directory.</param> internal void CleanupSource(ref NGVodPosterConfig config, CancellationToken cancelToken) { Trace.TraceInformation("Starting cleanup source..."); //Get all unused assets. //var unusedSrcFiles = Directory.EnumerateFiles(srcPath).Except(usedSourceFiles, new SourceFileComparer()); var unusedSrcFiles = Directory.EnumerateFiles(config.SourceDir) .Except(NGVodPosterDataController.GetAllPosterSourceMaps(config, cancelToken).Select(x => x.Item2).ToList(), new SourceFileComparer()); Trace.TraceInformation("Archiving unused posters. Count => {0}", unusedSrcFiles.Count()); var archiveDir = Path.Combine(config.SourceDir, "Archive"); //create an archive directory to store the newly cleaned up files if (!Directory.Exists(archiveDir)) { Directory.CreateDirectory(archiveDir); } Parallel.ForEach(unusedSrcFiles, (delFile) => { this.token.ThrowIfCancellationRequested(); //if the file exists and the filer is older than the amount of surpassed running days, copy to the archive and delete it if (File.Exists(delFile) && File.GetLastWriteTime(delFile).Date < DateTime.Now.Date.AddDays(-ngProgress.Time.Elapsed.Days)) { var archFileName = Path.Combine(archiveDir, Path.GetFileName(delFile)); try { #if DEBUG Trace.TraceInformation("Copying {0} to {1} and deleting {0}", Path.GetFileName(delFile), Path.GetFileName(archFileName)); #else File.Copy(delFile, archFileName, true); File.Delete(delFile); #endif } catch (Exception ex) { Trace.TraceError("Error while cleaning up source directory file {0}. {1}", delFile, ex.Message); try { if (File.Exists(archFileName) && !File.Exists(delFile)) { #if DEBUG Trace.TraceInformation("Copying {0} back to {1} and overwriting", Path.GetFileName(archFileName), Path.GetFileName(delFile)); #else File.Copy(archFileName, delFile, true); #endif } } catch (Exception exe) { Trace.TraceError("Failed to restore {0} from archive. {1}", archFileName, exe.Message); } } } }); //Cleanup the archive directory of any files older than 90 days Trace.TraceInformation("Cleaning old posters from archive directory..."); Parallel.ForEach(Directory.EnumerateFiles(archiveDir), (archiveFile) => { this.token.ThrowIfCancellationRequested(); if (File.Exists(archiveFile)) { FileInfo fInfo = new FileInfo(archiveFile); if (fInfo.LastWriteTime.CompareTo(DateTime.Now.AddDays(90)) <= 0) { try { #if DEBUG Trace.TraceInformation("Deleting {0}", fInfo.FullName); #else fInfo.Delete(); #endif } catch (Exception ex) { Trace.TraceError("Failed to delete {0} while cleaning up source directory. {1}", fInfo.FullName, ex.Message); } } } }); }
/// <summary> /// Begins matching asset id's to source image files, and if found it will resize and save the file to the destination /// </summary> /// <param name="VAssets">List of the VHO's VOD assets</param> /// <param name="config">The NGVodPosterConfig configuration</param> /// <param name="posterDest">The UNC path to the poster destination directory</param> /// <param name="vhoName">Name of the VHO that is being processed</param> /// <param name="dictSrcPath">Dictionary of asset id to source file mapping</param> /// <param name="indexFile">Index file path</param> /// <param name="cancelToken">Cancellation token</param> private void ProcessAsset(NGVodPosterConfig config, string vhoName, string destDir, string connectionString, CancellationToken cancelToken) { Trace.TraceInformation("INFO({0}): Processing VOD Asset Posters...", vhoName.ToUpper()); //Begin processing each VOD asset obtained from the database asyncronously ParallelOptions po = new ParallelOptions() { MaxDegreeOfParallelism = config.MaxThreads, CancellationToken = cancelToken }; //Add to vod asset count to progress total Interlocked.Add(ref this.ngProgress.Total, GetVODAssets(connectionString, MaxImages, vhoName, config.SourceDir, destDir, cancelToken).Count()); Interlocked.Add(ref this.ngProgress.TotalNoPoster, GetVODAssets(connectionString, MaxImages, vhoName, config.SourceDir, destDir, cancelToken) .Where(x => string.IsNullOrEmpty(x.PosterSource)).Count()); #if DEBUG Trace.TraceInformation("Number of missing posters in {0} ==> {1}", vhoName, GetVODAssets(connectionString, MaxImages, vhoName, config.SourceDir, destDir, cancelToken) .Where(x => string.IsNullOrEmpty(x.PosterSource)).Count()); Trace.TraceInformation("New missing poster total for all vhos ==> {0}", this.ngProgress.TotalNoPoster); #endif try { using (var dataController = new NGVodPosterDataController(connectionString)) { dataController.BeginTransaction(); Parallel.ForEach <VODAsset>(GetVODAssets(config.Vhos[vhoName].IMGConnectString, MaxImages, vhoName, config.SourceDir, destDir, cancelToken) .OrderByDescending(x => !string.IsNullOrEmpty(x.PosterSource)).ThenByDescending(x => x.AssetId), po, (va) => { try { po.CancellationToken.ThrowIfCancellationRequested(); //Make the poster source the full path if (!string.IsNullOrEmpty(va.PosterSource) && !va.PosterSource.Contains(config.SourceDir)) { va.PosterSource = Path.Combine(config.SourceDir, va.PosterSource); } Task insertTsk = null; //Get poster source if it doesn't already exist, or if it doesn't contain the PID/PAID values of the asset if (string.IsNullOrEmpty(va.PosterSource) || !File.Exists(va.PosterSource) || !va.PosterSource.ToLower().Contains(va.PID.ToLower()) || !va.PosterSource.ToLower().Contains(va.PAID.ToLower())) { try { va.PosterSource = GetSourceImagePath(va.PID, va.PAID, config.SourceDir); } catch (Exception ex) { //Increment progress failed value if error was thrown Interlocked.Increment(ref this.ngProgress.Failed); Trace.TraceError("Error getting source image path. {0}", ex.Message); return; } if (string.IsNullOrEmpty(va.PosterSource)) { //If file exists on destination server but does not on the source server, then delete it on the destination. //This prevents incorrect posters from being displayed if the asset ID is changed by the VOD provider. if (File.Exists(va.PosterDest)) { #if DEBUG Trace.TraceInformation("Deleting {0} in {1} because there is no source that matches it", Path.GetFileName(va.PosterDest), vhoName); #else File.Delete(va.PosterDest); #endif Interlocked.Increment(ref this.ngProgress.Deleted); } Interlocked.Increment(ref this.ngProgress.Failed); return; } else { //Insert new source map into database, or update the existing one insertTsk = dataController.InsertAssetAsync(va, cancelToken); } } //Skip if destination file is newer than or the same as the source file if (File.Exists(va.PosterDest) && (File.GetLastWriteTime(va.PosterDest).CompareTo(File.GetLastWriteTime(va.PosterSource)) >= 0 && File.GetCreationTime(va.PosterDest).CompareTo(File.GetCreationTime(va.PosterSource)) >= 0)) { #if DEBUG if (_onlyNew) { Trace.TraceInformation("Skipped: {0} - {1} - {2}", va.AssetId, va.PID, va.PAID); } #endif Interlocked.Increment(ref ngProgress.Skipped); //It is possible that the poster successfully processed but it was never inserted into the database, this will handle this scenario try { if (null != insertTsk) { insertTsk.Wait(); } } catch (Exception ex) { Trace.TraceError("Insert task failed for skipped asset {0} in {1}. {2}", va.AssetId, vhoName, ex.Message); } return; } try { //Resize and save the image to the destination var res = ProcessImage(va, config.ImgHeight, config.ImgWidth, vhoName, po.CancellationToken); switch (res) { case 0: Interlocked.Increment(ref this.ngProgress.Success); break; case 1: Interlocked.Increment(ref this.ngProgress.Skipped); break; default: Interlocked.Increment(ref this.ngProgress.Failed); break; } } catch (OperationCanceledException) { Interlocked.Increment(ref this.ngProgress.Skipped); } catch (Exception ex) { Interlocked.Increment(ref this.ngProgress.Failed); Trace.TraceError("Error processing image for {0} in {1}. {2}", va.AssetId, vhoName, ex.Message); try { sendToBadPosterDir(va.PosterSource); } catch (Exception ex2) { Trace.TraceError("Failed to send to bad poster directory in source folder. {0}", ex2.Message); } //Retry to get the poster source after sending the bad one to the BadImage directory on the source server try { Trace.TraceInformation("Attempting to re-process image for {0} in {1}.", va.AssetId, vhoName); va.PosterSource = GetSourceImagePath(va.PID, va.PID, config.SourceDir); if (!string.IsNullOrEmpty(va.PosterSource)) { var res2 = ProcessImage(va, config.ImgHeight, config.ImgWidth, vhoName, po.CancellationToken); if (res2 == 0) { Interlocked.Decrement(ref this.ngProgress.Failed); Interlocked.Increment(ref this.ngProgress.Success); } else if (null != insertTsk) { //If there is an existing insert task, and the image was not able to be processed //then we have to remove the newly created asset from the database insertTsk.Wait(); insertTsk = dataController.DeleteVodAssetAsync(va, cancelToken); } } } catch (Exception ex3) { Trace.TraceError("Failed re-processing image for {0} in {1}. {2}", va.AssetId, vhoName, ex3.Message); } } //Wait on the async task (will not cause deadlock since it is a console application) if (null != insertTsk) { try { insertTsk.Wait(cancelToken); } catch (AggregateException aex) { foreach (var ex in aex.InnerExceptions) { if (ex is OperationCanceledException || ex is TaskCanceledException) { continue; } Trace.TraceError("Insert task failed in {0} for {1}. {2}", vhoName, va.ToString(), ex.Message); } } finally { insertTsk.Dispose(); } } po.CancellationToken.ThrowIfCancellationRequested(); } catch (OperationCanceledException) { throw; } }); //end parallel foreach statement try { dataController.CommitTransaction(); } catch { dataController.RollbackTransaction(); } }//end using dataConnection } catch (OperationCanceledException) { this.ngProgress.StopProgress = true; this.ngProgress.IsCanceled = true; } }