private async Task cleanupDatabase(string connectionString, string vhoName, CancellationToken cancelToken)
{
Trace.TraceInformation("Cleaning up database in {0}", vhoName);
using (var dataController = new NGVodPosterDataController(connectionString))
{
if (!cancelToken.IsCancellationRequested)
{
try
{
await dataController.CleanupSourceMapTable(cancelToken);
Trace.TraceInformation("Successfully cleaned database in {0}", vhoName);
}
catch (AggregateException aex)
{
foreach (var ex in aex.InnerExceptions)
{
if (ex is OperationCanceledException || ex is TaskCanceledException)
{
throw ex;
}
Trace.TraceError("Failed to clean database in {0}. {1}", vhoName, ex.Message);
}
}
}
}
}
/// <summary>
/// Performs a cleanup of the source directory by removing any posters that are not assigned to any active
/// assets across all VHO's. Only perform this operation if all processes have performed successfully.
/// </summary>
/// <param name="usedSourceFiles">All active assets in all vho's.</param>
/// <param name="srcPath">Full path to the source directory.</param>
internal void CleanupSource(ref NGVodPosterConfig config, CancellationToken cancelToken)
{
Trace.TraceInformation("Starting cleanup source...");
//Get all unused assets.
//var unusedSrcFiles = Directory.EnumerateFiles(srcPath).Except(usedSourceFiles, new SourceFileComparer());
var unusedSrcFiles = Directory.EnumerateFiles(config.SourceDir)
.Except(NGVodPosterDataController.GetAllPosterSourceMaps(config, cancelToken).Select(x => x.Item2).ToList(), new SourceFileComparer());
Trace.TraceInformation("Archiving unused posters. Count => {0}", unusedSrcFiles.Count());
var archiveDir = Path.Combine(config.SourceDir, "Archive");
//create an archive directory to store the newly cleaned up files
if (!Directory.Exists(archiveDir))
{
Directory.CreateDirectory(archiveDir);
}
Parallel.ForEach(unusedSrcFiles, (delFile) =>
{
this.token.ThrowIfCancellationRequested();
//if the file exists and the filer is older than the amount of surpassed running days, copy to the archive and delete it
if (File.Exists(delFile) && File.GetLastWriteTime(delFile).Date < DateTime.Now.Date.AddDays(-ngProgress.Time.Elapsed.Days))
{
var archFileName = Path.Combine(archiveDir, Path.GetFileName(delFile));
try
{
#if DEBUG
Trace.TraceInformation("Copying {0} to {1} and deleting {0}", Path.GetFileName(delFile), Path.GetFileName(archFileName));
#else
File.Copy(delFile, archFileName, true);
File.Delete(delFile);
#endif
}
catch (Exception ex)
{
Trace.TraceError("Error while cleaning up source directory file {0}. {1}", delFile, ex.Message);
try
{
if (File.Exists(archFileName) && !File.Exists(delFile))
{
#if DEBUG
Trace.TraceInformation("Copying {0} back to {1} and overwriting", Path.GetFileName(archFileName), Path.GetFileName(delFile));
#else
File.Copy(archFileName, delFile, true);
#endif
}
}
catch (Exception exe)
{
Trace.TraceError("Failed to restore {0} from archive. {1}", archFileName, exe.Message);
}
}
}
});
//Cleanup the archive directory of any files older than 90 days
Trace.TraceInformation("Cleaning old posters from archive directory...");
Parallel.ForEach(Directory.EnumerateFiles(archiveDir), (archiveFile) =>
{
this.token.ThrowIfCancellationRequested();
if (File.Exists(archiveFile))
{
FileInfo fInfo = new FileInfo(archiveFile);
if (fInfo.LastWriteTime.CompareTo(DateTime.Now.AddDays(90)) <= 0)
{
try
{
#if DEBUG
Trace.TraceInformation("Deleting {0}", fInfo.FullName);
#else
fInfo.Delete();
#endif
}
catch (Exception ex)
{
Trace.TraceError("Failed to delete {0} while cleaning up source directory. {1}", fInfo.FullName, ex.Message);
}
}
}
});
}
/// <summary>
/// Begins matching asset id's to source image files, and if found it will resize and save the file to the destination
/// </summary>
/// <param name="VAssets">List of the VHO's VOD assets</param>
/// <param name="config">The NGVodPosterConfig configuration</param>
/// <param name="posterDest">The UNC path to the poster destination directory</param>
/// <param name="vhoName">Name of the VHO that is being processed</param>
/// <param name="dictSrcPath">Dictionary of asset id to source file mapping</param>
/// <param name="indexFile">Index file path</param>
/// <param name="cancelToken">Cancellation token</param>
private void ProcessAsset(NGVodPosterConfig config, string vhoName, string destDir, string connectionString, CancellationToken cancelToken)
{
Trace.TraceInformation("INFO({0}): Processing VOD Asset Posters...", vhoName.ToUpper());
//Begin processing each VOD asset obtained from the database asyncronously
ParallelOptions po = new ParallelOptions()
{
MaxDegreeOfParallelism = config.MaxThreads, CancellationToken = cancelToken
};
//Add to vod asset count to progress total
Interlocked.Add(ref this.ngProgress.Total, GetVODAssets(connectionString, MaxImages, vhoName, config.SourceDir, destDir, cancelToken).Count());
Interlocked.Add(ref this.ngProgress.TotalNoPoster, GetVODAssets(connectionString, MaxImages, vhoName, config.SourceDir, destDir, cancelToken)
.Where(x => string.IsNullOrEmpty(x.PosterSource)).Count());
#if DEBUG
Trace.TraceInformation("Number of missing posters in {0} ==> {1}", vhoName, GetVODAssets(connectionString, MaxImages, vhoName, config.SourceDir, destDir, cancelToken)
.Where(x => string.IsNullOrEmpty(x.PosterSource)).Count());
Trace.TraceInformation("New missing poster total for all vhos ==> {0}", this.ngProgress.TotalNoPoster);
#endif
try
{
using (var dataController = new NGVodPosterDataController(connectionString))
{
dataController.BeginTransaction();
Parallel.ForEach <VODAsset>(GetVODAssets(config.Vhos[vhoName].IMGConnectString, MaxImages, vhoName, config.SourceDir, destDir, cancelToken)
.OrderByDescending(x => !string.IsNullOrEmpty(x.PosterSource)).ThenByDescending(x => x.AssetId), po, (va) =>
{
try
{
po.CancellationToken.ThrowIfCancellationRequested();
//Make the poster source the full path
if (!string.IsNullOrEmpty(va.PosterSource) && !va.PosterSource.Contains(config.SourceDir))
{
va.PosterSource = Path.Combine(config.SourceDir, va.PosterSource);
}
Task insertTsk = null;
//Get poster source if it doesn't already exist, or if it doesn't contain the PID/PAID values of the asset
if (string.IsNullOrEmpty(va.PosterSource) || !File.Exists(va.PosterSource) || !va.PosterSource.ToLower().Contains(va.PID.ToLower()) || !va.PosterSource.ToLower().Contains(va.PAID.ToLower()))
{
try
{
va.PosterSource = GetSourceImagePath(va.PID, va.PAID, config.SourceDir);
}
catch (Exception ex)
{
//Increment progress failed value if error was thrown
Interlocked.Increment(ref this.ngProgress.Failed);
Trace.TraceError("Error getting source image path. {0}", ex.Message);
return;
}
if (string.IsNullOrEmpty(va.PosterSource))
{
//If file exists on destination server but does not on the source server, then delete it on the destination.
//This prevents incorrect posters from being displayed if the asset ID is changed by the VOD provider.
if (File.Exists(va.PosterDest))
{
#if DEBUG
Trace.TraceInformation("Deleting {0} in {1} because there is no source that matches it", Path.GetFileName(va.PosterDest), vhoName);
#else
File.Delete(va.PosterDest);
#endif
Interlocked.Increment(ref this.ngProgress.Deleted);
}
Interlocked.Increment(ref this.ngProgress.Failed);
return;
}
else
{
//Insert new source map into database, or update the existing one
insertTsk = dataController.InsertAssetAsync(va, cancelToken);
}
}
//Skip if destination file is newer than or the same as the source file
if (File.Exists(va.PosterDest) &&
(File.GetLastWriteTime(va.PosterDest).CompareTo(File.GetLastWriteTime(va.PosterSource)) >= 0 &&
File.GetCreationTime(va.PosterDest).CompareTo(File.GetCreationTime(va.PosterSource)) >= 0))
{
#if DEBUG
if (_onlyNew)
{
Trace.TraceInformation("Skipped: {0} - {1} - {2}", va.AssetId, va.PID, va.PAID);
}
#endif
Interlocked.Increment(ref ngProgress.Skipped);
//It is possible that the poster successfully processed but it was never inserted into the database, this will handle this scenario
try
{
if (null != insertTsk)
{
insertTsk.Wait();
}
}
catch (Exception ex)
{
Trace.TraceError("Insert task failed for skipped asset {0} in {1}. {2}", va.AssetId, vhoName, ex.Message);
}
return;
}
try
{
//Resize and save the image to the destination
var res = ProcessImage(va, config.ImgHeight, config.ImgWidth, vhoName, po.CancellationToken);
switch (res)
{
case 0:
Interlocked.Increment(ref this.ngProgress.Success);
break;
case 1:
Interlocked.Increment(ref this.ngProgress.Skipped);
break;
default:
Interlocked.Increment(ref this.ngProgress.Failed);
break;
}
}
catch (OperationCanceledException)
{
Interlocked.Increment(ref this.ngProgress.Skipped);
}
catch (Exception ex)
{
Interlocked.Increment(ref this.ngProgress.Failed);
Trace.TraceError("Error processing image for {0} in {1}. {2}", va.AssetId, vhoName, ex.Message);
try
{
sendToBadPosterDir(va.PosterSource);
}
catch (Exception ex2)
{
Trace.TraceError("Failed to send to bad poster directory in source folder. {0}", ex2.Message);
}
//Retry to get the poster source after sending the bad one to the BadImage directory on the source server
try
{
Trace.TraceInformation("Attempting to re-process image for {0} in {1}.", va.AssetId, vhoName);
va.PosterSource = GetSourceImagePath(va.PID, va.PID, config.SourceDir);
if (!string.IsNullOrEmpty(va.PosterSource))
{
var res2 = ProcessImage(va, config.ImgHeight, config.ImgWidth, vhoName, po.CancellationToken);
if (res2 == 0)
{
Interlocked.Decrement(ref this.ngProgress.Failed);
Interlocked.Increment(ref this.ngProgress.Success);
}
else if (null != insertTsk)
{
//If there is an existing insert task, and the image was not able to be processed
//then we have to remove the newly created asset from the database
insertTsk.Wait();
insertTsk = dataController.DeleteVodAssetAsync(va, cancelToken);
}
}
}
catch (Exception ex3)
{
Trace.TraceError("Failed re-processing image for {0} in {1}. {2}", va.AssetId, vhoName, ex3.Message);
}
}
//Wait on the async task (will not cause deadlock since it is a console application)
if (null != insertTsk)
{
try
{
insertTsk.Wait(cancelToken);
}
catch (AggregateException aex)
{
foreach (var ex in aex.InnerExceptions)
{
if (ex is OperationCanceledException || ex is TaskCanceledException)
{
continue;
}
Trace.TraceError("Insert task failed in {0} for {1}. {2}", vhoName, va.ToString(), ex.Message);
}
}
finally
{
insertTsk.Dispose();
}
}
po.CancellationToken.ThrowIfCancellationRequested();
}
catch (OperationCanceledException)
{
throw;
}
}); //end parallel foreach statement
try
{
dataController.CommitTransaction();
}
catch
{
dataController.RollbackTransaction();
}
}//end using dataConnection
}
catch (OperationCanceledException)
{
this.ngProgress.StopProgress = true;
this.ngProgress.IsCanceled = true;
}
}