private static void WriteLogs(FileEnvironment e, Dictionary<string, IBlockConsumer> blockConsumers) {
using(new ActivityTraceContext(ts, "ExecuteBlockConsumers")) {
bool acreqFile;
var p = CreateInfoProvider(e, blockConsumers, out acreqFile);
var detExt = p[StreamType.General, 0, EntryKey.Extension] != null ? p[StreamType.General, 0, EntryKey.Extension].Value.ToLower() : null;
var ext = e.File.Extension.Length <= 1 ? "" : e.File.Extension.Substring(1).ToLower();
if(!string.IsNullOrEmpty(extFixPath) && !string.IsNullOrEmpty(detExt) && !ext.Equals(detExt)) {
ts.TraceInformation("Logging: Extension Fix");
try {
if(!detExt.Contains(' ')) {
e.File.MoveTo(Path.ChangeExtension(e.File.FullName, detExt));
AppendAllText(extFixPath, ext + " => " + detExt + " " + e.File.FullName + Environment.NewLine, "Couldn't update extension fix file");
ext = detExt;
} else {
AppendAllText(extFixPath, "FAILED: " + ext + " => " + detExt + " " + e.File.FullName + Environment.NewLine, "Couldn't update extension fix file");
}
} catch(Exception ex) {
AppendAllText(extFixPath, "FAILED: " + ext + " => " + detExt + " " + e.File.FullName + Environment.NewLine, "Couldn't update extension fix file");
}
}
#region Generate/Write Logs
string log = "";
try {
if((switches & (eSwitches.CreqXmlFormat | eSwitches.NewCreqXmlFormat | eSwitches.MediaInfoXMLOutPut | eSwitches.MediaInfoOutPut)) != 0) {
log = e.File.FullName + Environment.NewLine;
}
if((switches & eSwitches.CreqXmlFormat) != 0) {
ts.TraceInformation("Logging: CreqXmlFormat");
var tw = new StringWriter();
Info.CreateAVDumpLog(p).Save(new SafeXmlWriter(tw, lowerCaseElements: true));
log += tw.ToString();
}
if((switches & eSwitches.NewCreqXmlFormat) != 0) {
ts.TraceInformation("Logging: NewCreqXmlFormat");
var tw = new StringWriter();
Info.CreateNewAVDumpLog(p).Save(new SafeXmlWriter(tw));
log += tw.ToString();
}
if((switches & eSwitches.MediaInfoXMLOutPut) != 0) {
ts.TraceInformation("Logging: MediaInfoXMLOutPut");
var tw = new StringWriter();
Info.CreateMediaInfoXMLLog(e.File.FullName, blockConsumers.Values).Save(new SafeXmlWriter(tw));
log += tw.ToString();
}
if((switches & eSwitches.MediaInfoOutPut) != 0) {
ts.TraceInformation("Logging: MediaInfoOutPut");
log += Info.CreateMediaInfoDump(e.File.FullName);
}
if((switches & eSwitches.TxtFormat) != 0) {
ts.TraceInformation("Logging: TxtFormat");
log += Info.CreateTxtLog(e.File.FullName, p);
}
if((switches & eSwitches.HashOutput) != 0) {
ts.TraceInformation("Logging: HashOutput");
log += Info.CreateHashLog(e.File.FullName, p);
}
if(logPath != null && !String.IsNullOrEmpty(log)) AppendAllText(logPath, log + Environment.NewLine + Environment.NewLine, "Couldn't update logfile");
} catch(FatalException ex) {
throw;
} catch(Exception ex) {
e.AddException("Error while generating info report", ex);
}
#endregion
if(!string.IsNullOrEmpty(log)) Console.WriteLine(log);
Console.WriteLine();
#if(HasACreq)
if(acreqFile && (dumpableExtensions.Contains(ext) || dumpableExtensions.Contains(detExt) || (switches & eSwitches.DumpAllExtensions) != 0) && !(string.IsNullOrEmpty(username) || string.IsNullOrEmpty(password))) {
MemoryStream stream = new MemoryStream();
byte[] creqBytes;
using(var writer = new SafeXmlWriter(stream, new UTF8Encoding(), Formatting.None, lowerCaseElements: true)) {
var xmlDoc = Info.CreateAVDumpLog(p);
if(xmlDoc["file"]["md4"] != null) xmlDoc["file"].RemoveChild(xmlDoc["file"]["md4"]);
if(xmlDoc["file"]["tiger"] != null) xmlDoc["file"].RemoveChild(xmlDoc["file"]["tiger"]);
xmlDoc.Save(writer);
creqBytes = new byte[stream.Length - 38];
stream.Position = 38;
stream.Read(creqBytes, 0, (int)stream.Length - 38);
}
anidb.QueryACReq(new ACReq(host, timeout * 1000, "avdumplib", appVersion.Build, username.ToLower(), password, creqBytes), new AniDB.QueryTag(e));
}
#endif
if(!string.IsNullOrEmpty(extDiffListPath) && !ext.Equals(detExt)) {
AppendAllText(extDiffListPath, ext + " => " + (detExt == null ? "unknown" : detExt) + " " + e.File.FullName + Environment.NewLine, "Couldn't update extension diff file");
}
#region DoneLog Stream Writing
if(doneListPath != null && (username == null || password == null) && e.Exceptions.Count == 0) {
ts.TraceInformation("Logging: DoneLog");
AppendAllText(doneListPath, e.File.FullName + Environment.NewLine, "Couldn't update donelist file");
int index = doneListContent.BinarySearch(e.File.FullName);
if(index < 0) doneListContent.Insert(~index, e.File.FullName);
}
#endregion
#region Ed2kLog Stream Writing
Ed2k ed2k = null;
if(blockConsumers.ContainsKey("ED2K")) ed2k = (Ed2k)((HashCalculator)blockConsumers["ED2K"]).HashObj;
if(ed2k != null) {
byte[] blueEd2kHash = ed2k.BlueHash; //Handle Ed2k screwup
byte[] redEd2kHash = ed2k.Hash;
if(ed2kListPath != null) {
ts.TraceInformation("Logging: Ed2kLog");
string ed2kStr = "ed2k://|file|" + e.File.Name + "|" + e.File.Length + "|" + BaseConverter.ToString(ed2k.Hash) + "|/";
if(!ed2k.BlueIsRed) {
ed2kStr += "*" + "ed2k://|file|" + e.File.Name + "|" + e.File.Length + "|" + BaseConverter.ToString(ed2k.BlueHash) + "|/";
}
AppendAllText(ed2kListPath, ed2kStr + Environment.NewLine, "Couldn't update ed2k list file");
}
}
#endregion
#region CRC Mismatch
if(crcMismatchListPath != null && blockConsumers.ContainsKey("CRC")) {
ts.TraceInformation("Logging: CRC Mismatch");
string crcHash = BaseConverter.ToString(((HashCalculator)blockConsumers["CRC"]).HashObj.Hash);
if(!e.File.Name.ToLower().Contains(crcHash.ToLower())) {
AppendAllText(crcMismatchListPath, crcHash + " " + e.File.FullName + Environment.NewLine, "Couldn't update crcerr list file");
ColoredWriteLine(ConsoleColor.Yellow, "Filename doesn't contain the calculated CRC (" + crcHash + ")");
}
}
#endregion
#region HashLog Stream Writing
if(hashListPath != null) {
ts.TraceInformation("Logging: HashLog");
string formattedStr = hashLogStyle;
foreach(HashCalculator hashExecute in blockConsumers.Values.Where(blockConsumer => { return blockConsumer is HashCalculator; })) {
//if(hashExecute.HashObj is Ed2k) {
// string ed2kStr = "ed2k://|file|" + e.File.Name + "|" + e.File.Length + "|" + BaseConverter.ToString(hashExecute.HashObj.Hash) + "|/";
// if(!((Ed2k)hashExecute.HashObj).BlueIsRed) {
// ed2kStr += "*" + "ed2k://|file|" + e.File.Name + "|" + e.File.Length + "|" + BaseConverter.ToString(((Ed2k)hashExecute.HashObj).BlueHash) + "|/";
// }
// formattedStr = formattedStr.Replace("$" + hashExecute.Name + "$", ed2kStr);
//} else {
formattedStr = formattedStr.Replace("$" + hashExecute.Name + "$", BaseConverter.ToString(hashExecute.HashObj.Hash));
//}
}
AppendAllText(hashListPath, formattedStr + Environment.NewLine, "Couldn't update hashlist file");
}
#endregion
}
}
private static void ExecuteBlockConsumers(FileEnvironment e, out Dictionary<string, IBlockConsumer> blockConsumers, Stream altStream = null) {
using(new ActivityTraceContext(ts, "ExecuteBlockConsumers")) {
Stream fileStream;
try {
fileStream = altStream != null ? altStream : File.Open(e.File.FullName, FileMode.Open, FileAccess.Read, FileShare.Read);
ts.TraceInformation("Successfully opened file");
} catch(Exception ex) {
ts.TraceData(TraceEventType.Error, 0, "Couldn't open file", ex);
Console.WriteLine(ex.Message + "\n");
blockConsumers = null;
return;
}
using(fileStream) {
Boolean isMatroska = MatroskaParser.IsMatroskaFile(fileStream);
Boolean isOgmOgg = OgmOggParser.IsOgmOggFile(fileStream);
BlockConsumerContainer.Progress progress = null;
if((switches & eSwitches.UseAllHashes) != 0 || isMatroska || isOgmOgg) {
ts.TraceInformation("Adding Blockconsumers");
#if(UseAICHHash)
if((switches & (eSwitches.Aich)) != 0) hashContainer.AddHashAlgorithm("AICH");
#endif
if((switches & (eSwitches.Crc)) != 0) e.Container.AddBlockConsumer("CRC");
if((switches & (eSwitches.Ed2k)) != 0) e.Container.AddBlockConsumer("ED2K");
if((switches & (eSwitches.Md4)) != 0) e.Container.AddBlockConsumer("MD4");
if((switches & (eSwitches.Md5)) != 0) e.Container.AddBlockConsumer("MD5");
if((switches & (eSwitches.Sha1)) != 0) e.Container.AddBlockConsumer("SHA1");
if((switches & (eSwitches.Tiger)) != 0) e.Container.AddBlockConsumer("TIGER");
if((switches & (eSwitches.Tth)) != 0) e.Container.AddBlockConsumer("TTH");
//if((switches & (eSwitches.Tth)) != 0) e.Container.AddBlockConsumer("TTH2");
//if((switches & (eSwitches.Tth)) != 0) e.Container.AddBlockConsumer("TTH3");
if(isMatroska) e.Container.AddBlockConsumer("Matroska");
if(isOgmOgg) e.Container.AddBlockConsumer("Ogm/Ogg");
ts.TraceInformation("Consume file");
progress = e.Container.Start(fileStream);
}
if((switches & eSwitches.SupressProgress) == 0) {
try {
DisplayBuffer(e, progress);
//throw new Exception("Just me testing again");
} catch(Exception ex) {
Console.WriteLine();
CursorVisible = true;
e.AddException("Error in DisplayBuffer", ex);
}
}
try {
blockConsumers = e.Container.Join().ToDictionary(b => b.Name);
} catch(Exception ex) {
e.AddException("Error while waiting for BlockConsumers", ex);
blockConsumers = null;
}
}
}
}
private static void ProcessMedia(List<string> mediaLst) {
using(new ActivityTraceContext(ts, "ProcessMedia")) {
DateTime startedOn = DateTime.Now;
isProcessing = true;
var searchOption = switches != eSwitches.ExcludeSubFolders ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly;
long processedBytes = 0, totalBytes;
sentACReqs = 0;
failedACReqs = 0;
ts.TraceInformation("Getting files to process");
#if(PublicRelease)
var files = GetFiles(mediaLst, searchOption, processExtensions, out totalBytes);
#else
var files = GetFiles(mediaLst, searchOption, processExtensions, out totalBytes);
#endif
if(files.Count == 0) {
ts.TraceInformation("No files to process found");
Console.WriteLine("No files to process");
}
ts.TraceInformation("Creating BlockConsumer cache");
var container = CreateContainer(blockCount, blockSize);
FileEnvironment e;
bool fatalExceptionThrown = false;
var retryOnIOExCount = retriesOnIOException;
for(int i = 0;i < files.Count;i++) {
e = new FileEnvironment(appVersion, container, files[i], startedOn, files.Count, i, totalBytes, processedBytes);
try {
ProcessMediaFile(e);
retryOnIOExCount = retriesOnIOException;
} catch(FatalException ex) {
ts.TraceData(TraceEventType.Error, 0, "Fatal error while processing the file.", ex);
e.AddException("Fatal error while processing the file.", ex);
fatalExceptionThrown = true;
} catch(IOException ex) {
if(--retryOnIOExCount != 0) {
ts.TraceData(TraceEventType.Error, 0, "File processing failed with an IOException. Retrying...", ex);
ColoredWriteLine(ConsoleColor.Red, "File processing failed with an IOException. Retrying...", true);
i--;
} else {
ts.TraceData(TraceEventType.Error, 0, "File processing failed " + retriesOnIOException + " times with an IOException. Skipping...", ex);
ColoredWriteLine(ConsoleColor.Red, "File processing failed " + retriesOnIOException + " times with an IOException. Skipping...", true);
retryOnIOExCount = retriesOnIOException;
}
} catch(Exception ex) {
ts.TraceData(TraceEventType.Error, 0, "Unhandled error while processing the file.", ex);
e.AddException("Unhandled error while processing the file.", ex);
}
container.Reset();
processedBytes += e.File.Length;
if(e.Exceptions.Count != 0) {
ts.TraceInformation("Processing finished with errors. Logging");
try {
var exElem = e.Exceptions.ToXElement(true);
string exPath = Path.Combine(CurrentDirectory, "Error");
string exFileName = "Err " + DateTime.Now.ToString("yyyyMMdd HH.mm.ss.ffff") + ".xml";
if(!Directory.Exists(exPath)) Directory.CreateDirectory(exPath);
using(var writer = new SafeXmlWriter(Path.Combine(exPath, exFileName), Encoding.Unicode)) exElem.Save(writer);
} catch(Exception) { }
#if(HasACreq)
if(username != null && password != null && (switches & eSwitches.NoErrorReporting) == 0) {
try {
var exElem = e.Exceptions.ToXElement(false);
MemoryStream memStream = new MemoryStream();
using(var writer = new SafeXmlWriter(memStream, Encoding.ASCII)) exElem.Save(writer);
anidb.CommitError(host, timeout * 1000, "avdumplib", appVersion.Build, username.ToLower(), password, memStream.ToArray());
} catch(Exception) { }
}
#endif
}
if(fatalExceptionThrown) Environment.Exit(1);
if((switches & eSwitches.PauseWhenFileDone) != 0) {
Console.WriteLine("Press any alpha-numeric key to continue");
Pause();
}
}
isProcessing = false;
if((switches & eSwitches.PrintTotalTimeUsed) != 0) Console.WriteLine("Total time elapsed: " + (DateTime.Now - startedOn).TotalSeconds + "s");
}
}
private static void ProcessMediaFile(FileEnvironment e) {
using(new ActivityTraceContext(ts, "ProcessMediaFile")) {
ts.TraceData(TraceEventType.Information, 0, e.File.FullName);
Console.WriteLine("Folder: " + e.File.DirectoryName);
Console.WriteLine("Filename: " + e.File.Name);
if(e.File.Length == 0) {
ts.TraceInformation("Skipping 0byte File");
ColoredWriteLine(ConsoleColor.Red, "Skipping 0byte File", true);
return;
}
ts.TraceInformation("Starting file processing");
var startTime = DateTime.Now;
Dictionary<string, IBlockConsumer> blockConsumers;
ExecuteBlockConsumers(e, out blockConsumers);
if(blockConsumers == null) {
ts.TraceEvent(TraceEventType.Warning, 0, "blockConsumers is NULL, skipping file");
return;
}
if((switches & eSwitches.PrintElapsedHashingTime) != 0) Console.WriteLine("Time elapsed after Hashing: " + (DateTime.Now - startTime).TotalSeconds + "s");
foreach(var blockConsumer in blockConsumers.Values) {
if(blockConsumer.Error != null) {
ts.TraceData(TraceEventType.Error, 0, "BlockConsumer (" + blockConsumer.Name + ") threw an error.", blockConsumer.Error);
e.AddException("BlockConsumer (" + blockConsumer.Name + ") threw an error.", blockConsumer.Error);
}
}
//if(blockConsumers.Values.Any(bc => bc.Error != null)) return;
ts.TraceInformation("Writing Logs");
WriteLogs(e, blockConsumers);
Ed2k ed2k = null;
if(blockConsumers.ContainsKey("ED2K")) ed2k = (Ed2k)((HashCalculator)blockConsumers["ED2K"]).HashObj;
if((switches & eSwitches.OpenInBrowser) != 0 || (switches & eSwitches.PrintAniDBLink) != 0) {
string aniDBLink = "http://anidb.info/perl-bin/animedb.pl?show=file&size=" + e.File.Length + "&hash=" + BaseConverter.ToString(ed2k.Hash);
if((switches & eSwitches.OpenInBrowser) != 0) System.Diagnostics.Process.Start(aniDBLink);
if((switches & eSwitches.PrintAniDBLink) != 0) Console.WriteLine(aniDBLink);
}
if((switches & eSwitches.PrintEd2kLink) != 0) Console.WriteLine("ed2k://|file|" + e.File.Name + "|" + e.File.Length + "|" + BaseConverter.ToString(ed2k.Hash) + "|/");
if((switches & eSwitches.DeleteFileWhenDone) != 0 && e.Exceptions.Count == 0) System.IO.File.Delete(e.File.FullName);
if((switches & eSwitches.PrintTimeUsedPerFile) != 0) Console.WriteLine("Time elapsed for file: " + (DateTime.Now - startTime).TotalSeconds.ToString() + "s");
Console.WriteLine(); Console.WriteLine();
}
}
private static InfoProviderBase CreateInfoProvider(FileEnvironment e, Dictionary<string, IBlockConsumer> blockConsumers, out bool noErrors) { //TODO: Refactor
using(new ActivityTraceContext(ts, "CreateInfoProvider")) {
MatroskaProvider mkvProvider = null;
OgmOggProvider ogmOggProvider = null;
MediaInfoProvider milProvider = null;
HashInfoProvider hashProvider = null;
FileExtensionProvider extProvider = null;
//CompositeInfoProvider p = null;
noErrors = true;
try {
if(blockConsumers.ContainsKey("Matroska")) {
var matroskaFile = ((MatroskaParser)blockConsumers["Matroska"]).MatroskaFileObj;
if(matroskaFile != null) mkvProvider = new MatroskaProvider(matroskaFile);
ts.TraceInformation("Added Matroska Provider");
}
} catch(Exception ex) { e.AddException("Failed to create MatroskaProvider", ex); noErrors = false; }
try {
if(blockConsumers.ContainsKey("Ogm/Ogg")) {
var ogmOggFile = ((OgmOggParser)blockConsumers["Ogm/Ogg"]).OgmOggFileObj;
if(ogmOggFile != null) ogmOggProvider = new OgmOggProvider(ogmOggFile);
ts.TraceInformation("Added Ogm/Ogg Provider");
}
} catch(Exception ex) { e.AddException("Failed to create OgmOggProvider", ex); noErrors = false; }
try {
extProvider = new FileExtensionProvider(e.File.FullName);
ts.TraceInformation("Added FileExtensionProvider Provider");
} catch(Exception ex) { e.AddException("Failed to create FileExtensionProvider", ex); /*noErrors = false;*/ }
try {
milProvider = new MediaInfoProvider(e.File.FullName);
ts.TraceInformation("Added MediaInfoLib Provider");
} catch(FatalException ex) {
throw;
} catch(Exception ex) {
e.AddException("Failed to create MediaInfoProvider", ex); noErrors = false;
}
try {
hashProvider = new HashInfoProvider(blockConsumers.Values.OfType<HashCalculator>());
ts.TraceInformation("Added Hash Provider");
} catch(Exception ex) { e.AddException("Failed to create HashInfoProvider", ex); noErrors = false; }
var providers = new Collection<InfoProviderBase>();
if(mkvProvider != null) providers.Add(mkvProvider);
if(ogmOggProvider != null) providers.Add(ogmOggProvider);
if(extProvider != null) providers.Add(extProvider);
if(milProvider != null) providers.Add(milProvider);
if(hashProvider != null) providers.Add(hashProvider);
return new CompositeInfoProvider(providers);
}
}