protected override void Add(StreamType type, int index, EntryKey entry, string value, string unit)
{
if (value == null)
{
return;
}
bool isValid = true;
foreach (var character in value)
{
isValid = SafeXmlWriter.IsLegalXmlChar(character);
if (!isValid)
{
break;
}
}
if (isValid)
{
base.Add(type, index, entry, value, unit);
}
}
private static void ProcessMedia(List<string> mediaLst) {
using(new ActivityTraceContext(ts, "ProcessMedia")) {
DateTime startedOn = DateTime.Now;
isProcessing = true;
var searchOption = switches != eSwitches.ExcludeSubFolders ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly;
long processedBytes = 0, totalBytes;
sentACReqs = 0;
failedACReqs = 0;
ts.TraceInformation("Getting files to process");
#if(PublicRelease)
var files = GetFiles(mediaLst, searchOption, processExtensions, out totalBytes);
#else
var files = GetFiles(mediaLst, searchOption, processExtensions, out totalBytes);
#endif
if(files.Count == 0) {
ts.TraceInformation("No files to process found");
Console.WriteLine("No files to process");
}
ts.TraceInformation("Creating BlockConsumer cache");
var container = CreateContainer(blockCount, blockSize);
FileEnvironment e;
bool fatalExceptionThrown = false;
var retryOnIOExCount = retriesOnIOException;
for(int i = 0;i < files.Count;i++) {
e = new FileEnvironment(appVersion, container, files[i], startedOn, files.Count, i, totalBytes, processedBytes);
try {
ProcessMediaFile(e);
retryOnIOExCount = retriesOnIOException;
} catch(FatalException ex) {
ts.TraceData(TraceEventType.Error, 0, "Fatal error while processing the file.", ex);
e.AddException("Fatal error while processing the file.", ex);
fatalExceptionThrown = true;
} catch(IOException ex) {
if(--retryOnIOExCount != 0) {
ts.TraceData(TraceEventType.Error, 0, "File processing failed with an IOException. Retrying...", ex);
ColoredWriteLine(ConsoleColor.Red, "File processing failed with an IOException. Retrying...", true);
i--;
} else {
ts.TraceData(TraceEventType.Error, 0, "File processing failed " + retriesOnIOException + " times with an IOException. Skipping...", ex);
ColoredWriteLine(ConsoleColor.Red, "File processing failed " + retriesOnIOException + " times with an IOException. Skipping...", true);
retryOnIOExCount = retriesOnIOException;
}
} catch(Exception ex) {
ts.TraceData(TraceEventType.Error, 0, "Unhandled error while processing the file.", ex);
e.AddException("Unhandled error while processing the file.", ex);
}
container.Reset();
processedBytes += e.File.Length;
if(e.Exceptions.Count != 0) {
ts.TraceInformation("Processing finished with errors. Logging");
try {
var exElem = e.Exceptions.ToXElement(true);
string exPath = Path.Combine(CurrentDirectory, "Error");
string exFileName = "Err " + DateTime.Now.ToString("yyyyMMdd HH.mm.ss.ffff") + ".xml";
if(!Directory.Exists(exPath)) Directory.CreateDirectory(exPath);
using(var writer = new SafeXmlWriter(Path.Combine(exPath, exFileName), Encoding.Unicode)) exElem.Save(writer);
} catch(Exception) { }
#if(HasACreq)
if(username != null && password != null && (switches & eSwitches.NoErrorReporting) == 0) {
try {
var exElem = e.Exceptions.ToXElement(false);
MemoryStream memStream = new MemoryStream();
using(var writer = new SafeXmlWriter(memStream, Encoding.ASCII)) exElem.Save(writer);
anidb.CommitError(host, timeout * 1000, "avdumplib", appVersion.Build, username.ToLower(), password, memStream.ToArray());
} catch(Exception) { }
}
#endif
}
if(fatalExceptionThrown) Environment.Exit(1);
if((switches & eSwitches.PauseWhenFileDone) != 0) {
Console.WriteLine("Press any alpha-numeric key to continue");
Pause();
}
}
isProcessing = false;
if((switches & eSwitches.PrintTotalTimeUsed) != 0) Console.WriteLine("Total time elapsed: " + (DateTime.Now - startedOn).TotalSeconds + "s");
}
}
private static void WriteLogs(FileEnvironment e, Dictionary<string, IBlockConsumer> blockConsumers) {
using(new ActivityTraceContext(ts, "ExecuteBlockConsumers")) {
bool acreqFile;
var p = CreateInfoProvider(e, blockConsumers, out acreqFile);
var detExt = p[StreamType.General, 0, EntryKey.Extension] != null ? p[StreamType.General, 0, EntryKey.Extension].Value.ToLower() : null;
var ext = e.File.Extension.Length <= 1 ? "" : e.File.Extension.Substring(1).ToLower();
if(!string.IsNullOrEmpty(extFixPath) && !string.IsNullOrEmpty(detExt) && !ext.Equals(detExt)) {
ts.TraceInformation("Logging: Extension Fix");
try {
if(!detExt.Contains(' ')) {
e.File.MoveTo(Path.ChangeExtension(e.File.FullName, detExt));
AppendAllText(extFixPath, ext + " => " + detExt + " " + e.File.FullName + Environment.NewLine, "Couldn't update extension fix file");
ext = detExt;
} else {
AppendAllText(extFixPath, "FAILED: " + ext + " => " + detExt + " " + e.File.FullName + Environment.NewLine, "Couldn't update extension fix file");
}
} catch(Exception ex) {
AppendAllText(extFixPath, "FAILED: " + ext + " => " + detExt + " " + e.File.FullName + Environment.NewLine, "Couldn't update extension fix file");
}
}
#region Generate/Write Logs
string log = "";
try {
if((switches & (eSwitches.CreqXmlFormat | eSwitches.NewCreqXmlFormat | eSwitches.MediaInfoXMLOutPut | eSwitches.MediaInfoOutPut)) != 0) {
log = e.File.FullName + Environment.NewLine;
}
if((switches & eSwitches.CreqXmlFormat) != 0) {
ts.TraceInformation("Logging: CreqXmlFormat");
var tw = new StringWriter();
Info.CreateAVDumpLog(p).Save(new SafeXmlWriter(tw, lowerCaseElements: true));
log += tw.ToString();
}
if((switches & eSwitches.NewCreqXmlFormat) != 0) {
ts.TraceInformation("Logging: NewCreqXmlFormat");
var tw = new StringWriter();
Info.CreateNewAVDumpLog(p).Save(new SafeXmlWriter(tw));
log += tw.ToString();
}
if((switches & eSwitches.MediaInfoXMLOutPut) != 0) {
ts.TraceInformation("Logging: MediaInfoXMLOutPut");
var tw = new StringWriter();
Info.CreateMediaInfoXMLLog(e.File.FullName, blockConsumers.Values).Save(new SafeXmlWriter(tw));
log += tw.ToString();
}
if((switches & eSwitches.MediaInfoOutPut) != 0) {
ts.TraceInformation("Logging: MediaInfoOutPut");
log += Info.CreateMediaInfoDump(e.File.FullName);
}
if((switches & eSwitches.TxtFormat) != 0) {
ts.TraceInformation("Logging: TxtFormat");
log += Info.CreateTxtLog(e.File.FullName, p);
}
if((switches & eSwitches.HashOutput) != 0) {
ts.TraceInformation("Logging: HashOutput");
log += Info.CreateHashLog(e.File.FullName, p);
}
if(logPath != null && !String.IsNullOrEmpty(log)) AppendAllText(logPath, log + Environment.NewLine + Environment.NewLine, "Couldn't update logfile");
} catch(FatalException ex) {
throw;
} catch(Exception ex) {
e.AddException("Error while generating info report", ex);
}
#endregion
if(!string.IsNullOrEmpty(log)) Console.WriteLine(log);
Console.WriteLine();
#if(HasACreq)
if(acreqFile && (dumpableExtensions.Contains(ext) || dumpableExtensions.Contains(detExt) || (switches & eSwitches.DumpAllExtensions) != 0) && !(string.IsNullOrEmpty(username) || string.IsNullOrEmpty(password))) {
MemoryStream stream = new MemoryStream();
byte[] creqBytes;
using(var writer = new SafeXmlWriter(stream, new UTF8Encoding(), Formatting.None, lowerCaseElements: true)) {
var xmlDoc = Info.CreateAVDumpLog(p);
if(xmlDoc["file"]["md4"] != null) xmlDoc["file"].RemoveChild(xmlDoc["file"]["md4"]);
if(xmlDoc["file"]["tiger"] != null) xmlDoc["file"].RemoveChild(xmlDoc["file"]["tiger"]);
xmlDoc.Save(writer);
creqBytes = new byte[stream.Length - 38];
stream.Position = 38;
stream.Read(creqBytes, 0, (int)stream.Length - 38);
}
anidb.QueryACReq(new ACReq(host, timeout * 1000, "avdumplib", appVersion.Build, username.ToLower(), password, creqBytes), new AniDB.QueryTag(e));
}
#endif
if(!string.IsNullOrEmpty(extDiffListPath) && !ext.Equals(detExt)) {
AppendAllText(extDiffListPath, ext + " => " + (detExt == null ? "unknown" : detExt) + " " + e.File.FullName + Environment.NewLine, "Couldn't update extension diff file");
}
#region DoneLog Stream Writing
if(doneListPath != null && (username == null || password == null) && e.Exceptions.Count == 0) {
ts.TraceInformation("Logging: DoneLog");
AppendAllText(doneListPath, e.File.FullName + Environment.NewLine, "Couldn't update donelist file");
int index = doneListContent.BinarySearch(e.File.FullName);
if(index < 0) doneListContent.Insert(~index, e.File.FullName);
}
#endregion
#region Ed2kLog Stream Writing
Ed2k ed2k = null;
if(blockConsumers.ContainsKey("ED2K")) ed2k = (Ed2k)((HashCalculator)blockConsumers["ED2K"]).HashObj;
if(ed2k != null) {
byte[] blueEd2kHash = ed2k.BlueHash; //Handle Ed2k screwup
byte[] redEd2kHash = ed2k.Hash;
if(ed2kListPath != null) {
ts.TraceInformation("Logging: Ed2kLog");
string ed2kStr = "ed2k://|file|" + e.File.Name + "|" + e.File.Length + "|" + BaseConverter.ToString(ed2k.Hash) + "|/";
if(!ed2k.BlueIsRed) {
ed2kStr += "*" + "ed2k://|file|" + e.File.Name + "|" + e.File.Length + "|" + BaseConverter.ToString(ed2k.BlueHash) + "|/";
}
AppendAllText(ed2kListPath, ed2kStr + Environment.NewLine, "Couldn't update ed2k list file");
}
}
#endregion
#region CRC Mismatch
if(crcMismatchListPath != null && blockConsumers.ContainsKey("CRC")) {
ts.TraceInformation("Logging: CRC Mismatch");
string crcHash = BaseConverter.ToString(((HashCalculator)blockConsumers["CRC"]).HashObj.Hash);
if(!e.File.Name.ToLower().Contains(crcHash.ToLower())) {
AppendAllText(crcMismatchListPath, crcHash + " " + e.File.FullName + Environment.NewLine, "Couldn't update crcerr list file");
ColoredWriteLine(ConsoleColor.Yellow, "Filename doesn't contain the calculated CRC (" + crcHash + ")");
}
}
#endregion
#region HashLog Stream Writing
if(hashListPath != null) {
ts.TraceInformation("Logging: HashLog");
string formattedStr = hashLogStyle;
foreach(HashCalculator hashExecute in blockConsumers.Values.Where(blockConsumer => { return blockConsumer is HashCalculator; })) {
//if(hashExecute.HashObj is Ed2k) {
// string ed2kStr = "ed2k://|file|" + e.File.Name + "|" + e.File.Length + "|" + BaseConverter.ToString(hashExecute.HashObj.Hash) + "|/";
// if(!((Ed2k)hashExecute.HashObj).BlueIsRed) {
// ed2kStr += "*" + "ed2k://|file|" + e.File.Name + "|" + e.File.Length + "|" + BaseConverter.ToString(((Ed2k)hashExecute.HashObj).BlueHash) + "|/";
// }
// formattedStr = formattedStr.Replace("$" + hashExecute.Name + "$", ed2kStr);
//} else {
formattedStr = formattedStr.Replace("$" + hashExecute.Name + "$", BaseConverter.ToString(hashExecute.HashObj.Hash));
//}
}
AppendAllText(hashListPath, formattedStr + Environment.NewLine, "Couldn't update hashlist file");
}
#endregion
}
}
private static void GlobalExceptionHandler(object sender, UnhandledExceptionEventArgs e) {
string path = Path.Combine(CurrentDirectory, "Error");
string fileName = "Err " + DateTime.Now.ToString("yyyyMMdd HH.mm.ss.ffff") + ".xml";
ExceptionXElement exElem;
try {
if(!Directory.Exists(path)) Directory.CreateDirectory(path);
exElem = new ExceptionXElement((Exception)e.ExceptionObject, false);
using(var writer = new SafeXmlWriter(Path.Combine(path, fileName), Encoding.ASCII)) {
exElem.Save(XmlWriter.Create(writer, new XmlWriterSettings { OmitXmlDeclaration = true }));
}
} catch(UnauthorizedAccessException ex) {
exElem = new ExceptionXElement(ex, false);
} catch(Exception ex) {
exElem = new ExceptionXElement(new Exception("Couldn't save Exception", ex), false);
}
#if(HasACreq)
if(username != null && password != null && (switches & eSwitches.NoErrorReporting) == 0) {
try {
MemoryStream memStream = new MemoryStream();
using(var writer = new SafeXmlWriter(memStream, Encoding.ASCII)) exElem.Save(writer);
anidb.CommitError(host, timeout * 1000, "avdumplib", appVersion.Build, username.ToLower(), password, memStream.ToArray());
} catch(Exception) { }
}
#endif
}