private void WriteResponseBody(IGraph graph, string mediaType, IResponseInfo response) { var writer = CreateWriter(mediaType); if (writer is RdfXmlWriter) { Stream buffer = new MemoryStream(); buffer = new UnclosableStream(buffer); using (var textWriter = new StreamWriter(buffer)) { writer.Save(graph, textWriter); } buffer.Seek(0, SeekOrigin.Begin); XmlDocument document = new XmlDocument(); document.Load(buffer); document.InsertAfter( document.CreateProcessingInstruction("xml-stylesheet", "type=\"text/xsl\" href=\"" + DocumentationStylesheet + "\""), document.FirstChild); document.Save(response.Body); } else { using (var textWriter = new StreamWriter(response.Body)) { writer.Save(graph, textWriter); } } }
static void RunTest(IEnumerable <string> files, bool verbose, int threading) { SimisProvider provider; try { provider = new SimisProvider(Path.GetDirectoryName(Application.ExecutablePath) + @"\Resources"); } catch (FileException ex) { Console.WriteLine(ex.ToString()); return; } var totalCount = new TestFormatCount(); var supportedCount = new TestFormatCount(); var formatCounts = new Dictionary <string, TestFormatCount>(); var timeStart = DateTime.Now; Func <SimisJinxFormat, TestFormatCount> GetFormatFor = (simisFormat) => { var formatName = simisFormat.Name; if (!formatCounts.ContainsKey(formatName)) { formatCounts[formatName] = new TestFormatCount() { FormatName = formatName, SortKey = formatName }; } return(formatCounts[formatName]); }; Func <string, ProcessFileResults> ProcessFile = (file) => { if (verbose && (threading > 1)) { lock (formatCounts) { Console.WriteLine(String.Format("[Thread {0}] {1}", Thread.CurrentThread.ManagedThreadId, file)); } } var result = new ProcessFileResults(); var formatCount = new TestFormatCount(); var fileProvider = provider.GetForPath(file); SimisFile newFile = null; Stream readStream = new UnclosableStream(new BufferedInMemoryStream(File.OpenRead(file))); Stream saveStream = new UnclosableStream(new MemoryStream()); { result.Total = true; try { using (var reader = SimisReader.FromStream(readStream, fileProvider)) { var readerJinx = reader as SimisJinxReader; var readerAce = reader as SimisAceReader; if (readerJinx != null) { readerJinx.ReadToken(); if (readerJinx.JinxStreamFormat == null) { return(result); } result.JinxStreamFormat = readerJinx.JinxStreamFormat; } else if (readerAce != null) { if (fileProvider.Formats.FirstOrDefault() == null) { return(result); } result.JinxStreamFormat = fileProvider.Formats.First(); } else { return(result); } } } catch (ReaderException) { return(result); } readStream.Position = 0; } // First, read the file in. try { try { newFile = new SimisFile(readStream, fileProvider); } catch (Exception e) { throw new FileException(file, e); } result.ReadSuccess = true; } catch (FileException ex) { if (verbose) { lock (formatCounts) { Console.WriteLine("Read: " + ex + "\n"); } } return(result); } // Second, write the file out into memory. try { try { newFile.Write(saveStream); } catch (Exception e) { throw new FileException(file, e); } // WriteSuccess is delayed until after the comparison. We won't claim write support without comparison support. } catch (FileException ex) { if (verbose) { lock (formatCounts) { Console.WriteLine("Write: " + ex + "\n"); } } return(result); } // Third, verify that the output is the same as the input. readStream.Seek(0, SeekOrigin.Begin); saveStream.Seek(0, SeekOrigin.Begin); var readReader = new BinaryReader(new SimisTestableStream(readStream), newFile.StreamIsBinary ? ByteEncoding.Encoding : Encoding.Unicode); var saveReader = new BinaryReader(new SimisTestableStream(saveStream), newFile.StreamIsBinary ? ByteEncoding.Encoding : Encoding.Unicode); var isDXTACE = (result.JinxStreamFormat.Extension == "ace") && ((newFile.Ace.Format & 0x10) != 0); var readChars = readReader.ReadChars((int)readReader.BaseStream.Length); var saveChars = saveReader.ReadChars((int)saveReader.BaseStream.Length); var charBytes = newFile.StreamIsBinary ? 1 : 2; var charMin = Math.Min(readChars.Length, saveChars.Length); for (var i = 0; i < charMin; i++) { if (isDXTACE && (i > 168)) { break; } if (readChars[i] != saveChars[i]) { readReader.BaseStream.Position = charBytes * (i + 1); saveReader.BaseStream.Position = charBytes * (i + 1); var readEx = new ReaderException(readReader, newFile.StreamIsBinary, charBytes, ""); var saveEx = new ReaderException(saveReader, newFile.StreamIsBinary, charBytes, ""); if (verbose) { lock (formatCounts) { Console.WriteLine("Compare: " + String.Format(CultureInfo.CurrentCulture, "{0}\n\nFile character {1:N0} does not match: {2:X4} vs {3:X4}.\n\n{4}{5}\n", file, charBytes * i, readChars[i], saveChars[i], readEx.ToString(), saveEx.ToString())); } } return(result); } } if ((result.JinxStreamFormat.Extension == "ace") && ((newFile.Ace.Format & 0x10) != 0)) { // DXT images are a massive pain because it is a lossy compression. saveStream.Seek(0, SeekOrigin.Begin); var saveOutput = new SimisFile(saveStream, fileProvider); Debug.Assert(saveOutput.Ace != null); Debug.Assert(saveOutput.Ace.Format == newFile.Ace.Format); try { if (newFile.Ace.Width != saveOutput.Ace.Width) { throw new InvalidDataException(String.Format(CultureInfo.CurrentCulture, "ACE width expected {0}; got {1}.", newFile.Ace.Width, saveOutput.Ace.Width)); } if (newFile.Ace.Height != saveOutput.Ace.Height) { throw new InvalidDataException(String.Format(CultureInfo.CurrentCulture, "ACE height expected {0}; got {1}.", newFile.Ace.Height, saveOutput.Ace.Height)); } if (newFile.Ace.Unknown7 != saveOutput.Ace.Unknown7) { throw new InvalidDataException(String.Format(CultureInfo.CurrentCulture, "ACE unknown7 expected {0}; got {1}.", newFile.Ace.Unknown7, saveOutput.Ace.Unknown7)); } if (newFile.Ace.Creator != saveOutput.Ace.Creator) { throw new InvalidDataException(String.Format(CultureInfo.CurrentCulture, "ACE creator expected {0}; got {1}.", newFile.Ace.Creator, saveOutput.Ace.Creator)); } var newFileChannels = String.Join(",", newFile.Ace.Channel.Select(c => c.Type.ToString() + ":" + c.Size).ToArray()); var saveFileChannels = String.Join(",", saveOutput.Ace.Channel.Select(c => c.Type.ToString() + ":" + c.Size).ToArray()); if (newFileChannels != saveFileChannels) { throw new InvalidDataException(String.Format(CultureInfo.CurrentCulture, "ACE channels expected {0}; got {1}.", newFileChannels, saveFileChannels)); } if (newFile.Ace.Image.Count != saveOutput.Ace.Image.Count) { throw new InvalidDataException(String.Format(CultureInfo.CurrentCulture, "ACE image count expected {0}; got {1}.", newFile.Ace.Image.Count, saveOutput.Ace.Image.Count)); } var errors = new List <double>(); for (var i = 0; i < newFile.Ace.Image.Count; i++) { if (newFile.Ace.Image[i].Width != saveOutput.Ace.Image[i].Width) { throw new InvalidDataException(String.Format(CultureInfo.CurrentCulture, "ACE image {2} width expected {0}; got {1}.", newFile.Ace.Image[i].Width, saveOutput.Ace.Image[i].Width, i)); } if (newFile.Ace.Image[i].Height != saveOutput.Ace.Image[i].Height) { throw new InvalidDataException(String.Format(CultureInfo.CurrentCulture, "ACE image {2} height expected {0}; got {1}.", newFile.Ace.Image[i].Height, saveOutput.Ace.Image[i].Height, i)); } errors.Add(ImageComparison.GetRootMeanSquareError(newFile.Ace.Image[i].ImageColor, saveOutput.Ace.Image[i].ImageColor, newFile.Ace.Width, newFile.Ace.Height)); errors.Add(ImageComparison.GetRootMeanSquareError(newFile.Ace.Image[i].ImageMask, saveOutput.Ace.Image[i].ImageMask, newFile.Ace.Width, newFile.Ace.Height)); } // Any error over 10.0 is considered a fail. var maxError = 10.0; if (errors.Max() > maxError) { throw new InvalidDataException(String.Format(CultureInfo.CurrentCulture, "Image RMS (root mean square) errors are too high; highest: {2,5:F1} > {0,5:F1}; all: {1}.", maxError, String.Join(", ", errors.Select(e => e.ToString("F1").PadLeft(5)).ToArray()), errors.Max())); } } catch (InvalidDataException ex) { if (verbose) { lock (formatCounts) { Console.WriteLine("Compare: " + String.Format(CultureInfo.CurrentCulture, "{0}\n\n{1}\n", file, ex.Message)); } } return(result); } } else { if (readChars.Length != saveChars.Length) { readReader.BaseStream.Position = charBytes * charMin; saveReader.BaseStream.Position = charBytes * charMin; var readEx = new ReaderException(readReader, newFile.StreamIsBinary, 0, ""); var saveEx = new ReaderException(saveReader, newFile.StreamIsBinary, 0, ""); if (verbose) { lock (formatCounts) { Console.WriteLine("Compare: " + String.Format(CultureInfo.CurrentCulture, "{0}\n\nFile and stream length do not match: {1:N0} vs {2:N0}.\n\n{3}{4}\n", file, readReader.BaseStream.Length, saveReader.BaseStream.Length, readEx.ToString(), saveEx.ToString())); } } return(result); } } // It all worked! result.WriteSuccess = true; return(result); }; if (threading > 1) { var filesEnumerator = files.GetEnumerator(); var filesFinished = false; var threads = new List <Thread>(threading); for (var i = 0; i < threading; i++) { threads.Add(new Thread(() => { var file = ""; var results = new List <ProcessFileResults>(); while (true) { lock (filesEnumerator) { if (filesFinished || !filesEnumerator.MoveNext()) { filesFinished = true; break; } file = filesEnumerator.Current; } results.Add(ProcessFile(file)); } lock (totalCount) { foreach (var result in results) { if (result.Total) { totalCount.Total++; } if (result.ReadSuccess) { totalCount.ReadSuccess++; } if (result.WriteSuccess) { totalCount.WriteSuccess++; } if (result.JinxStreamFormat != null) { var formatCount = GetFormatFor(result.JinxStreamFormat); if (result.Total) { supportedCount.Total++; } if (result.ReadSuccess) { supportedCount.ReadSuccess++; } if (result.WriteSuccess) { supportedCount.WriteSuccess++; } if (result.Total) { formatCount.Total++; } if (result.ReadSuccess) { formatCount.ReadSuccess++; } if (result.WriteSuccess) { formatCount.WriteSuccess++; } } } } })); } foreach (var thread in threads) { thread.Start(); } foreach (var thread in threads) { thread.Join(); } } else { foreach (var file in files) { var result = ProcessFile(file); if (result.Total) { totalCount.Total++; } if (result.ReadSuccess) { totalCount.ReadSuccess++; } if (result.WriteSuccess) { totalCount.WriteSuccess++; } if (result.JinxStreamFormat != null) { var formatCount = GetFormatFor(result.JinxStreamFormat); if (result.Total) { supportedCount.Total++; } if (result.ReadSuccess) { supportedCount.ReadSuccess++; } if (result.WriteSuccess) { supportedCount.WriteSuccess++; } if (result.Total) { formatCount.Total++; } if (result.ReadSuccess) { formatCount.ReadSuccess++; } if (result.WriteSuccess) { formatCount.WriteSuccess++; } } } } supportedCount.FormatName = "(Total supported files of " + totalCount.Total + ")"; supportedCount.SortKey = "ZZZ"; formatCounts[""] = supportedCount; var outFormat = "{0,-40:S} {1,1:S}{2,-7:D} {3,1:S}{4,-7:D} {5,1:S}{6,-7:D}"; Console.WriteLine(String.Format(CultureInfo.CurrentCulture, outFormat, "Format Name", "", "Total", "", "Read", "", "Write")); Console.WriteLine(String.Empty.PadLeft(69, '=')); foreach (var formatCount in formatCounts.OrderBy(kvp => kvp.Value.SortKey).Select(kvp => kvp.Value)) { Console.WriteLine(String.Format(CultureInfo.CurrentCulture, outFormat, formatCount.FormatName, "", formatCount.Total, formatCount.Total == formatCount.ReadSuccess ? "*" : "", formatCount.ReadSuccess, formatCount.Total == formatCount.WriteSuccess ? "*" : formatCount.ReadSuccess == formatCount.WriteSuccess ? "+" : "", formatCount.WriteSuccess)); } }
public SimisTestableStream(Stream baseStream) { UncompressedStream = new UnclosableStream(new MemoryStream()); baseStream = new UnclosableStream(baseStream); baseStream.Seek(0, SeekOrigin.Begin); var start = baseStream.Position; var streamCompressed = false; var binaryReader = new BinaryReader(baseStream, ByteEncoding.Encoding); var binaryWriter = new BinaryWriter(UncompressedStream, ByteEncoding.Encoding); { var sr = new StreamReader(baseStream, true); sr.ReadLine(); if (!(sr.CurrentEncoding is UTF8Encoding)) { binaryReader.Close(); binaryWriter.Close(); binaryReader = new BinaryReader(baseStream, sr.CurrentEncoding); binaryWriter = new BinaryWriter(UncompressedStream, sr.CurrentEncoding); start += sr.CurrentEncoding.GetPreamble().Length; binaryWriter.Write(sr.CurrentEncoding.GetPreamble()); } } baseStream.Position = start; { var signature = new String(binaryReader.ReadChars(8)); if ((signature != "SIMISA@F") && (signature != "SIMISA@@")) { throw new InvalidDataException("Signature '" + signature + "' is invalid."); } streamCompressed = (signature == "SIMISA@F"); binaryWriter.Write("SIMISA@@".ToCharArray()); } if (streamCompressed) { // This is a compressed stream. Read in the uncompressed size and DEFLATE the rest. binaryReader.ReadUInt32(); { var signature = new String(binaryReader.ReadChars(4)); if (signature != "@@@@") { throw new InvalidDataException("Signature '" + signature + "' is invalid."); } } // The stream is technically ZLIB, but we assume the selected ZLIB compression is DEFLATE (though we verify that here just in case). The ZLIB // header for DEFLATE is 0x78 0x9C (apparently). { var zlibHeader = binaryReader.ReadBytes(2); if ((zlibHeader[0] != 0x78) || (zlibHeader[1] != 0x9C)) { throw new InvalidDataException("ZLIB signature is invalid."); } } // BinaryReader -> BufferedInMemoryStream -> DeflateStream -> BinaryReader -> BaseStream. // The BufferedInMemoryStream is needed because DeflateStream only supports reading forwards - no seeking - and we'll potentially be jumping around. binaryReader.Close(); binaryReader = new BinaryReader(new BufferedInMemoryStream(new DeflateStream(baseStream, CompressionMode.Decompress)), ByteEncoding.Encoding); } else { var signature = new String(binaryReader.ReadChars(8)); if (signature != "@@@@@@@@") { throw new InvalidDataException("Signature '" + signature + "' is invalid."); } } binaryWriter.Write("@@@@@@@@".ToCharArray()); var isText = false; { var signature = new String(binaryReader.ReadChars(16)); if (signature.Substring(0, 4) == "\x01\x00\x00\x00") { // Texture/ACE format. isText = false; } else { if (signature.Substring(0, 5) != "JINX0") { throw new InvalidDataException("Signature '" + signature + "' is invalid."); } if (signature.Substring(8, 8) != "______\r\n") { throw new InvalidDataException("Signature '" + signature + "' is invalid."); } isText = (signature[7] == 't'); } binaryWriter.Write(signature.ToCharArray()); } var inWhitespace = true; var inString = false; var stringBuffer = ""; while (binaryReader.BaseStream.Position < binaryReader.BaseStream.Length) { if (isText) { var bite = binaryReader.ReadChar(); if (inString || (bite == '"')) { inString = inString ^ (bite == '"'); if (inString) { if (bite != '"') { stringBuffer += bite; } } else { while ((binaryReader.PeekChar() == '\t') || (binaryReader.PeekChar() == '\n') || (binaryReader.PeekChar() == '\r') || (binaryReader.PeekChar() == ' ')) { binaryReader.ReadChar(); } if (binaryReader.PeekChar() == '+') { binaryReader.ReadChar(); while ((binaryReader.PeekChar() == '\t') || (binaryReader.PeekChar() == '\n') || (binaryReader.PeekChar() == '\r') || (binaryReader.PeekChar() == ' ')) { binaryReader.ReadChar(); } if (binaryReader.PeekChar() != '"') { throw new InvalidDataException("Data is invalid."); } binaryReader.ReadChar(); inString = true; } if (!inString) { var stringChars = stringBuffer.ToCharArray(); if (stringChars.All(c => SimisJinxWriter.SafeTokenCharacters.Contains(c))) { binaryWriter.Write(stringChars); } else { binaryWriter.Write('"'); binaryWriter.Write(stringChars); binaryWriter.Write('"'); } stringBuffer = ""; binaryWriter.Write(' '); } inWhitespace = !inString; } } else { if (inWhitespace && "+-.0123456789aAbBcCdDeEfF".Contains(bite)) { var biteStart = bite; var numberStart = binaryReader.BaseStream.Position; var numberString = ""; var isHex = true; var hasDP = false; var hasExp = false; if ((bite == '+') || (bite == '-')) { numberString += bite; isHex = false; bite = binaryReader.ReadChar(); } var allowed = isHex ? ".0123456789aAbBcCdDeEfF" : ".0123456789eE"; while (allowed.Contains(bite)) { numberString += bite; if (numberString.Length > 8) { isHex = false; } if (".".Contains(bite)) { isHex = false; hasDP = true; allowed = "0123456789eE"; } else if ("eE".Contains(bite)) { hasExp = true; allowed = isHex ? "0123456789aAbBcCdDeEfF" : "0123456789"; } else if ("aAbBcCdDeEfF".Contains(bite)) { allowed = "0123456789aAbBcCdDeEfF"; } bite = binaryReader.ReadChar(); } if (numberString.Length != 8) { isHex = false; } var value = 0d; if ("\t\n\r: )".Contains(bite)) { if (isHex) { var valueH = 0; if (!int.TryParse(numberString, NumberStyles.HexNumber, CultureInfo.InvariantCulture, out valueH)) { numberString = ""; } value = valueH; } else { if (!double.TryParse(numberString, out value)) { numberString = ""; } } } else { numberString = ""; } if (numberString.Length > 0) { if (isHex) { binaryWriter.Write(((uint)value).ToString("X8", CultureInfo.InvariantCulture).ToCharArray()); } else if (!hasDP && !hasExp) { binaryWriter.Write(value.ToString("F0", CultureInfo.InvariantCulture).ToCharArray()); } else if (hasExp) { binaryWriter.Write(value.ToString("0.#####e000", CultureInfo.InvariantCulture).ToCharArray()); } else { binaryWriter.Write(value.ToString("G6", CultureInfo.InvariantCulture).ToCharArray()); } inWhitespace = false; } else { bite = biteStart; binaryReader.BaseStream.Position = numberStart; } } if ((bite == '(') || (bite == ')') || (bite == '+')) { if (!inWhitespace) { binaryWriter.Write(' '); } } var isWhitespace = (bite == '\t') || (bite == '\n') || (bite == '\r') || (bite == ':') || (bite == ' '); if (!isWhitespace || !inWhitespace) { binaryWriter.Write(isWhitespace ? ' ' : bite); } inWhitespace = isWhitespace; if ((bite == '(') || (bite == ')') || (bite == '+')) { binaryWriter.Write(' '); inWhitespace = true; } } } else { binaryWriter.Write(binaryReader.ReadChars(1024)); } } binaryReader.Close(); binaryWriter.Close(); UncompressedStream.Seek(0, SeekOrigin.Begin); }