public static IEnumerable <dynamic> EnumerateDynamicRecords(this CsvReader csvReader) { if (csvReader.Configuration.HasHeaderRecord && csvReader.HeaderRecord == null) { if (!csvReader.Read()) { yield break; } csvReader.ReadHeader(); } var csvHeader = new CsvHeader(csvReader.HeaderRecord .Select((x, i) => csvReader.Configuration.PrepareHeaderForMatch(new PrepareHeaderForMatchArgs(x, i))) .ToArray()); while (csvReader.Read()) { CsvRecord record; try { var values = new object[csvReader.HeaderRecord.Length]; for (var i = 0; i < csvReader.HeaderRecord.Length; i++) { values[i] = csvReader.Parser[i]; } record = new CsvRecord(csvHeader, values); } catch (Exception ex) { var readerException = new ReaderException(csvReader.Context, "An unexpected error occurred.", ex); if (csvReader.Configuration.ReadingExceptionOccurred?.Invoke(new ReadingExceptionOccurredArgs(readerException)) ?? true) { throw readerException; } continue; } yield return(record); } }
internal static IEnumerable <IDictionary <string, object> > EnumerateDictionaryRecords(this CsvReader csvReader) { if (csvReader.Configuration.HasHeaderRecord && csvReader.HeaderRecord == null) { if (!csvReader.Read()) { yield break; } csvReader.ReadHeader(); } var headerRecord = csvReader.HeaderRecord .Select((x, i) => csvReader.Configuration.PrepareHeaderForMatch(new PrepareHeaderForMatchArgs(x, i))) .ToArray(); while (csvReader.Read()) { Dictionary <string, object> record; try { record = new Dictionary <string, object>(headerRecord.Length); for (var i = 0; i < headerRecord.Length; i++) { record[headerRecord[i]] = csvReader.Parser[i]; } } catch (Exception ex) { var readerException = new ReaderException(csvReader.Context, "An unexpected error occurred.", ex); if (csvReader.Configuration.ReadingExceptionOccurred?.Invoke(new ReadingExceptionOccurredArgs(readerException)) ?? true) { throw readerException; } continue; } yield return(record); } }
public static async IAsyncEnumerable <dynamic> EnumerateDynamicRecordsAsync(this CsvReader csvReader) { if (csvReader.Configuration.HasHeaderRecord && csvReader.HeaderRecord == null) { if (!await csvReader.ReadAsync().ConfigureAwait(false)) { yield break; } csvReader.ReadHeader(); } var csvHeader = new CsvHeader(csvReader.HeaderRecord .Select((x, i) => csvReader.Configuration.PrepareHeaderForMatch(new PrepareHeaderForMatchArgs(x, i))) .ToArray()); while (await csvReader.ReadAsync().ConfigureAwait(false)) { CsvRecord record; try { var values = new object[csvReader.HeaderRecord.Length]; Array.Copy(csvReader.Parser.Record, values, csvReader.Parser.Record.Length); record = new CsvRecord(csvHeader, values); } catch (Exception ex) { var readerException = new ReaderException(csvReader.Context, "An unexpected error occurred.", ex); if (csvReader.Configuration.ReadingExceptionOccurred?.Invoke(new ReadingExceptionOccurredArgs(readerException)) ?? true) { throw readerException; } continue; } yield return(record); } }
static void RunTest(IEnumerable <string> files, bool verbose, int threading) { SimisProvider provider; try { provider = new SimisProvider(Path.GetDirectoryName(Application.ExecutablePath) + @"\Resources"); } catch (FileException ex) { Console.WriteLine(ex.ToString()); return; } var totalCount = new TestFormatCount(); var supportedCount = new TestFormatCount(); var formatCounts = new Dictionary <string, TestFormatCount>(); var timeStart = DateTime.Now; Func <SimisJinxFormat, TestFormatCount> GetFormatFor = (simisFormat) => { var formatName = simisFormat.Name; if (!formatCounts.ContainsKey(formatName)) { formatCounts[formatName] = new TestFormatCount() { FormatName = formatName, SortKey = formatName }; } return(formatCounts[formatName]); }; Func <string, ProcessFileResults> ProcessFile = (file) => { if (verbose && (threading > 1)) { lock (formatCounts) { Console.WriteLine(String.Format("[Thread {0}] {1}", Thread.CurrentThread.ManagedThreadId, file)); } } var result = new ProcessFileResults(); var formatCount = new TestFormatCount(); var fileProvider = provider.GetForPath(file); SimisFile newFile = null; Stream readStream = new UnclosableStream(new BufferedInMemoryStream(File.OpenRead(file))); Stream saveStream = new UnclosableStream(new MemoryStream()); { result.Total = true; try { using (var reader = SimisReader.FromStream(readStream, fileProvider)) { var readerJinx = reader as SimisJinxReader; var readerAce = reader as SimisAceReader; if (readerJinx != null) { readerJinx.ReadToken(); if (readerJinx.JinxStreamFormat == null) { return(result); } result.JinxStreamFormat = readerJinx.JinxStreamFormat; } else if (readerAce != null) { if (fileProvider.Formats.FirstOrDefault() == null) { return(result); } result.JinxStreamFormat = fileProvider.Formats.First(); } else { return(result); } } } catch (ReaderException) { return(result); } readStream.Position = 0; } // First, read the file in. try { try { newFile = new SimisFile(readStream, fileProvider); } catch (Exception e) { throw new FileException(file, e); } result.ReadSuccess = true; } catch (FileException ex) { if (verbose) { lock (formatCounts) { Console.WriteLine("Read: " + ex + "\n"); } } return(result); } // Second, write the file out into memory. try { try { newFile.Write(saveStream); } catch (Exception e) { throw new FileException(file, e); } // WriteSuccess is delayed until after the comparison. We won't claim write support without comparison support. } catch (FileException ex) { if (verbose) { lock (formatCounts) { Console.WriteLine("Write: " + ex + "\n"); } } return(result); } // Third, verify that the output is the same as the input. readStream.Seek(0, SeekOrigin.Begin); saveStream.Seek(0, SeekOrigin.Begin); var readReader = new BinaryReader(new SimisTestableStream(readStream), newFile.StreamIsBinary ? ByteEncoding.Encoding : Encoding.Unicode); var saveReader = new BinaryReader(new SimisTestableStream(saveStream), newFile.StreamIsBinary ? ByteEncoding.Encoding : Encoding.Unicode); var isDXTACE = (result.JinxStreamFormat.Extension == "ace") && ((newFile.Ace.Format & 0x10) != 0); var readChars = readReader.ReadChars((int)readReader.BaseStream.Length); var saveChars = saveReader.ReadChars((int)saveReader.BaseStream.Length); var charBytes = newFile.StreamIsBinary ? 1 : 2; var charMin = Math.Min(readChars.Length, saveChars.Length); for (var i = 0; i < charMin; i++) { if (isDXTACE && (i > 168)) { break; } if (readChars[i] != saveChars[i]) { readReader.BaseStream.Position = charBytes * (i + 1); saveReader.BaseStream.Position = charBytes * (i + 1); var readEx = new ReaderException(readReader, newFile.StreamIsBinary, charBytes, ""); var saveEx = new ReaderException(saveReader, newFile.StreamIsBinary, charBytes, ""); if (verbose) { lock (formatCounts) { Console.WriteLine("Compare: " + String.Format(CultureInfo.CurrentCulture, "{0}\n\nFile character {1:N0} does not match: {2:X4} vs {3:X4}.\n\n{4}{5}\n", file, charBytes * i, readChars[i], saveChars[i], readEx.ToString(), saveEx.ToString())); } } return(result); } } if ((result.JinxStreamFormat.Extension == "ace") && ((newFile.Ace.Format & 0x10) != 0)) { // DXT images are a massive pain because it is a lossy compression. saveStream.Seek(0, SeekOrigin.Begin); var saveOutput = new SimisFile(saveStream, fileProvider); Debug.Assert(saveOutput.Ace != null); Debug.Assert(saveOutput.Ace.Format == newFile.Ace.Format); try { if (newFile.Ace.Width != saveOutput.Ace.Width) { throw new InvalidDataException(String.Format(CultureInfo.CurrentCulture, "ACE width expected {0}; got {1}.", newFile.Ace.Width, saveOutput.Ace.Width)); } if (newFile.Ace.Height != saveOutput.Ace.Height) { throw new InvalidDataException(String.Format(CultureInfo.CurrentCulture, "ACE height expected {0}; got {1}.", newFile.Ace.Height, saveOutput.Ace.Height)); } if (newFile.Ace.Unknown7 != saveOutput.Ace.Unknown7) { throw new InvalidDataException(String.Format(CultureInfo.CurrentCulture, "ACE unknown7 expected {0}; got {1}.", newFile.Ace.Unknown7, saveOutput.Ace.Unknown7)); } if (newFile.Ace.Creator != saveOutput.Ace.Creator) { throw new InvalidDataException(String.Format(CultureInfo.CurrentCulture, "ACE creator expected {0}; got {1}.", newFile.Ace.Creator, saveOutput.Ace.Creator)); } var newFileChannels = String.Join(",", newFile.Ace.Channel.Select(c => c.Type.ToString() + ":" + c.Size).ToArray()); var saveFileChannels = String.Join(",", saveOutput.Ace.Channel.Select(c => c.Type.ToString() + ":" + c.Size).ToArray()); if (newFileChannels != saveFileChannels) { throw new InvalidDataException(String.Format(CultureInfo.CurrentCulture, "ACE channels expected {0}; got {1}.", newFileChannels, saveFileChannels)); } if (newFile.Ace.Image.Count != saveOutput.Ace.Image.Count) { throw new InvalidDataException(String.Format(CultureInfo.CurrentCulture, "ACE image count expected {0}; got {1}.", newFile.Ace.Image.Count, saveOutput.Ace.Image.Count)); } var errors = new List <double>(); for (var i = 0; i < newFile.Ace.Image.Count; i++) { if (newFile.Ace.Image[i].Width != saveOutput.Ace.Image[i].Width) { throw new InvalidDataException(String.Format(CultureInfo.CurrentCulture, "ACE image {2} width expected {0}; got {1}.", newFile.Ace.Image[i].Width, saveOutput.Ace.Image[i].Width, i)); } if (newFile.Ace.Image[i].Height != saveOutput.Ace.Image[i].Height) { throw new InvalidDataException(String.Format(CultureInfo.CurrentCulture, "ACE image {2} height expected {0}; got {1}.", newFile.Ace.Image[i].Height, saveOutput.Ace.Image[i].Height, i)); } errors.Add(ImageComparison.GetRootMeanSquareError(newFile.Ace.Image[i].ImageColor, saveOutput.Ace.Image[i].ImageColor, newFile.Ace.Width, newFile.Ace.Height)); errors.Add(ImageComparison.GetRootMeanSquareError(newFile.Ace.Image[i].ImageMask, saveOutput.Ace.Image[i].ImageMask, newFile.Ace.Width, newFile.Ace.Height)); } // Any error over 10.0 is considered a fail. var maxError = 10.0; if (errors.Max() > maxError) { throw new InvalidDataException(String.Format(CultureInfo.CurrentCulture, "Image RMS (root mean square) errors are too high; highest: {2,5:F1} > {0,5:F1}; all: {1}.", maxError, String.Join(", ", errors.Select(e => e.ToString("F1").PadLeft(5)).ToArray()), errors.Max())); } } catch (InvalidDataException ex) { if (verbose) { lock (formatCounts) { Console.WriteLine("Compare: " + String.Format(CultureInfo.CurrentCulture, "{0}\n\n{1}\n", file, ex.Message)); } } return(result); } } else { if (readChars.Length != saveChars.Length) { readReader.BaseStream.Position = charBytes * charMin; saveReader.BaseStream.Position = charBytes * charMin; var readEx = new ReaderException(readReader, newFile.StreamIsBinary, 0, ""); var saveEx = new ReaderException(saveReader, newFile.StreamIsBinary, 0, ""); if (verbose) { lock (formatCounts) { Console.WriteLine("Compare: " + String.Format(CultureInfo.CurrentCulture, "{0}\n\nFile and stream length do not match: {1:N0} vs {2:N0}.\n\n{3}{4}\n", file, readReader.BaseStream.Length, saveReader.BaseStream.Length, readEx.ToString(), saveEx.ToString())); } } return(result); } } // It all worked! result.WriteSuccess = true; return(result); }; if (threading > 1) { var filesEnumerator = files.GetEnumerator(); var filesFinished = false; var threads = new List <Thread>(threading); for (var i = 0; i < threading; i++) { threads.Add(new Thread(() => { var file = ""; var results = new List <ProcessFileResults>(); while (true) { lock (filesEnumerator) { if (filesFinished || !filesEnumerator.MoveNext()) { filesFinished = true; break; } file = filesEnumerator.Current; } results.Add(ProcessFile(file)); } lock (totalCount) { foreach (var result in results) { if (result.Total) { totalCount.Total++; } if (result.ReadSuccess) { totalCount.ReadSuccess++; } if (result.WriteSuccess) { totalCount.WriteSuccess++; } if (result.JinxStreamFormat != null) { var formatCount = GetFormatFor(result.JinxStreamFormat); if (result.Total) { supportedCount.Total++; } if (result.ReadSuccess) { supportedCount.ReadSuccess++; } if (result.WriteSuccess) { supportedCount.WriteSuccess++; } if (result.Total) { formatCount.Total++; } if (result.ReadSuccess) { formatCount.ReadSuccess++; } if (result.WriteSuccess) { formatCount.WriteSuccess++; } } } } })); } foreach (var thread in threads) { thread.Start(); } foreach (var thread in threads) { thread.Join(); } } else { foreach (var file in files) { var result = ProcessFile(file); if (result.Total) { totalCount.Total++; } if (result.ReadSuccess) { totalCount.ReadSuccess++; } if (result.WriteSuccess) { totalCount.WriteSuccess++; } if (result.JinxStreamFormat != null) { var formatCount = GetFormatFor(result.JinxStreamFormat); if (result.Total) { supportedCount.Total++; } if (result.ReadSuccess) { supportedCount.ReadSuccess++; } if (result.WriteSuccess) { supportedCount.WriteSuccess++; } if (result.Total) { formatCount.Total++; } if (result.ReadSuccess) { formatCount.ReadSuccess++; } if (result.WriteSuccess) { formatCount.WriteSuccess++; } } } } supportedCount.FormatName = "(Total supported files of " + totalCount.Total + ")"; supportedCount.SortKey = "ZZZ"; formatCounts[""] = supportedCount; var outFormat = "{0,-40:S} {1,1:S}{2,-7:D} {3,1:S}{4,-7:D} {5,1:S}{6,-7:D}"; Console.WriteLine(String.Format(CultureInfo.CurrentCulture, outFormat, "Format Name", "", "Total", "", "Read", "", "Write")); Console.WriteLine(String.Empty.PadLeft(69, '=')); foreach (var formatCount in formatCounts.OrderBy(kvp => kvp.Value.SortKey).Select(kvp => kvp.Value)) { Console.WriteLine(String.Format(CultureInfo.CurrentCulture, outFormat, formatCount.FormatName, "", formatCount.Total, formatCount.Total == formatCount.ReadSuccess ? "*" : "", formatCount.ReadSuccess, formatCount.Total == formatCount.WriteSuccess ? "*" : formatCount.ReadSuccess == formatCount.WriteSuccess ? "+" : "", formatCount.WriteSuccess)); } }