public void TestWriteMultipleRecord([Values(EndOfLine.CRLF, EndOfLine.LF, EndOfLine.CR)] EndOfLine eol) { var record = new MultipleColumnRecord { Value = 1 }; var settings = new CsvWriterSettings() { RecordDelimiter = eol, }; using (var stream = new MemoryStream()) using (var streamWriter = new StreamWriter(stream)) using (var writer = new CsvWriter(streamWriter, settings) { AutoFlush = true }) { writer.WriteRecord(record); stream.Seek(0, SeekOrigin.Begin); using (var reader = new StreamReader(stream)) { var line = reader.ReadLine(); Assert.That(line, Is.Not.Null); var actual = line.Split(','); Assert.That(actual.Length, Is.EqualTo(2)); Assert.That(actual[0], Is.EqualTo("\"1\"")); Assert.That(actual[1], Is.EqualTo("\"一\"")); } } }
public static bool IsTextPlain(string text, out EndOfLine eol) { eol = EndOfLine.None; foreach (char c in text) { if (!char.IsControl(c)) { continue; } else if (c == '\r') { eol |= EndOfLine.CarriageReturn; } else if (c == '\n') { eol |= EndOfLine.LineFeed; } else //char.IsControl and not new line { return(false); } } return(true); }
public void TestReadArrayRecord([Values(EndOfLine.CRLF, EndOfLine.LF, EndOfLine.CR)] EndOfLine eol) { var csv = string.Format("1,2,3,4,5,6,一,二{0}", eol.AsNewline()); var settings = new CsvReaderSettings() { RecordDelimiter = eol, }; using (var stringReader = new StringReader(csv)) using (var reader = new CsvReader(stringReader, settings)) { Assert.That(reader.MoveNext(), Is.True); var record = reader.Current.Convert <ArrayRecord>(); Assert.That(record, Is.Not.Null); var property = record.Property; var field = record.Field; Assert.That(property, Is.Not.Null); Assert.That(field, Is.Not.Null); Assert.That(property[0, 0], Is.EqualTo(1)); Assert.That(property[0, 1], Is.EqualTo(2)); Assert.That(property[1, 0], Is.EqualTo(3)); Assert.That(property[1, 1], Is.EqualTo(4)); Assert.That(property[2, 0], Is.EqualTo(5)); Assert.That(property[2, 1], Is.EqualTo(6)); Assert.That(field[0], Is.EqualTo(1)); Assert.That(field[1], Is.EqualTo(2)); Assert.That(reader.MoveNext(), Is.False); } }
public void TestReadOmittableRecord([Values(EndOfLine.CRLF, EndOfLine.LF, EndOfLine.CR)] EndOfLine eol) { var csv = string.Format("1,2,3{0}4,5{0}6{0}", eol.AsNewline()); var settings = new CsvReaderSettings() { RecordDelimiter = eol, }; using (var stringReader = new StringReader(csv)) using (var reader = new CsvReader(stringReader, settings)) { Assert.That(reader.MoveNext(), Is.True); var record = reader.Current.Convert <OmittableRecord>(); Assert.That(record, Is.Not.Null); Assert.That(record.Column1, Is.EqualTo(1)); Assert.That(record.Column2, Is.EqualTo(2)); Assert.That(record.Column3, Is.EqualTo(3)); Assert.That(reader.MoveNext(), Is.True); record = reader.Current.Convert <OmittableRecord>(); Assert.That(record, Is.Not.Null); Assert.That(record.Column1, Is.EqualTo(4)); Assert.That(record.Column2, Is.EqualTo(5)); Assert.That(record.Column3, Is.EqualTo(default(int))); Assert.That(reader.MoveNext(), Is.True); record = reader.Current.Convert <OmittableRecord>(); Assert.That(record, Is.Not.Null); Assert.That(record.Column1, Is.EqualTo(6)); Assert.That(record.Column2, Is.EqualTo(default(int))); Assert.That(record.Column3, Is.EqualTo(default(int))); Assert.That(reader.MoveNext(), Is.False); } }
public void TestReadToEndWithParameteredConstructor([Values(EndOfLine.CRLF, EndOfLine.LF, EndOfLine.CR)] EndOfLine eol) { var csv = string.Format("1,2,3,4,5{0}6,7,8{0}", eol.AsNewline()); var settings = new CsvReaderSettings() { RecordDelimiter = eol, }; using (var stringReader = new StringReader(csv)) using (var reader = new CsvReader(stringReader, settings)) { var records = reader.ReadToEnd <RecordWithParameteredConstructor>().ToList(); Assert.That(records.Count, Is.EqualTo(2)); var record = records[0]; Assert.That(record, Is.Not.Null); Assert.That(record.Column1, Is.EqualTo(1)); Assert.That(record.Column2, Is.EqualTo(2)); Assert.That(record.Column3, Is.EqualTo(3)); //Assert.That(record.Column4, Is.EqualTo(4)); Assert.That(record.Column5, Is.EqualTo(5)); record = records[1]; Assert.That(record, Is.Not.Null); Assert.That(record.Column1, Is.EqualTo(6)); Assert.That(record.Column2, Is.EqualTo(7)); Assert.That(record.Column3, Is.EqualTo(8)); //Assert.That(record.Column4, Is.EqualTo(default(int))); Assert.That(record.Column5, Is.EqualTo(-1)); } }
/// <summary> /// 指定されたテキストファイルの文字コードを推測します。 /// iso-2022-jp(JIS), euc-jp, shift_jis, UTF-16 UTF-8 に対応。 /// BOM がある場合は、 UTF-16BE UTF-32(LE/BE) にも対応。 /// </summary> /// <param name="path">ファイル名</param> /// <returns>エンコード</returns> public static Encoding GuessEncode(string path, out bool withBOM, out EndOfLine eol) { Encoding enc; using (Stream fs = File.OpenRead(path)) LoadText(fs, out enc, out withBOM, out eol); return enc; }
public void TestWriteIndexerRecord([Values(EndOfLine.CRLF, EndOfLine.LF, EndOfLine.CR)] EndOfLine eol) { var record = new IndexerRecord(); record["A"] = 1; record["B"] = 2; record["C"] = 3; var settings = new CsvWriterSettings() { RecordDelimiter = eol, }; using (var stream = new MemoryStream()) using (var streamWriter = new StreamWriter(stream)) using (var writer = new CsvWriter(streamWriter, settings) { AutoFlush = true }) { writer.WriteRecord(record); stream.Seek(0, SeekOrigin.Begin); using (var reader = new StreamReader(stream)) { var line = reader.ReadLine(); Assert.That(line, Is.Not.Null); var actual = line.Split(','); Assert.That(actual.Length, Is.EqualTo(3)); Assert.That(actual[0], Is.EqualTo("\"1\"")); Assert.That(actual[1], Is.EqualTo("\"二\"")); Assert.That(actual[2], Is.EqualTo("\"3\"")); } } }
/// <summary> /// Gets a plain text alphabet that contains exactly 128 characters /// the first 126 non-control characters and \r and \n as applicable /// </summary> public static IReadOnlyList <char> GetPlainTextAlphabet(EndOfLine eol = EndOfLine.Mixed) { if (_plainTextAlphabet != null) { return(_plainTextAlphabet); } char[] plainTextChars = new char[128]; int charCount = 0; if (eol.HasFlag(EndOfLine.CarriageReturn)) { plainTextChars[charCount++] = '\r'; } if (eol.HasFlag(EndOfLine.LineFeed)) { plainTextChars[charCount++] = '\n'; } for (int a = 0; a < 256; ++a) { char c = (char)a; if (!char.IsControl(c)) { plainTextChars[charCount++] = c; } if (charCount == 128) { break; } } _plainTextAlphabet = plainTextChars; return(_plainTextAlphabet); }
public void TestReadRecordRaw([Values(EndOfLine.CRLF, EndOfLine.LF, EndOfLine.CR)] EndOfLine eol) { const string ascii = "ASCII"; const string nonAscii = "日本語"; const string empty = ""; var withNewline = string.Format("hello{0}world", eol.AsNewline()); var csv = string.Format("{0},{1},{2},\"{3}\",{0}{4}", ascii, nonAscii, empty, withNewline, eol.AsNewline()); var settings = new CsvReaderSettings() { RecordDelimiter = eol, }; using (var stringReader = new StringReader(csv)) using (var reader = new CsvReader(stringReader, settings)) { Assert.That(reader.MoveNext(), Is.True); var record = reader.Current; Assert.That(record, Is.Not.Null); Assert.That(record.FieldCount, Is.EqualTo(5)); Assert.That(record[0], Is.EqualTo(ascii)); Assert.That(record[1], Is.EqualTo(nonAscii)); Assert.That(record[2], Is.EqualTo(empty)); Assert.That(record[3], Is.EqualTo(withNewline)); Assert.That(record[0], Is.EqualTo(ascii)); Assert.That(reader.MoveNext(), Is.False); } }
public void TestWriteRecord1([Values(EndOfLine.CRLF, EndOfLine.LF, EndOfLine.CR)] EndOfLine eol) { const string ascii = "ASCII"; const string nonAscii = "日本語"; const string empty = ""; const string withNewline = "hello\r\nworld"; var record = new[] { ascii, nonAscii, empty, withNewline }; var settings = new CsvWriterSettings() { RecordDelimiter = eol, }; using (var stream = new MemoryStream()) using (var streamWriter = new StreamWriter(stream)) using (var writer = new CsvWriter(streamWriter, settings) { AutoFlush = true }) { writer.WriteRecordRaw(record); stream.Seek(0, SeekOrigin.Begin); using (var reader = new StreamReader(stream)) { var expected = string.Join(",", record.Select(value => "\"" + value + "\"")) + eol.AsNewline(); var actual = reader.ReadToEnd(); Assert.That(actual, Is.EqualTo(expected)); } } }
/// <summary> /// 指定されたテキストファイルの文字コードを推測します。 /// iso-2022-jp(JIS), euc-jp, shift_jis, UTF-16 UTF-8 に対応。 /// BOM がある場合は、 UTF-16BE UTF-32(LE/BE) にも対応。 /// </summary> /// <param name="path">ファイル名</param> /// <returns>エンコード</returns> public static Encoding GuessEncode(string path, out bool withBOM, out EndOfLine eol) { Encoding enc; using (Stream fs = File.OpenRead(path)) LoadText(fs, out enc, out withBOM, out eol); return(enc); }
/// <summary> /// 変数名を決めるための条件 /// </summary> /// <param name="c"></param> /// <returns></returns> private static bool IdentifierTake(char c) { return(!Assign.IsMatch(c) && !EndOfLine.IsMatch(c.ToString()) && !LeftParenthesis.IsMatch(c) && !RightParenthesis.IsMatch(c) && c != ' ' && c != '\t'); }
public void LineEndings_In_Verbatim_String_Should_Respect_Options( string newLine, EndOfLine endOfLine ) { var code = @$ "class ClassName {{ string value = @" "one{newLine}two" "; }}
public void TestReadToEnd([Values(EndOfLine.CR, EndOfLine.CRLF, EndOfLine.LF)] EndOfLine eol) { var s = string.Format("hello{0}world{0}", eol.AsNewline()); using (var reader = new StackBufferedTextReader(new StringReader(s))) { Assert.That(reader.ReadToEnd(), Is.EqualTo(s)); Assert.That(reader.ReadToEnd(), Is.EqualTo("")); } }
public void GetLineEndings_Should_Return_Easy_Cases(EndOfLine endOfLine, string expected) { var code = "tester\n"; var result = CodeFormatter.GetLineEnding( code, new PrinterOptions() { EndOfLine = endOfLine } ); result.Should().Be(expected); }
/// <summary> /// 改行コードを変換する /// </summary> /// <param name="s">変換元テキスト</param> /// <param name="to">改行コード</param> /// <returns>変換されたテキスト</returns> public static string ConvertNewLineCharacter(string s, EndOfLine to) { switch (to) { case EndOfLine.LF: return s.Replace("\r\n", "\n").Replace("\r", "\n"); case EndOfLine.CR: return s.Replace("\r\n", "\r").Replace("\n", "\r"); case EndOfLine.CRLF: return s.Replace("\r\n", "\n").Replace("\r", "\n").Replace("\n", "\r\n"); } return s; }
public IEnumerable<DataEntry> Refine(DataBlock source) { if (_lastEol == null) { _lastEol = new EndOfLine(source); _lastEol.RelativePosition = 0; } foreach(var eol in FindLineEnds(source)) { yield return Build(eol); _lastEol = eol; } }
public void TestReadToEndGeneric([Values(EndOfLine.CRLF, EndOfLine.LF, EndOfLine.CR)] EndOfLine eol) { var headerLine = "string1,int1,double1,parsed1,ignored1,string2,int2,double2,parsed2,ignored2"; var csv = string.Format( "{0}{1}" + "hello,1,-4.0,一,_,world,-1,3.0,二,_{1}" + "good,2,NA,NA,_,bye,0,NA,NA,_{1}", headerLine, eol.AsNewline() ); var settings = new CsvReaderSettings() { RecordDelimiter = eol, }; using (var stringReader = new StringReader(csv)) using (var reader = new CsvReader(stringReader, settings)) { Assert.That(reader.MoveNext(), Is.True); var header = reader.Current.ToArray(); reader.SetHeader(header); Assert.That(header, Is.Not.Null); Assert.That(header, Is.EquivalentTo(headerLine.Split(','))); var records = reader.ReadToEnd <PartialRecord>().ToList(); Assert.That(records.Count, Is.EqualTo(2)); Assert.That(reader.MoveNext(), Is.False); var record = records[0]; Assert.That(record, Is.Not.Null); Assert.That(record.StringField, Is.EqualTo("hello")); Assert.That(record.IntField, Is.EqualTo(1)); Assert.That(record.NullableDoubleField, Is.EqualTo(-4.0)); Assert.That(record.ParsedField, Is.EqualTo(1)); Assert.That(record.IgnoredField, Is.EqualTo(default(object))); Assert.That(record.StringProperty, Is.EqualTo("world")); Assert.That(record.IntProperty, Is.EqualTo(-1)); Assert.That(record.NullableDoubleProperty, Is.EqualTo(3.0)); Assert.That(record.ParsedProperty, Is.EqualTo(2)); Assert.That(record.IgnoredProperty, Is.EqualTo(default(int))); record = records[1]; Assert.That(record, Is.Not.Null); Assert.That(record.StringField, Is.EqualTo("good")); Assert.That(record.IntField, Is.EqualTo(2)); Assert.That(record.NullableDoubleField, Is.Null); Assert.That(record.ParsedField, Is.Null); Assert.That(record.IgnoredField, Is.EqualTo(default(object))); Assert.That(record.StringProperty, Is.EqualTo("bye")); Assert.That(record.IntProperty, Is.EqualTo(0)); Assert.That(record.NullableDoubleProperty, Is.Null); Assert.That(record.ParsedProperty, Is.Null); Assert.That(record.IgnoredProperty, Is.EqualTo(default(int))); } }
/// <summary> /// 改行コードを変換する /// </summary> /// <param name="s">変換元テキスト</param> /// <param name="to">改行コード</param> /// <returns>変換されたテキスト</returns> public static string ConvertNewLineCharacter(string s, EndOfLine to) { switch (to) { case EndOfLine.LF: return(s.Replace("\r\n", "\n").Replace("\r", "\n")); case EndOfLine.CR: return(s.Replace("\r\n", "\r").Replace("\n", "\r")); case EndOfLine.CRLF: return(s.Replace("\r\n", "\n").Replace("\r", "\n").Replace("\n", "\r\n")); } return(s); }
[ExpectedException(typeof(InvalidOperationException))] // Single() throws InvalidOperationException with multiple elements. public void TestRecordWithMultipleRecordConstructor([Values(EndOfLine.CRLF, EndOfLine.LF, EndOfLine.CR)] EndOfLine eol) { var csv = string.Format("1,2,3{0}4,5,6{0}", eol.AsNewline()); var settings = new CsvReaderSettings() { RecordDelimiter = eol, }; using (var stringReader = new StringReader(csv)) using (var reader = new CsvReader(stringReader, settings)) { Assert.That(reader.MoveNext(), Is.True); Assert.That(reader.Current.Convert <RecordWithMultipleRecordConstructor>(), Throws.Exception); } }
public void TestWriteRecord2([Values(EndOfLine.CRLF, EndOfLine.LF, EndOfLine.CR)] EndOfLine eol) { var record = new FullRecord { IntField = 3, IntProperty = -1, NullableDoubleField = null, NullableDoubleProperty = 1.5, ParsedField = 1, ParsedProperty = null, StringField = "hello", StringProperty = null, }; var settings = new CsvWriterSettings { RecordDelimiter = eol, }; using (var stream = new MemoryStream()) using (var streamWriter = new StreamWriter(stream)) using (var writer = new CsvWriter(streamWriter, settings) { AutoFlush = true }) { writer.WriteRecord(record); stream.Seek(0, SeekOrigin.Begin); using (var reader = new StreamReader(stream)) { var line = reader.ReadLine(); Assert.That(line, Is.Not.Null); var actual = line.Split(','); Assert.That(actual.Length, Is.EqualTo(8)); Assert.That(actual[0], Is.EqualTo("\"hello\"")); Assert.That(actual[1], Is.EqualTo("\"3\"")); Assert.That(actual[6], Is.EqualTo("\"1.5\"")); Assert.That(actual[7], Is.EqualTo("\"NA(Int32)\"")); var restActual = new HashSet <string> { actual[2], actual[3], actual[4], actual[5] }; var restExpected = new HashSet <string> { "\"NA\"", "\"一\"", "\"\"", "\"-1\"" }; Assert.That(restActual.SetEquals(restExpected), Is.True); } } }
/// <summary> /// Gets the string representation of the specified end-of-line. /// </summary> /// <param name="eol">The end-of-line enumeration.</param> /// <returns>The newline.</returns> public static string AsNewline(this EndOfLine eol) { switch (eol) { case EndOfLine.CRLF: return("\r\n"); case EndOfLine.LF: return("\n"); case EndOfLine.CR: return("\r"); default: throw new InvalidEnumArgumentException("eol", (int)eol, typeof(EndOfLine)); } }
public void TestMultipleColumn([Values(EndOfLine.CRLF, EndOfLine.LF, EndOfLine.CR)] EndOfLine eol) { var csv = string.Format("1,二{0}", eol.AsNewline()); var settings = new CsvReaderSettings() { RecordDelimiter = eol, }; using (var stringReader = new StringReader(csv)) using (var reader = new CsvReader(stringReader, settings)) { Assert.That(reader.MoveNext(), Is.True); var record = reader.Current.Convert <MultipleColumnRecord>(); Assert.That(record, Is.Not.Null); Assert.That(record.Value, Is.EqualTo(1).Or.EqualTo(2)); Assert.That(reader.MoveNext(), Is.False); } }
private DataEntry Build(EndOfLine end) { long absoluteStart = _lastEol.AbsolutePosition; // simple case: string is in only one block if (_lastEol.Block == end.Block) { return new DataEntry { StartPosition = absoluteStart, EndPosition = end.AbsolutePosition, TextLine = GetFromStartToEnd(_lastEol, end) }; } // build text from start block to end block var buffer = FindLast(end.Block); var sb = new StringBuilder(); foreach (var b in buffer) { bool IsFirstBlock = (b == _lastEol.Block); bool IsLastBlock = (b == end.Block); if (IsFirstBlock) { sb.Append(GetFromLastUntilEnd(_lastEol)); continue; } if (IsLastBlock) { sb.Append(GetFromStart(end)); continue; } sb.Append(b.Data); } // return all text in StringBuilder return new DataEntry { StartPosition = absoluteStart, EndPosition = end.AbsolutePosition, TextLine = sb.ToString() }; }
public void TestReadIndexerRecord([Values(EndOfLine.CRLF, EndOfLine.LF, EndOfLine.CR)] EndOfLine eol) { var csv = string.Format("1,二,3{0}", eol.AsNewline()); var settings = new CsvReaderSettings() { RecordDelimiter = eol, }; using (var stringReader = new StringReader(csv)) using (var reader = new CsvReader(stringReader, settings)) { Assert.That(reader.MoveNext(), Is.True); var record = reader.Current.Convert <IndexerRecord>(); Assert.That(record, Is.Not.Null); Assert.That(record["A"], Is.EqualTo(1)); Assert.That(record["B"], Is.EqualTo(2)); Assert.That(record["C"], Is.EqualTo(3)); Assert.That(reader.MoveNext(), Is.False); } }
public void TestWriteArrayRecord([Values(EndOfLine.CRLF, EndOfLine.LF, EndOfLine.CR)] EndOfLine eol) { var record = new ArrayRecord { Property = new[, ] { { 1, 2 }, { 3, 4 }, { 5, 6 } }, Field = new[] { 1, 2 }, }; var settings = new CsvWriterSettings() { RecordDelimiter = eol, }; using (var stream = new MemoryStream()) using (var streamWriter = new StreamWriter(stream)) using (var writer = new CsvWriter(streamWriter, settings) { AutoFlush = true }) { writer.WriteRecord(record); stream.Seek(0, SeekOrigin.Begin); using (var reader = new StreamReader(stream)) { var line = reader.ReadLine(); Assert.That(line, Is.Not.Null); var actual = line.Split(','); Assert.That(actual.Length, Is.EqualTo(8)); Assert.That(actual[0], Is.EqualTo("\"1\"")); Assert.That(actual[1], Is.EqualTo("\"2\"")); Assert.That(actual[2], Is.EqualTo("\"3\"")); Assert.That(actual[3], Is.EqualTo("\"4\"")); Assert.That(actual[4], Is.EqualTo("\"5\"")); Assert.That(actual[5], Is.EqualTo("\"6\"")); Assert.That(actual[6], Is.EqualTo("\"一\"")); Assert.That(actual[7], Is.EqualTo("\"二\"")); } } }
public void TestReadRecordRaw([Values(EndOfLine.CRLF, EndOfLine.LF, EndOfLine.CR)] EndOfLine eol) { const string ascii = "ASCII"; const string nonAscii = "日本語"; const string empty = ""; var withNewline = string.Format("hello{0}world", eol.AsNewline()); var csv = string.Format("{0},{1},{2},\"{3}\",{0}{4}", ascii, nonAscii, empty, withNewline, eol.AsNewline()); using (var stringReader = new StringReader(csv)) using (var reader = new FastCsvReader(stringReader)) { Assert.That(reader.MoveNext(), Is.True); var record = reader.Current; Assert.That(record, Is.Not.Null); Assert.That(record.FieldCount, Is.EqualTo(5)); Assert.That(record[0], Is.EqualTo(ascii)); Assert.That(record[1], Is.EqualTo(nonAscii)); Assert.That(record[2], Is.EqualTo(empty)); Assert.That(record[3], Is.EqualTo(withNewline.Replace(eol.AsNewline(), "\n"))); // All newlines are converted into '\n'. Assert.That(record[0], Is.EqualTo(ascii)); Assert.That(reader.MoveNext(), Is.False); } }
private string GetFromLastUntilEnd(EndOfLine start) { var text = start.Block.Data .Skip(start.RelativePosition + 1) .ToArray(); return new string(text); }
/// <summary> /// Détection automatique des séparateurs /// </summary> /// <returns><c>true</c> détection réussie, <c>false</c> sinon</returns> bool detectSeparators() { if (!File.Exists(FFilename)) return false; using (StreamReader sr = new StreamReader(FFilename, FEncoding)) { string aLine = sr.ReadLine(); if (!string.IsNullOrEmpty(aLine)) { // On vérifie qu'il y a des délimiteurs de chaine if ((aLine[0] == aLine[aLine.Length - 1]) && (!AlphaChar(aLine[0])) && (aLine[0] != '\t')) { FTextIdentifier = aLine[0].ToString(); Regex regex = new Regex(".+?\\" + FTextIdentifier + "(.{1})\\" + FTextIdentifier + ".+"); FColumnSeparator = regex.Matches(aLine)[0].Groups[1].Value; FEndOfLine = GetEndOfLine(); return true; } else { FTextIdentifier = string.Empty; Dictionary<char, int> dic1 = new Dictionary<char, int>(); Dictionary<char, int> dic2 = new Dictionary<char, int>(); foreach (char c in aLine) { if (!AlphaChar(c)) { if (dic1.ContainsKey(c)) { dic1[c]++; } else dic1.Add(c, 1); } } string aLine2 = sr.ReadLine(); if (!string.IsNullOrEmpty(aLine2)) { foreach (char c in aLine2) { if (!AlphaChar(c)) { if (dic2.ContainsKey(c)) { dic2[c]++; } else dic2.Add(c, 1); } } } foreach (KeyValuePair<char, int> pair in dic1) { if (dic2.ContainsKey(pair.Key) && dic2[pair.Key] == pair.Value) { FColumnSeparator = pair.Key.ToString(); break; } } if (String.IsNullOrEmpty(FColumnSeparator)) { Char _c = '\0'; foreach (KeyValuePair<char, int> pair in dic1) { if (_c == '\0') _c = pair.Key; else { if (dic1[_c] < pair.Value) _c = pair.Key; } } FColumnSeparator = _c.ToString(); } FEndOfLine = GetEndOfLine(); return true; } } else return false; } }
static bool OverwriteFileIfChanged(FormatItem formatItem, IEnumerable <string> lines, EndOfLine eol) { // write to a buffer so we can do an exact comparison vs the file already on disk. let's avoid generating io writes if // there is no actual change. but we need to have the raw bytes so that differences in encoding and EOL's are not masked. var newFileBuffer = new MemoryStream(); WriteLines(newFileBuffer, lines, formatItem.EditorConfig.Charset, eol); var newFileBytes = newFileBuffer.GetBuffer(); var newFileLength = (int)newFileBuffer.Length; // not newFileBytes.Length! var match = new FileInfo(formatItem.Path).Length == newFileLength; // do cheap length check first if (match) { var oldFileBytes = File.ReadAllBytes(formatItem.Path); // must do the byte compare vs disk for (var i = 0; i < newFileLength; ++i) { if (newFileBytes[i] != oldFileBytes[i]) { match = false; break; } } } // ok we have to write it if (!match) { // TODO: copy the permission bits over ($mode & 0777) to the new file // TODO: backup under ./Temp/Format (subfolders? replace folder names with _?) configurable via a new FormatContext.BackupRoot SafeFile.AtomicWrite(formatItem.Path, writePath => { using (var writeFile = File.OpenWrite(writePath)) writeFile.Write(newFileBytes, 0, newFileLength); }); } return(!match); }
/// <summary> /// ストリームに書き込み /// </summary> /// <param name="stream">保存先ストリーム</param> /// <param name="text">テキスト</param> /// <param name="encoding">エンコード</param> /// <param name="withBOM">BOMをつけるか?</param> /// <param name="endOfLine">改行コード</param> public static void SaveText(Stream stream, string text, Encoding encoding, bool withBOM, EndOfLine endOfLine) { if (withBOM) { byte[] preamble = encoding.GetPreamble(); stream.Write(preamble, 0, preamble.Length); } text = ConvertNewLineCharacter(text, endOfLine); byte[] buf = encoding.GetBytes(text); stream.Write(buf, 0, buf.Length); }
/// <summary> /// Convert all line endings in the document to one mode. /// </summary> public virtual void ConvertEOLs(EndOfLine eolMode) { this.SendMessageDirect(2029, (int)eolMode); }
private int GetTotalLength(EndOfLine start, EndOfLine end) { var total = start.AbsolutePosition - end.AbsolutePosition; if (end.IsWindowsEndOfLine) { total -= 1; } return (int)total; }
/// <summary> /// テキストファイルに保存。 /// </summary> /// <param name="path">ファイル</param> /// <param name="text">テキスト</param> /// <param name="encoding">エンコード</param> /// <param name="withBOM">BOMをつけるか?</param> /// <param name="endOfLine">改行コード</param> public static void SaveText(string path, string text, Encoding encoding, bool withBOM, EndOfLine endOfLine) { using (Stream fs = File.OpenWrite(path)) SaveText(fs, text, encoding, withBOM, endOfLine); }
/// <summary> /// テキストファイルを読み込む(エンコード推測+改行を\n単体に変換)。 /// </summary> /// <param name="path">ファイル</param> /// <param name="encoding">読み取りに使ったエンコード</param> /// <param name="withBOM">BOMがあったか?</param> /// <param name="eol">推測される改行コード</param> public static string LoadText(string path, out Encoding encoding, out bool withBOM, out EndOfLine endOfLine) { using (Stream fs = File.OpenRead(path)) return LoadText(fs, out encoding, out withBOM, out endOfLine); }
private string GetFromStart(EndOfLine eol) { var len = eol.RelativePosition; if (eol.IsWindowsEndOfLine) { len -= 1; } var text = eol.Block.Data .Take(len) .ToArray(); return new string(text); }
private string GetFromStartToEnd(EndOfLine start, EndOfLine end) { if (start.Block != end.Block) throw new ArgumentException("start end end is not in the same block"); var len = end.RelativePosition; if (end.IsWindowsEndOfLine) { len -= 1; } var text = start.Block.Data .Skip(start.RelativePosition) .Take(len) .ToArray(); return new string(text); }
/// <summary> /// テキストファイルを読み込む(エンコード推測+改行を\n単体に変換)。 /// </summary> /// <param name="path">ファイル</param> /// <param name="encoding">読み取りに使ったエンコード</param> /// <param name="withBOM">BOMがあったか?</param> /// <param name="eol">推測される改行コード</param> public static string LoadText(string path, out Encoding encoding, out bool withBOM, out EndOfLine endOfLine) { using (Stream fs = File.OpenRead(path)) return(LoadText(fs, out encoding, out withBOM, out endOfLine)); }
/// <summary> /// Nettoie les retours chariots indésirables (si si ! il y en a des fois ^^ ) /// </summary> void CleanCRLF() { if (FTextIdentifier == string.Empty) return; string sFile = string.Empty; //bool shrinked = false; //int idx_shrink = 1; string sep = FTextIdentifier + FColumnSeparator + FTextIdentifier; FEndOfLine = GetEndOfLine(); FileInfo fi = new FileInfo(FFilename); long fileSize = fi.Length; FileStream fs; /* if (fileSize > MAXSIZEFILEBEFORESHRINK) { ShrinkFile(FFilename, FEndOfLine); shrinked = true; } */ //while (true) //{ string filename/*; if (shrinked) filename = FFilename + idx_shrink.ToString(); else filename */= FFilename; if (File.Exists(filename)) { fs = fi.OpenRead(); byte[] fileData = new byte[256]; // buffer int lBytes = fs.Read(fileData, 0, fileData.Length); string aBloc; while (lBytes != 0) { aBloc = Encoding.ASCII.GetString(fileData).Trim('\0'); if ((FEndOfLine == EndOfLine.CR) || (FEndOfLine == EndOfLine.CRLF)) { // On vire les LF aBloc = aBloc.Replace("\n", ""); } sFile += aBloc; lBytes = fs.Read(fileData, 0, fileData.Length); } // On remplace les intersections de colonne par un char 255 sFile = sFile.Replace(sep, ((char)255).ToString()); if ((FEndOfLine == EndOfLine.CR) || (FEndOfLine == EndOfLine.CRLF)) { // On remplace le couple TextIdentifier + CR par un char 254 sFile = sFile.Replace(FTextIdentifier + "\r", ((char)254).ToString()); // On vire les CR sFile = sFile.Replace("\r", ""); if (FEndOfLine == EndOfLine.CRLF) { // On remet les fins de ligne sFile = sFile.Replace(((char)254).ToString(), FTextIdentifier + "\r\n"); } else { // On remet les fins de ligne sFile = sFile.Replace(((char)254).ToString(), FTextIdentifier + "\r"); } } else { // On remplace le couple TextIdentifier + LF par un char 254 sFile = sFile.Replace(FTextIdentifier + "\n", ((char)254).ToString()); // On vire les LF sFile = sFile.Replace("\n", ""); // On remet les fins de ligne sFile = sFile.Replace(((char)254).ToString(), FTextIdentifier + "\n"); } // On remet les intersections de ligne sFile = sFile.Replace(((char)255).ToString(), sep); fs.Close(); /* using (FileStream fs = new FileStream(filename, FileMode.OpenOrCreate)) { sFile = fs.ReadToEnd(); if ((FEndOfLine == EndOfLine.CR) || (FEndOfLine == EndOfLine.CRLF)) { // On vire les LF sFile = sFile.Replace("\n", ""); } // On remplace les intersections de colonne par un char 255 sFile = sFile.Replace(sep, ((char)255).ToString()); if ((FEndOfLine == EndOfLine.CR) || (FEndOfLine == EndOfLine.CRLF)) { // On remplace le couple TextIdentifier + CR par un char 254 sFile = sFile.Replace(FTextIdentifier + "\r", ((char)254).ToString()); // On vire les CR sFile = sFile.Replace("\r", ""); if (FEndOfLine == EndOfLine.CRLF) { // On remet les fins de ligne sFile = sFile.Replace(((char)254).ToString(), FTextIdentifier + "\r\n"); } else { // On remet les fins de ligne sFile = sFile.Replace(((char)254).ToString(), FTextIdentifier + "\r"); } } else { // On remplace le couple TextIdentifier + LF par un char 254 sFile = sFile.Replace(FTextIdentifier + "\n", ((char)254).ToString()); // On vire les LF sFile = sFile.Replace("\n", ""); // On remet les fins de ligne sFile = sFile.Replace(((char)254).ToString(), FTextIdentifier + "\n"); } // On remet les intersections de ligne sFile = sFile.Replace(((char)255).ToString(), sep); sr.Close(); } if (shrinked) idx_shrink++; else break; */ // On réécrit le fichier using (StreamWriter sw = new StreamWriter(filename, false, FEncoding)) { sw.Write(sFile); sw.Flush(); sw.Close(); } } //else break; //} //if (shrinked) UnshrinkFile(FFilename); }
void ShrinkFile(string filename, EndOfLine eol) { int idx = 1; using (StreamReader sr = new StreamReader(filename, Encoding.Default)) { for (string Line = sr.ReadLine(); Line != null; Line = sr.ReadLine()) { if (File.Exists(filename + idx.ToString())) { Line = Line + (((eol == EndOfLine.CRLF) ? "\r\n" : (eol == EndOfLine.CR) ? "\r" : "\n")); FileInfo fi = new FileInfo(filename + idx.ToString()); if ((fi.Length + Line.Length) > MAXSIZEFILEBEFORESHRINK) idx++; } using (StreamWriter sw = new StreamWriter(filename + idx.ToString(), true, Encoding.Default)) { sw.Write(Line); } } } }
private IEnumerable<EndOfLine> FindLineEnds(DataBlock block) { var currentResult = new EndOfLine(block); for(int currentPos = 0; currentPos < block.Length; currentPos++) { if (block.Data[currentPos] == EndOfLine.LineFeed) { currentResult.RelativePosition = currentPos; yield return currentResult; currentResult = new EndOfLine(block); } if (block.Data[currentPos] == EndOfLine.CarriageReturn) { currentResult.IsWindowsEndOfLine = true; } } yield break; }
/// <summary> /// ストリームからテキストを読み込む(エンコード推測+改行を\n単体に変換)。 /// </summary> /// <param name="stream">読み取り元ストリーム</param> /// <param name="encoding">読み取りに使ったエンコード</param> /// <param name="withBOM">BOMがあったか?</param> /// <param name="eol">推測される改行コード</param> /// <returns></returns> public static string LoadText(Stream stream, out Encoding encoding, out bool withBOM, out EndOfLine eol) { byte[] buf = new byte[stream.Length]; stream.Read(buf, 0, buf.Length); encoding = GuessEncode(buf); string text = encoding.GetString(buf); eol = GuessFileFormat(text); text = ConvertNewLineCharacter(text, EndOfLine.LF); withBOM = text[0] == 0xFEFF; // BOMで始まっているか? if (withBOM) { text = text.Substring(1); //BOMは消す。 } return(text); }
/// <summary> /// Renvoit les characteres correspondants au <c>EndOfLine</c> passé en parametre /// </summary> /// <param name="eol">le <c>EndOfLine</c></param> /// <returns>Les characteres correspondants</returns> public static string GetEndOfLine(EndOfLine eol) { switch (eol) { case EndOfLine.CRLF: return "\r\n"; case EndOfLine.LF: return "\n"; default: return "\r\n"; } }
/// <summary> /// ストリームからテキストを読み込む(エンコード推測+改行を\n単体に変換)。 /// </summary> /// <param name="stream">読み取り元ストリーム</param> /// <param name="encoding">読み取りに使ったエンコード</param> /// <param name="withBOM">BOMがあったか?</param> /// <param name="eol">推測される改行コード</param> /// <returns></returns> public static string LoadText(Stream stream, out Encoding encoding, out bool withBOM, out EndOfLine eol) { byte[] buf = new byte[stream.Length]; stream.Read(buf, 0, buf.Length); encoding = GuessEncode(buf); string text = encoding.GetString(buf); eol = GuessFileFormat(text); text = ConvertNewLineCharacter(text, EndOfLine.LF); withBOM = text[0] == 0xFEFF; // BOMで始まっているか? if (withBOM) text = text.Substring(1); //BOMは消す。 return text; }
/// <summary> /// ソースコードを解析してトークン列を生成 /// </summary> /// <param name="targets"></param> /// <returns></returns> private static IEnumerable <Token> Analysis(string targets) { if (targets.EndsWith(";") == false || targets.EndsWith("\r\n") == false) { targets += "\r\n"; } while (targets.Length > 0) { //スペースとタブは変換しない if (targets[0] == ' ' || targets[0] == '\t') { targets = targets.Remove(0, 1); continue; } if (targets[0] == '\"') { var literal = targets .Skip(1) .TakeWhile(c => c != '\"') .Aggregate(new StringBuilder(), (sb, c) => sb.Append(c)) .ToString(); targets = targets.Remove(0, literal.Length + 2); yield return(new StringLiteral(literal)); continue; } if (LeftParenthesis.IsMatch(targets[0])) { targets = targets.Remove(0, 1); yield return(new LeftParenthesis()); continue; } if (RightParenthesis.IsMatch(targets[0])) { targets = targets.Remove(0, 1); yield return(new RightParenthesis()); continue; } if (BlockBegin.IsMatch(targets[0])) { targets = targets.Remove(0, 1); yield return(new BlockBegin()); continue; } if (BlockEnd.IsMatch(targets[0])) { targets = targets.Remove(0, 1); yield return(new BlockEnd()); continue; } if (targets.StartsWith("Writeln")) { targets = targets.Remove(0, 7); yield return(new StandardOutput()); continue; } if (targets.StartsWith("Readln()")) { targets = targets.Remove(0, 8); yield return(new StandardInput()); continue; } if (Int32Literal.SpecifiedCollection.Contains(targets[0])) { var value = targets.TakeWhile(c => Int32Literal.SpecifiedCollection.Contains(c)).ToArray(); targets = targets.Remove(0, value.Length); yield return(new Int32Literal(new string(value))); continue; } if (EndOfLine.IsMatch(targets[0].ToString()) || EndOfLine.IsMatch(new string(targets.Take(2).ToArray()))) { var value = targets.TakeWhile(i => i == ';' || i == '\r' || i == '\n').Count(); targets = targets.Remove(0, value); yield return(new EndOfLine()); continue; } if (targets.Length > 1 && Assign.IsMatch(targets[0]) && !Assign.IsMatch(targets[1])) { targets = targets.Remove(0, 1); yield return(new Assign()); continue; } if (targets.Length > 1 && AddAssign.IsMatch(targets[0], targets[1])) { targets = targets.Remove(0, 2); yield return(new AddAssign()); continue; } if (targets.StartsWith("var ")) { var varVariableName = targets.Skip(4) .TakeWhile(c => c != ' ' && c != '\t' && c != '=') .Aggregate(new StringBuilder(), (sb, c) => sb.Append(c)) .ToString(); if (!(JudgeWord(varVariableName) is Identifier)) { throw new Exception("変数にキーワードが使われています。"); } targets = targets.Remove(0, 4 + varVariableName.Length); yield return(new VarDeclaration(varVariableName)); continue; } if (PlusOperator.IsMatch(targets[0])) { targets = targets.Remove(0, 1); yield return(new PlusOperator()); continue; } if (MinusOperator.IsMatch(targets[0])) { targets = targets.Remove(0, 1); yield return(new MinusOperator()); continue; } if (MultiplyOperator.IsMatch(targets[0])) { targets = targets.Remove(0, 1); yield return(new MultiplyOperator()); continue; } if (DivideOperator.IsMatch(targets[0])) { targets = targets.Remove(0, 1); yield return(new DivideOperator()); continue; } if (ModuloOperator.IsMatch(targets[0])) { targets = targets.Remove(0, 1); yield return(new ModuloOperator()); continue; } if (targets.Length > 1 && targets[0] == '=' && targets[1] == '=') { targets = targets.Remove(0, 2); yield return(new EqualOperator()); continue; } if (targets.Length > 1 && targets[0] == '!' && targets[1] == '=') { targets = targets.Remove(0, 2); yield return(new NotEqualOperator()); continue; } if (targets.Length > 1 && targets[0] == '>') { if (targets[1] == '=') { targets = targets.Remove(0, 2); yield return(new GreaterThanOrEqualOperator()); continue; } targets = targets.Remove(0, 1); yield return(new GreaterThanOperator()); continue; } if (targets.Length > 1 && targets[0] == '<') { if (targets[1] == '=') { targets = targets.Remove(0, 2); yield return(new LessThanOrEqualOperator()); continue; } targets = targets.Remove(0, 1); yield return(new LessThanOperator()); continue; } if (targets.Length > 1 && targets[0] == '&' && targets[1] == '&') { targets = targets.Remove(0, 2); yield return(new AndAlsoOperator()); continue; } if (targets.Length > 1 && targets[0] == '|' && targets[1] == '|') { targets = targets.Remove(0, 2); yield return(new OrElseOperator()); continue; } var word = targets.TakeWhile(IdentifierTake) .Aggregate(new StringBuilder(), (sb, c) => sb.Append(c)) .ToString(); if (word.Length == 0) { throw new Exception("変数名なのに文字数がゼロ"); } targets = targets.Remove(0, word.Length); yield return(JudgeWord(word)); } }