public void TestWriteMultipleRecord([Values(EndOfLine.CRLF, EndOfLine.LF, EndOfLine.CR)] EndOfLine eol)
{
var record = new MultipleColumnRecord
{
Value = 1
};
var settings = new CsvWriterSettings()
{
RecordDelimiter = eol,
};
using (var stream = new MemoryStream())
using (var streamWriter = new StreamWriter(stream))
using (var writer = new CsvWriter(streamWriter, settings)
{
AutoFlush = true
})
{
writer.WriteRecord(record);
stream.Seek(0, SeekOrigin.Begin);
using (var reader = new StreamReader(stream))
{
var line = reader.ReadLine();
Assert.That(line, Is.Not.Null);
var actual = line.Split(',');
Assert.That(actual.Length, Is.EqualTo(2));
Assert.That(actual[0], Is.EqualTo("\"1\""));
Assert.That(actual[1], Is.EqualTo("\"一\""));
}
}
}
public static bool IsTextPlain(string text, out EndOfLine eol)
{
eol = EndOfLine.None;
foreach (char c in text)
{
if (!char.IsControl(c))
{
continue;
}
else if (c == '\r')
{
eol |= EndOfLine.CarriageReturn;
}
else if (c == '\n')
{
eol |= EndOfLine.LineFeed;
}
else //char.IsControl and not new line
{
return(false);
}
}
return(true);
}
public void TestReadArrayRecord([Values(EndOfLine.CRLF, EndOfLine.LF, EndOfLine.CR)] EndOfLine eol)
{
var csv = string.Format("1,2,3,4,5,6,一,二{0}", eol.AsNewline());
var settings = new CsvReaderSettings()
{
RecordDelimiter = eol,
};
using (var stringReader = new StringReader(csv))
using (var reader = new CsvReader(stringReader, settings))
{
Assert.That(reader.MoveNext(), Is.True);
var record = reader.Current.Convert <ArrayRecord>();
Assert.That(record, Is.Not.Null);
var property = record.Property;
var field = record.Field;
Assert.That(property, Is.Not.Null);
Assert.That(field, Is.Not.Null);
Assert.That(property[0, 0], Is.EqualTo(1));
Assert.That(property[0, 1], Is.EqualTo(2));
Assert.That(property[1, 0], Is.EqualTo(3));
Assert.That(property[1, 1], Is.EqualTo(4));
Assert.That(property[2, 0], Is.EqualTo(5));
Assert.That(property[2, 1], Is.EqualTo(6));
Assert.That(field[0], Is.EqualTo(1));
Assert.That(field[1], Is.EqualTo(2));
Assert.That(reader.MoveNext(), Is.False);
}
}
public void TestReadOmittableRecord([Values(EndOfLine.CRLF, EndOfLine.LF, EndOfLine.CR)] EndOfLine eol)
{
var csv = string.Format("1,2,3{0}4,5{0}6{0}", eol.AsNewline());
var settings = new CsvReaderSettings()
{
RecordDelimiter = eol,
};
using (var stringReader = new StringReader(csv))
using (var reader = new CsvReader(stringReader, settings))
{
Assert.That(reader.MoveNext(), Is.True);
var record = reader.Current.Convert <OmittableRecord>();
Assert.That(record, Is.Not.Null);
Assert.That(record.Column1, Is.EqualTo(1));
Assert.That(record.Column2, Is.EqualTo(2));
Assert.That(record.Column3, Is.EqualTo(3));
Assert.That(reader.MoveNext(), Is.True);
record = reader.Current.Convert <OmittableRecord>();
Assert.That(record, Is.Not.Null);
Assert.That(record.Column1, Is.EqualTo(4));
Assert.That(record.Column2, Is.EqualTo(5));
Assert.That(record.Column3, Is.EqualTo(default(int)));
Assert.That(reader.MoveNext(), Is.True);
record = reader.Current.Convert <OmittableRecord>();
Assert.That(record, Is.Not.Null);
Assert.That(record.Column1, Is.EqualTo(6));
Assert.That(record.Column2, Is.EqualTo(default(int)));
Assert.That(record.Column3, Is.EqualTo(default(int)));
Assert.That(reader.MoveNext(), Is.False);
}
}
public void TestReadToEndWithParameteredConstructor([Values(EndOfLine.CRLF, EndOfLine.LF, EndOfLine.CR)] EndOfLine eol)
{
var csv = string.Format("1,2,3,4,5{0}6,7,8{0}", eol.AsNewline());
var settings = new CsvReaderSettings()
{
RecordDelimiter = eol,
};
using (var stringReader = new StringReader(csv))
using (var reader = new CsvReader(stringReader, settings))
{
var records = reader.ReadToEnd <RecordWithParameteredConstructor>().ToList();
Assert.That(records.Count, Is.EqualTo(2));
var record = records[0];
Assert.That(record, Is.Not.Null);
Assert.That(record.Column1, Is.EqualTo(1));
Assert.That(record.Column2, Is.EqualTo(2));
Assert.That(record.Column3, Is.EqualTo(3));
//Assert.That(record.Column4, Is.EqualTo(4));
Assert.That(record.Column5, Is.EqualTo(5));
record = records[1];
Assert.That(record, Is.Not.Null);
Assert.That(record.Column1, Is.EqualTo(6));
Assert.That(record.Column2, Is.EqualTo(7));
Assert.That(record.Column3, Is.EqualTo(8));
//Assert.That(record.Column4, Is.EqualTo(default(int)));
Assert.That(record.Column5, Is.EqualTo(-1));
}
}
/// <summary>
/// 指定されたテキストファイルの文字コードを推測します。
/// iso-2022-jp(JIS), euc-jp, shift_jis, UTF-16 UTF-8 に対応。
/// BOM がある場合は、 UTF-16BE UTF-32(LE/BE) にも対応。
/// </summary>
/// <param name="path">ファイル名</param>
/// <returns>エンコード</returns>
public static Encoding GuessEncode(string path, out bool withBOM, out EndOfLine eol)
{
Encoding enc;
using (Stream fs = File.OpenRead(path))
LoadText(fs, out enc, out withBOM, out eol);
return enc;
}
public void TestWriteIndexerRecord([Values(EndOfLine.CRLF, EndOfLine.LF, EndOfLine.CR)] EndOfLine eol)
{
var record = new IndexerRecord();
record["A"] = 1;
record["B"] = 2;
record["C"] = 3;
var settings = new CsvWriterSettings()
{
RecordDelimiter = eol,
};
using (var stream = new MemoryStream())
using (var streamWriter = new StreamWriter(stream))
using (var writer = new CsvWriter(streamWriter, settings)
{
AutoFlush = true
})
{
writer.WriteRecord(record);
stream.Seek(0, SeekOrigin.Begin);
using (var reader = new StreamReader(stream))
{
var line = reader.ReadLine();
Assert.That(line, Is.Not.Null);
var actual = line.Split(',');
Assert.That(actual.Length, Is.EqualTo(3));
Assert.That(actual[0], Is.EqualTo("\"1\""));
Assert.That(actual[1], Is.EqualTo("\"二\""));
Assert.That(actual[2], Is.EqualTo("\"3\""));
}
}
}
/// <summary>
/// Gets a plain text alphabet that contains exactly 128 characters
/// the first 126 non-control characters and \r and \n as applicable
/// </summary>
public static IReadOnlyList <char> GetPlainTextAlphabet(EndOfLine eol = EndOfLine.Mixed)
{
if (_plainTextAlphabet != null)
{
return(_plainTextAlphabet);
}
char[] plainTextChars = new char[128];
int charCount = 0;
if (eol.HasFlag(EndOfLine.CarriageReturn))
{
plainTextChars[charCount++] = '\r';
}
if (eol.HasFlag(EndOfLine.LineFeed))
{
plainTextChars[charCount++] = '\n';
}
for (int a = 0; a < 256; ++a)
{
char c = (char)a;
if (!char.IsControl(c))
{
plainTextChars[charCount++] = c;
}
if (charCount == 128)
{
break;
}
}
_plainTextAlphabet = plainTextChars;
return(_plainTextAlphabet);
}
public void TestReadRecordRaw([Values(EndOfLine.CRLF, EndOfLine.LF, EndOfLine.CR)] EndOfLine eol)
{
const string ascii = "ASCII";
const string nonAscii = "日本語";
const string empty = "";
var withNewline = string.Format("hello{0}world", eol.AsNewline());
var csv = string.Format("{0},{1},{2},\"{3}\",{0}{4}", ascii, nonAscii, empty, withNewline, eol.AsNewline());
var settings = new CsvReaderSettings()
{
RecordDelimiter = eol,
};
using (var stringReader = new StringReader(csv))
using (var reader = new CsvReader(stringReader, settings))
{
Assert.That(reader.MoveNext(), Is.True);
var record = reader.Current;
Assert.That(record, Is.Not.Null);
Assert.That(record.FieldCount, Is.EqualTo(5));
Assert.That(record[0], Is.EqualTo(ascii));
Assert.That(record[1], Is.EqualTo(nonAscii));
Assert.That(record[2], Is.EqualTo(empty));
Assert.That(record[3], Is.EqualTo(withNewline));
Assert.That(record[0], Is.EqualTo(ascii));
Assert.That(reader.MoveNext(), Is.False);
}
}
public void TestWriteRecord1([Values(EndOfLine.CRLF, EndOfLine.LF, EndOfLine.CR)] EndOfLine eol)
{
const string ascii = "ASCII";
const string nonAscii = "日本語";
const string empty = "";
const string withNewline = "hello\r\nworld";
var record = new[] { ascii, nonAscii, empty, withNewline };
var settings = new CsvWriterSettings()
{
RecordDelimiter = eol,
};
using (var stream = new MemoryStream())
using (var streamWriter = new StreamWriter(stream))
using (var writer = new CsvWriter(streamWriter, settings)
{
AutoFlush = true
})
{
writer.WriteRecordRaw(record);
stream.Seek(0, SeekOrigin.Begin);
using (var reader = new StreamReader(stream))
{
var expected = string.Join(",", record.Select(value => "\"" + value + "\"")) + eol.AsNewline();
var actual = reader.ReadToEnd();
Assert.That(actual, Is.EqualTo(expected));
}
}
}
/// <summary>
/// 指定されたテキストファイルの文字コードを推測します。
/// iso-2022-jp(JIS), euc-jp, shift_jis, UTF-16 UTF-8 に対応。
/// BOM がある場合は、 UTF-16BE UTF-32(LE/BE) にも対応。
/// </summary>
/// <param name="path">ファイル名</param>
/// <returns>エンコード</returns>
public static Encoding GuessEncode(string path, out bool withBOM, out EndOfLine eol)
{
Encoding enc;
using (Stream fs = File.OpenRead(path))
LoadText(fs, out enc, out withBOM, out eol);
return(enc);
}
/// <summary>
/// 変数名を決めるための条件
/// </summary>
/// <param name="c"></param>
/// <returns></returns>
private static bool IdentifierTake(char c)
{
return(!Assign.IsMatch(c) &&
!EndOfLine.IsMatch(c.ToString()) &&
!LeftParenthesis.IsMatch(c) &&
!RightParenthesis.IsMatch(c) &&
c != ' ' &&
c != '\t');
}
public void LineEndings_In_Verbatim_String_Should_Respect_Options(
string newLine,
EndOfLine endOfLine
)
{
var code =
@$ "class ClassName
{{
string value = @" "one{newLine}two" ";
}}
public void TestReadToEnd([Values(EndOfLine.CR, EndOfLine.CRLF, EndOfLine.LF)] EndOfLine eol)
{
var s = string.Format("hello{0}world{0}", eol.AsNewline());
using (var reader = new StackBufferedTextReader(new StringReader(s)))
{
Assert.That(reader.ReadToEnd(), Is.EqualTo(s));
Assert.That(reader.ReadToEnd(), Is.EqualTo(""));
}
}
public void GetLineEndings_Should_Return_Easy_Cases(EndOfLine endOfLine, string expected)
{
var code = "tester\n";
var result = CodeFormatter.GetLineEnding(
code,
new PrinterOptions()
{
EndOfLine = endOfLine
}
);
result.Should().Be(expected);
}
/// <summary>
/// 改行コードを変換する
/// </summary>
/// <param name="s">変換元テキスト</param>
/// <param name="to">改行コード</param>
/// <returns>変換されたテキスト</returns>
public static string ConvertNewLineCharacter(string s, EndOfLine to)
{
switch (to)
{
case EndOfLine.LF:
return s.Replace("\r\n", "\n").Replace("\r", "\n");
case EndOfLine.CR:
return s.Replace("\r\n", "\r").Replace("\n", "\r");
case EndOfLine.CRLF:
return s.Replace("\r\n", "\n").Replace("\r", "\n").Replace("\n", "\r\n");
}
return s;
}
public IEnumerable<DataEntry> Refine(DataBlock source)
{
if (_lastEol == null)
{
_lastEol = new EndOfLine(source);
_lastEol.RelativePosition = 0;
}
foreach(var eol in FindLineEnds(source))
{
yield return Build(eol);
_lastEol = eol;
}
}
public void TestReadToEndGeneric([Values(EndOfLine.CRLF, EndOfLine.LF, EndOfLine.CR)] EndOfLine eol)
{
var headerLine = "string1,int1,double1,parsed1,ignored1,string2,int2,double2,parsed2,ignored2";
var csv = string.Format(
"{0}{1}" +
"hello,1,-4.0,一,_,world,-1,3.0,二,_{1}" +
"good,2,NA,NA,_,bye,0,NA,NA,_{1}",
headerLine, eol.AsNewline()
);
var settings = new CsvReaderSettings()
{
RecordDelimiter = eol,
};
using (var stringReader = new StringReader(csv))
using (var reader = new CsvReader(stringReader, settings))
{
Assert.That(reader.MoveNext(), Is.True);
var header = reader.Current.ToArray();
reader.SetHeader(header);
Assert.That(header, Is.Not.Null);
Assert.That(header, Is.EquivalentTo(headerLine.Split(',')));
var records = reader.ReadToEnd <PartialRecord>().ToList();
Assert.That(records.Count, Is.EqualTo(2));
Assert.That(reader.MoveNext(), Is.False);
var record = records[0];
Assert.That(record, Is.Not.Null);
Assert.That(record.StringField, Is.EqualTo("hello"));
Assert.That(record.IntField, Is.EqualTo(1));
Assert.That(record.NullableDoubleField, Is.EqualTo(-4.0));
Assert.That(record.ParsedField, Is.EqualTo(1));
Assert.That(record.IgnoredField, Is.EqualTo(default(object)));
Assert.That(record.StringProperty, Is.EqualTo("world"));
Assert.That(record.IntProperty, Is.EqualTo(-1));
Assert.That(record.NullableDoubleProperty, Is.EqualTo(3.0));
Assert.That(record.ParsedProperty, Is.EqualTo(2));
Assert.That(record.IgnoredProperty, Is.EqualTo(default(int)));
record = records[1];
Assert.That(record, Is.Not.Null);
Assert.That(record.StringField, Is.EqualTo("good"));
Assert.That(record.IntField, Is.EqualTo(2));
Assert.That(record.NullableDoubleField, Is.Null);
Assert.That(record.ParsedField, Is.Null);
Assert.That(record.IgnoredField, Is.EqualTo(default(object)));
Assert.That(record.StringProperty, Is.EqualTo("bye"));
Assert.That(record.IntProperty, Is.EqualTo(0));
Assert.That(record.NullableDoubleProperty, Is.Null);
Assert.That(record.ParsedProperty, Is.Null);
Assert.That(record.IgnoredProperty, Is.EqualTo(default(int)));
}
}
/// <summary>
/// 改行コードを変換する
/// </summary>
/// <param name="s">変換元テキスト</param>
/// <param name="to">改行コード</param>
/// <returns>変換されたテキスト</returns>
public static string ConvertNewLineCharacter(string s, EndOfLine to)
{
switch (to)
{
case EndOfLine.LF:
return(s.Replace("\r\n", "\n").Replace("\r", "\n"));
case EndOfLine.CR:
return(s.Replace("\r\n", "\r").Replace("\n", "\r"));
case EndOfLine.CRLF:
return(s.Replace("\r\n", "\n").Replace("\r", "\n").Replace("\n", "\r\n"));
}
return(s);
}
[ExpectedException(typeof(InvalidOperationException))] // Single() throws InvalidOperationException with multiple elements.
public void TestRecordWithMultipleRecordConstructor([Values(EndOfLine.CRLF, EndOfLine.LF, EndOfLine.CR)] EndOfLine eol)
{
var csv = string.Format("1,2,3{0}4,5,6{0}", eol.AsNewline());
var settings = new CsvReaderSettings()
{
RecordDelimiter = eol,
};
using (var stringReader = new StringReader(csv))
using (var reader = new CsvReader(stringReader, settings))
{
Assert.That(reader.MoveNext(), Is.True);
Assert.That(reader.Current.Convert <RecordWithMultipleRecordConstructor>(), Throws.Exception);
}
}
public void TestWriteRecord2([Values(EndOfLine.CRLF, EndOfLine.LF, EndOfLine.CR)] EndOfLine eol)
{
var record = new FullRecord
{
IntField = 3,
IntProperty = -1,
NullableDoubleField = null,
NullableDoubleProperty = 1.5,
ParsedField = 1,
ParsedProperty = null,
StringField = "hello",
StringProperty = null,
};
var settings = new CsvWriterSettings
{
RecordDelimiter = eol,
};
using (var stream = new MemoryStream())
using (var streamWriter = new StreamWriter(stream))
using (var writer = new CsvWriter(streamWriter, settings)
{
AutoFlush = true
})
{
writer.WriteRecord(record);
stream.Seek(0, SeekOrigin.Begin);
using (var reader = new StreamReader(stream))
{
var line = reader.ReadLine();
Assert.That(line, Is.Not.Null);
var actual = line.Split(',');
Assert.That(actual.Length, Is.EqualTo(8));
Assert.That(actual[0], Is.EqualTo("\"hello\""));
Assert.That(actual[1], Is.EqualTo("\"3\""));
Assert.That(actual[6], Is.EqualTo("\"1.5\""));
Assert.That(actual[7], Is.EqualTo("\"NA(Int32)\""));
var restActual = new HashSet <string> {
actual[2], actual[3], actual[4], actual[5]
};
var restExpected = new HashSet <string> {
"\"NA\"", "\"一\"", "\"\"", "\"-1\""
};
Assert.That(restActual.SetEquals(restExpected), Is.True);
}
}
}
/// <summary>
/// Gets the string representation of the specified end-of-line.
/// </summary>
/// <param name="eol">The end-of-line enumeration.</param>
/// <returns>The newline.</returns>
public static string AsNewline(this EndOfLine eol)
{
switch (eol)
{
case EndOfLine.CRLF:
return("\r\n");
case EndOfLine.LF:
return("\n");
case EndOfLine.CR:
return("\r");
default:
throw new InvalidEnumArgumentException("eol", (int)eol, typeof(EndOfLine));
}
}
public void TestMultipleColumn([Values(EndOfLine.CRLF, EndOfLine.LF, EndOfLine.CR)] EndOfLine eol)
{
var csv = string.Format("1,二{0}", eol.AsNewline());
var settings = new CsvReaderSettings()
{
RecordDelimiter = eol,
};
using (var stringReader = new StringReader(csv))
using (var reader = new CsvReader(stringReader, settings))
{
Assert.That(reader.MoveNext(), Is.True);
var record = reader.Current.Convert <MultipleColumnRecord>();
Assert.That(record, Is.Not.Null);
Assert.That(record.Value, Is.EqualTo(1).Or.EqualTo(2));
Assert.That(reader.MoveNext(), Is.False);
}
}
private DataEntry Build(EndOfLine end)
{
long absoluteStart = _lastEol.AbsolutePosition;
// simple case: string is in only one block
if (_lastEol.Block == end.Block)
{
return new DataEntry
{
StartPosition = absoluteStart,
EndPosition = end.AbsolutePosition,
TextLine = GetFromStartToEnd(_lastEol, end)
};
}
// build text from start block to end block
var buffer = FindLast(end.Block);
var sb = new StringBuilder();
foreach (var b in buffer)
{
bool IsFirstBlock = (b == _lastEol.Block);
bool IsLastBlock = (b == end.Block);
if (IsFirstBlock)
{
sb.Append(GetFromLastUntilEnd(_lastEol));
continue;
}
if (IsLastBlock)
{
sb.Append(GetFromStart(end));
continue;
}
sb.Append(b.Data);
}
// return all text in StringBuilder
return new DataEntry
{
StartPosition = absoluteStart,
EndPosition = end.AbsolutePosition,
TextLine = sb.ToString()
};
}
public void TestReadIndexerRecord([Values(EndOfLine.CRLF, EndOfLine.LF, EndOfLine.CR)] EndOfLine eol)
{
var csv = string.Format("1,二,3{0}", eol.AsNewline());
var settings = new CsvReaderSettings()
{
RecordDelimiter = eol,
};
using (var stringReader = new StringReader(csv))
using (var reader = new CsvReader(stringReader, settings))
{
Assert.That(reader.MoveNext(), Is.True);
var record = reader.Current.Convert <IndexerRecord>();
Assert.That(record, Is.Not.Null);
Assert.That(record["A"], Is.EqualTo(1));
Assert.That(record["B"], Is.EqualTo(2));
Assert.That(record["C"], Is.EqualTo(3));
Assert.That(reader.MoveNext(), Is.False);
}
}
public void TestWriteArrayRecord([Values(EndOfLine.CRLF, EndOfLine.LF, EndOfLine.CR)] EndOfLine eol)
{
var record = new ArrayRecord
{
Property = new[, ] {
{ 1, 2 }, { 3, 4 }, { 5, 6 }
},
Field = new[] { 1, 2 },
};
var settings = new CsvWriterSettings()
{
RecordDelimiter = eol,
};
using (var stream = new MemoryStream())
using (var streamWriter = new StreamWriter(stream))
using (var writer = new CsvWriter(streamWriter, settings)
{
AutoFlush = true
})
{
writer.WriteRecord(record);
stream.Seek(0, SeekOrigin.Begin);
using (var reader = new StreamReader(stream))
{
var line = reader.ReadLine();
Assert.That(line, Is.Not.Null);
var actual = line.Split(',');
Assert.That(actual.Length, Is.EqualTo(8));
Assert.That(actual[0], Is.EqualTo("\"1\""));
Assert.That(actual[1], Is.EqualTo("\"2\""));
Assert.That(actual[2], Is.EqualTo("\"3\""));
Assert.That(actual[3], Is.EqualTo("\"4\""));
Assert.That(actual[4], Is.EqualTo("\"5\""));
Assert.That(actual[5], Is.EqualTo("\"6\""));
Assert.That(actual[6], Is.EqualTo("\"一\""));
Assert.That(actual[7], Is.EqualTo("\"二\""));
}
}
}
public void TestReadRecordRaw([Values(EndOfLine.CRLF, EndOfLine.LF, EndOfLine.CR)] EndOfLine eol)
{
const string ascii = "ASCII";
const string nonAscii = "日本語";
const string empty = "";
var withNewline = string.Format("hello{0}world", eol.AsNewline());
var csv = string.Format("{0},{1},{2},\"{3}\",{0}{4}", ascii, nonAscii, empty, withNewline, eol.AsNewline());
using (var stringReader = new StringReader(csv))
using (var reader = new FastCsvReader(stringReader))
{
Assert.That(reader.MoveNext(), Is.True);
var record = reader.Current;
Assert.That(record, Is.Not.Null);
Assert.That(record.FieldCount, Is.EqualTo(5));
Assert.That(record[0], Is.EqualTo(ascii));
Assert.That(record[1], Is.EqualTo(nonAscii));
Assert.That(record[2], Is.EqualTo(empty));
Assert.That(record[3], Is.EqualTo(withNewline.Replace(eol.AsNewline(), "\n"))); // All newlines are converted into '\n'.
Assert.That(record[0], Is.EqualTo(ascii));
Assert.That(reader.MoveNext(), Is.False);
}
}
private string GetFromLastUntilEnd(EndOfLine start)
{
var text = start.Block.Data
.Skip(start.RelativePosition + 1)
.ToArray();
return new string(text);
}
/// <summary>
/// Détection automatique des séparateurs
/// </summary>
/// <returns><c>true</c> détection réussie, <c>false</c> sinon</returns>
bool detectSeparators()
{
if (!File.Exists(FFilename)) return false;
using (StreamReader sr = new StreamReader(FFilename, FEncoding))
{
string aLine = sr.ReadLine();
if (!string.IsNullOrEmpty(aLine))
{
// On vérifie qu'il y a des délimiteurs de chaine
if ((aLine[0] == aLine[aLine.Length - 1]) && (!AlphaChar(aLine[0])) && (aLine[0] != '\t'))
{
FTextIdentifier = aLine[0].ToString();
Regex regex = new Regex(".+?\\" + FTextIdentifier + "(.{1})\\" + FTextIdentifier + ".+");
FColumnSeparator = regex.Matches(aLine)[0].Groups[1].Value;
FEndOfLine = GetEndOfLine();
return true;
}
else
{
FTextIdentifier = string.Empty;
Dictionary<char, int> dic1 = new Dictionary<char, int>();
Dictionary<char, int> dic2 = new Dictionary<char, int>();
foreach (char c in aLine)
{
if (!AlphaChar(c))
{
if (dic1.ContainsKey(c))
{
dic1[c]++;
}
else
dic1.Add(c, 1);
}
}
string aLine2 = sr.ReadLine();
if (!string.IsNullOrEmpty(aLine2))
{
foreach (char c in aLine2)
{
if (!AlphaChar(c))
{
if (dic2.ContainsKey(c))
{
dic2[c]++;
}
else
dic2.Add(c, 1);
}
}
}
foreach (KeyValuePair<char, int> pair in dic1)
{
if (dic2.ContainsKey(pair.Key) && dic2[pair.Key] == pair.Value)
{
FColumnSeparator = pair.Key.ToString();
break;
}
}
if (String.IsNullOrEmpty(FColumnSeparator))
{
Char _c = '\0';
foreach (KeyValuePair<char, int> pair in dic1)
{
if (_c == '\0')
_c = pair.Key;
else
{
if (dic1[_c] < pair.Value)
_c = pair.Key;
}
}
FColumnSeparator = _c.ToString();
}
FEndOfLine = GetEndOfLine();
return true;
}
}
else
return false;
}
}
static bool OverwriteFileIfChanged(FormatItem formatItem, IEnumerable <string> lines, EndOfLine eol)
{
// write to a buffer so we can do an exact comparison vs the file already on disk. let's avoid generating io writes if
// there is no actual change. but we need to have the raw bytes so that differences in encoding and EOL's are not masked.
var newFileBuffer = new MemoryStream();
WriteLines(newFileBuffer, lines, formatItem.EditorConfig.Charset, eol);
var newFileBytes = newFileBuffer.GetBuffer();
var newFileLength = (int)newFileBuffer.Length; // not newFileBytes.Length!
var match = new FileInfo(formatItem.Path).Length == newFileLength; // do cheap length check first
if (match)
{
var oldFileBytes = File.ReadAllBytes(formatItem.Path);
// must do the byte compare vs disk
for (var i = 0; i < newFileLength; ++i)
{
if (newFileBytes[i] != oldFileBytes[i])
{
match = false;
break;
}
}
}
// ok we have to write it
if (!match)
{
// TODO: copy the permission bits over ($mode & 0777) to the new file
// TODO: backup under ./Temp/Format (subfolders? replace folder names with _?) configurable via a new FormatContext.BackupRoot
SafeFile.AtomicWrite(formatItem.Path, writePath =>
{
using (var writeFile = File.OpenWrite(writePath))
writeFile.Write(newFileBytes, 0, newFileLength);
});
}
return(!match);
}
/// <summary>
/// ストリームに書き込み
/// </summary>
/// <param name="stream">保存先ストリーム</param>
/// <param name="text">テキスト</param>
/// <param name="encoding">エンコード</param>
/// <param name="withBOM">BOMをつけるか？</param>
/// <param name="endOfLine">改行コード</param>
public static void SaveText(Stream stream, string text, Encoding encoding, bool withBOM, EndOfLine endOfLine)
{
if (withBOM)
{
byte[] preamble = encoding.GetPreamble();
stream.Write(preamble, 0, preamble.Length);
}
text = ConvertNewLineCharacter(text, endOfLine);
byte[] buf = encoding.GetBytes(text);
stream.Write(buf, 0, buf.Length);
}
/// <summary>
/// Convert all line endings in the document to one mode.
/// </summary>
public virtual void ConvertEOLs(EndOfLine eolMode)
{
this.SendMessageDirect(2029, (int)eolMode);
}
private int GetTotalLength(EndOfLine start, EndOfLine end)
{
var total = start.AbsolutePosition - end.AbsolutePosition;
if (end.IsWindowsEndOfLine)
{
total -= 1;
}
return (int)total;
}
/// <summary>
/// ストリームに書き込み
/// </summary>
/// <param name="stream">保存先ストリーム</param>
/// <param name="text">テキスト</param>
/// <param name="encoding">エンコード</param>
/// <param name="withBOM">BOMをつけるか？</param>
/// <param name="endOfLine">改行コード</param>
public static void SaveText(Stream stream, string text, Encoding encoding, bool withBOM, EndOfLine endOfLine)
{
if (withBOM)
{
byte[] preamble = encoding.GetPreamble();
stream.Write(preamble, 0, preamble.Length);
}
text = ConvertNewLineCharacter(text, endOfLine);
byte[] buf = encoding.GetBytes(text);
stream.Write(buf, 0, buf.Length);
}
/// <summary>
/// テキストファイルに保存。
/// </summary>
/// <param name="path">ファイル</param>
/// <param name="text">テキスト</param>
/// <param name="encoding">エンコード</param>
/// <param name="withBOM">BOMをつけるか？</param>
/// <param name="endOfLine">改行コード</param>
public static void SaveText(string path, string text, Encoding encoding, bool withBOM, EndOfLine endOfLine)
{
using (Stream fs = File.OpenWrite(path))
SaveText(fs, text, encoding, withBOM, endOfLine);
}
/// <summary>
/// テキストファイルに保存。
/// </summary>
/// <param name="path">ファイル</param>
/// <param name="text">テキスト</param>
/// <param name="encoding">エンコード</param>
/// <param name="withBOM">BOMをつけるか？</param>
/// <param name="endOfLine">改行コード</param>
public static void SaveText(string path, string text, Encoding encoding, bool withBOM, EndOfLine endOfLine)
{
using (Stream fs = File.OpenWrite(path))
SaveText(fs, text, encoding, withBOM, endOfLine);
}
/// <summary>
/// テキストファイルを読み込む(エンコード推測＋改行を\n単体に変換)。
/// </summary>
/// <param name="path">ファイル</param>
/// <param name="encoding">読み取りに使ったエンコード</param>
/// <param name="withBOM">BOMがあったか？</param>
/// <param name="eol">推測される改行コード</param>
public static string LoadText(string path, out Encoding encoding, out bool withBOM, out EndOfLine endOfLine)
{
using (Stream fs = File.OpenRead(path))
return LoadText(fs, out encoding, out withBOM, out endOfLine);
}
private string GetFromStart(EndOfLine eol)
{
var len = eol.RelativePosition;
if (eol.IsWindowsEndOfLine)
{
len -= 1;
}
var text = eol.Block.Data
.Take(len)
.ToArray();
return new string(text);
}
private string GetFromStartToEnd(EndOfLine start, EndOfLine end)
{
if (start.Block != end.Block)
throw new ArgumentException("start end end is not in the same block");
var len = end.RelativePosition;
if (end.IsWindowsEndOfLine)
{
len -= 1;
}
var text = start.Block.Data
.Skip(start.RelativePosition)
.Take(len)
.ToArray();
return new string(text);
}
/// <summary>
/// テキストファイルを読み込む(エンコード推測＋改行を\n単体に変換)。
/// </summary>
/// <param name="path">ファイル</param>
/// <param name="encoding">読み取りに使ったエンコード</param>
/// <param name="withBOM">BOMがあったか？</param>
/// <param name="eol">推測される改行コード</param>
public static string LoadText(string path, out Encoding encoding, out bool withBOM, out EndOfLine endOfLine)
{
using (Stream fs = File.OpenRead(path))
return(LoadText(fs, out encoding, out withBOM, out endOfLine));
}
/// <summary>
/// Nettoie les retours chariots indésirables (si si ! il y en a des fois ^^ )
/// </summary>
void CleanCRLF()
{
if (FTextIdentifier == string.Empty) return;
string sFile = string.Empty;
//bool shrinked = false;
//int idx_shrink = 1;
string sep = FTextIdentifier + FColumnSeparator + FTextIdentifier;
FEndOfLine = GetEndOfLine();
FileInfo fi = new FileInfo(FFilename);
long fileSize = fi.Length;
FileStream fs;
/*
if (fileSize > MAXSIZEFILEBEFORESHRINK)
{
ShrinkFile(FFilename, FEndOfLine);
shrinked = true;
}
*/
//while (true)
//{
string filename/*;
if (shrinked)
filename = FFilename + idx_shrink.ToString();
else
filename */= FFilename;
if (File.Exists(filename))
{
fs = fi.OpenRead();
byte[] fileData = new byte[256]; // buffer
int lBytes = fs.Read(fileData, 0, fileData.Length);
string aBloc;
while (lBytes != 0)
{
aBloc = Encoding.ASCII.GetString(fileData).Trim('\0');
if ((FEndOfLine == EndOfLine.CR) || (FEndOfLine == EndOfLine.CRLF))
{
// On vire les LF
aBloc = aBloc.Replace("\n", "");
}
sFile += aBloc;
lBytes = fs.Read(fileData, 0, fileData.Length);
}
// On remplace les intersections de colonne par un char 255
sFile = sFile.Replace(sep, ((char)255).ToString());
if ((FEndOfLine == EndOfLine.CR) || (FEndOfLine == EndOfLine.CRLF))
{
// On remplace le couple TextIdentifier + CR par un char 254
sFile = sFile.Replace(FTextIdentifier + "\r", ((char)254).ToString());
// On vire les CR
sFile = sFile.Replace("\r", "");
if (FEndOfLine == EndOfLine.CRLF)
{
// On remet les fins de ligne
sFile = sFile.Replace(((char)254).ToString(), FTextIdentifier + "\r\n");
}
else
{
// On remet les fins de ligne
sFile = sFile.Replace(((char)254).ToString(), FTextIdentifier + "\r");
}
}
else
{
// On remplace le couple TextIdentifier + LF par un char 254
sFile = sFile.Replace(FTextIdentifier + "\n", ((char)254).ToString());
// On vire les LF
sFile = sFile.Replace("\n", "");
// On remet les fins de ligne
sFile = sFile.Replace(((char)254).ToString(), FTextIdentifier + "\n");
}
// On remet les intersections de ligne
sFile = sFile.Replace(((char)255).ToString(), sep);
fs.Close();
/*
using (FileStream fs = new FileStream(filename, FileMode.OpenOrCreate))
{
sFile = fs.ReadToEnd();
if ((FEndOfLine == EndOfLine.CR) || (FEndOfLine == EndOfLine.CRLF))
{
// On vire les LF
sFile = sFile.Replace("\n", "");
}
// On remplace les intersections de colonne par un char 255
sFile = sFile.Replace(sep, ((char)255).ToString());
if ((FEndOfLine == EndOfLine.CR) || (FEndOfLine == EndOfLine.CRLF))
{
// On remplace le couple TextIdentifier + CR par un char 254
sFile = sFile.Replace(FTextIdentifier + "\r", ((char)254).ToString());
// On vire les CR
sFile = sFile.Replace("\r", "");
if (FEndOfLine == EndOfLine.CRLF)
{
// On remet les fins de ligne
sFile = sFile.Replace(((char)254).ToString(), FTextIdentifier + "\r\n");
}
else
{
// On remet les fins de ligne
sFile = sFile.Replace(((char)254).ToString(), FTextIdentifier + "\r");
}
}
else
{
// On remplace le couple TextIdentifier + LF par un char 254
sFile = sFile.Replace(FTextIdentifier + "\n", ((char)254).ToString());
// On vire les LF
sFile = sFile.Replace("\n", "");
// On remet les fins de ligne
sFile = sFile.Replace(((char)254).ToString(), FTextIdentifier + "\n");
}
// On remet les intersections de ligne
sFile = sFile.Replace(((char)255).ToString(), sep);
sr.Close();
}
if (shrinked)
idx_shrink++;
else
break;
*/
// On réécrit le fichier
using (StreamWriter sw = new StreamWriter(filename, false, FEncoding))
{
sw.Write(sFile);
sw.Flush();
sw.Close();
}
}
//else break;
//}
//if (shrinked) UnshrinkFile(FFilename);
}
void ShrinkFile(string filename, EndOfLine eol)
{
int idx = 1;
using (StreamReader sr = new StreamReader(filename, Encoding.Default))
{
for (string Line = sr.ReadLine(); Line != null; Line = sr.ReadLine())
{
if (File.Exists(filename + idx.ToString()))
{
Line = Line + (((eol == EndOfLine.CRLF) ? "\r\n" : (eol == EndOfLine.CR) ? "\r" : "\n"));
FileInfo fi = new FileInfo(filename + idx.ToString());
if ((fi.Length + Line.Length) > MAXSIZEFILEBEFORESHRINK) idx++;
}
using (StreamWriter sw = new StreamWriter(filename + idx.ToString(), true, Encoding.Default))
{
sw.Write(Line);
}
}
}
}
private IEnumerable<EndOfLine> FindLineEnds(DataBlock block)
{
var currentResult = new EndOfLine(block);
for(int currentPos = 0; currentPos < block.Length; currentPos++)
{
if (block.Data[currentPos] == EndOfLine.LineFeed)
{
currentResult.RelativePosition = currentPos;
yield return currentResult;
currentResult = new EndOfLine(block);
}
if (block.Data[currentPos] == EndOfLine.CarriageReturn)
{
currentResult.IsWindowsEndOfLine = true;
}
}
yield break;
}
/// <summary>
/// ストリームからテキストを読み込む(エンコード推測＋改行を\n単体に変換)。
/// </summary>
/// <param name="stream">読み取り元ストリーム</param>
/// <param name="encoding">読み取りに使ったエンコード</param>
/// <param name="withBOM">BOMがあったか？</param>
/// <param name="eol">推測される改行コード</param>
/// <returns></returns>
public static string LoadText(Stream stream, out Encoding encoding, out bool withBOM, out EndOfLine eol)
{
byte[] buf = new byte[stream.Length];
stream.Read(buf, 0, buf.Length);
encoding = GuessEncode(buf);
string text = encoding.GetString(buf);
eol = GuessFileFormat(text);
text = ConvertNewLineCharacter(text, EndOfLine.LF);
withBOM = text[0] == 0xFEFF; // BOMで始まっているか？
if (withBOM)
{
text = text.Substring(1); //BOMは消す。
}
return(text);
}
/// <summary>
/// Convert all line endings in the document to one mode.
/// </summary>
public virtual void ConvertEOLs(EndOfLine eolMode)
{
this.SendMessageDirect(2029, (int)eolMode);
}
/// <summary>
/// Renvoit les characteres correspondants au <c>EndOfLine</c> passé en parametre
/// </summary>
/// <param name="eol">le <c>EndOfLine</c></param>
/// <returns>Les characteres correspondants</returns>
public static string GetEndOfLine(EndOfLine eol)
{
switch (eol)
{
case EndOfLine.CRLF:
return "\r\n";
case EndOfLine.LF:
return "\n";
default:
return "\r\n";
}
}
/// <summary>
/// ストリームからテキストを読み込む(エンコード推測＋改行を\n単体に変換)。
/// </summary>
/// <param name="stream">読み取り元ストリーム</param>
/// <param name="encoding">読み取りに使ったエンコード</param>
/// <param name="withBOM">BOMがあったか？</param>
/// <param name="eol">推測される改行コード</param>
/// <returns></returns>
public static string LoadText(Stream stream, out Encoding encoding, out bool withBOM, out EndOfLine eol)
{
byte[] buf = new byte[stream.Length];
stream.Read(buf, 0, buf.Length);
encoding = GuessEncode(buf);
string text = encoding.GetString(buf);
eol = GuessFileFormat(text);
text = ConvertNewLineCharacter(text, EndOfLine.LF);
withBOM = text[0] == 0xFEFF; // BOMで始まっているか？
if (withBOM)
text = text.Substring(1); //BOMは消す。
return text;
}
/// <summary>
/// ソースコードを解析してトークン列を生成
/// </summary>
/// <param name="targets"></param>
/// <returns></returns>
private static IEnumerable <Token> Analysis(string targets)
{
if (targets.EndsWith(";") == false || targets.EndsWith("\r\n") == false)
{
targets += "\r\n";
}
while (targets.Length > 0)
{
//スペースとタブは変換しない
if (targets[0] == ' ' || targets[0] == '\t')
{
targets = targets.Remove(0, 1);
continue;
}
if (targets[0] == '\"')
{
var literal = targets
.Skip(1)
.TakeWhile(c => c != '\"')
.Aggregate(new StringBuilder(), (sb, c) => sb.Append(c))
.ToString();
targets = targets.Remove(0, literal.Length + 2);
yield return(new StringLiteral(literal));
continue;
}
if (LeftParenthesis.IsMatch(targets[0]))
{
targets = targets.Remove(0, 1);
yield return(new LeftParenthesis());
continue;
}
if (RightParenthesis.IsMatch(targets[0]))
{
targets = targets.Remove(0, 1);
yield return(new RightParenthesis());
continue;
}
if (BlockBegin.IsMatch(targets[0]))
{
targets = targets.Remove(0, 1);
yield return(new BlockBegin());
continue;
}
if (BlockEnd.IsMatch(targets[0]))
{
targets = targets.Remove(0, 1);
yield return(new BlockEnd());
continue;
}
if (targets.StartsWith("Writeln"))
{
targets = targets.Remove(0, 7);
yield return(new StandardOutput());
continue;
}
if (targets.StartsWith("Readln()"))
{
targets = targets.Remove(0, 8);
yield return(new StandardInput());
continue;
}
if (Int32Literal.SpecifiedCollection.Contains(targets[0]))
{
var value = targets.TakeWhile(c => Int32Literal.SpecifiedCollection.Contains(c)).ToArray();
targets = targets.Remove(0, value.Length);
yield return(new Int32Literal(new string(value)));
continue;
}
if (EndOfLine.IsMatch(targets[0].ToString()) || EndOfLine.IsMatch(new string(targets.Take(2).ToArray())))
{
var value = targets.TakeWhile(i => i == ';' || i == '\r' || i == '\n').Count();
targets = targets.Remove(0, value);
yield return(new EndOfLine());
continue;
}
if (targets.Length > 1 && Assign.IsMatch(targets[0]) && !Assign.IsMatch(targets[1]))
{
targets = targets.Remove(0, 1);
yield return(new Assign());
continue;
}
if (targets.Length > 1 && AddAssign.IsMatch(targets[0], targets[1]))
{
targets = targets.Remove(0, 2);
yield return(new AddAssign());
continue;
}
if (targets.StartsWith("var "))
{
var varVariableName = targets.Skip(4)
.TakeWhile(c => c != ' ' && c != '\t' && c != '=')
.Aggregate(new StringBuilder(), (sb, c) => sb.Append(c))
.ToString();
if (!(JudgeWord(varVariableName) is Identifier))
{
throw new Exception("変数にキーワードが使われています。");
}
targets = targets.Remove(0, 4 + varVariableName.Length);
yield return(new VarDeclaration(varVariableName));
continue;
}
if (PlusOperator.IsMatch(targets[0]))
{
targets = targets.Remove(0, 1);
yield return(new PlusOperator());
continue;
}
if (MinusOperator.IsMatch(targets[0]))
{
targets = targets.Remove(0, 1);
yield return(new MinusOperator());
continue;
}
if (MultiplyOperator.IsMatch(targets[0]))
{
targets = targets.Remove(0, 1);
yield return(new MultiplyOperator());
continue;
}
if (DivideOperator.IsMatch(targets[0]))
{
targets = targets.Remove(0, 1);
yield return(new DivideOperator());
continue;
}
if (ModuloOperator.IsMatch(targets[0]))
{
targets = targets.Remove(0, 1);
yield return(new ModuloOperator());
continue;
}
if (targets.Length > 1 && targets[0] == '=' && targets[1] == '=')
{
targets = targets.Remove(0, 2);
yield return(new EqualOperator());
continue;
}
if (targets.Length > 1 && targets[0] == '!' && targets[1] == '=')
{
targets = targets.Remove(0, 2);
yield return(new NotEqualOperator());
continue;
}
if (targets.Length > 1 && targets[0] == '>')
{
if (targets[1] == '=')
{
targets = targets.Remove(0, 2);
yield return(new GreaterThanOrEqualOperator());
continue;
}
targets = targets.Remove(0, 1);
yield return(new GreaterThanOperator());
continue;
}
if (targets.Length > 1 && targets[0] == '<')
{
if (targets[1] == '=')
{
targets = targets.Remove(0, 2);
yield return(new LessThanOrEqualOperator());
continue;
}
targets = targets.Remove(0, 1);
yield return(new LessThanOperator());
continue;
}
if (targets.Length > 1 && targets[0] == '&' && targets[1] == '&')
{
targets = targets.Remove(0, 2);
yield return(new AndAlsoOperator());
continue;
}
if (targets.Length > 1 && targets[0] == '|' && targets[1] == '|')
{
targets = targets.Remove(0, 2);
yield return(new OrElseOperator());
continue;
}
var word = targets.TakeWhile(IdentifierTake)
.Aggregate(new StringBuilder(), (sb, c) => sb.Append(c))
.ToString();
if (word.Length == 0)
{
throw new Exception("変数名なのに文字数がゼロ");
}
targets = targets.Remove(0, word.Length);
yield return(JudgeWord(word));
}
}
