public void Newline_should_match(string text)
{
// arrange
var buffer = text.ToSinglyLinked();
// act
var(isMatch, start, end) = Newline()(buffer);
// assert
isMatch.ShouldBeTrue();
start.Head.ShouldBe(text[0]);
end.ShouldBeNull();
}
private static Parser <SyntaxToken, JassGlobalDeclarationSyntax> GetGlobalDeclarationParser()
{
return(Keyword(SyntaxTokenType.ConstantKeyword).Then(Map((type, id, value) => (IVariableDeclarator) new JassVariableDeclaratorSyntax(type, id, value), TypeParser, IdentifierNameParser, EqualsValueClauseParser))
.Or(VariableDeclaratorParser)
.Before(Newline.AtLeastOnce())
.Select(declarator => new JassGlobalDeclarationSyntax(declarator)));
}
private static Parser <SyntaxToken, IDeclaration> GetDeclarationParser()
{
return(OneOf(
TypeDeclarationParser,
GlobalDeclarationListParser,
NativeFunctionDeclarationParser)
.Before(Newline.AtLeastOnce()));
}
public void CrLfConvert(string srcstr, string expectedstr, Newline nl)
{
var srcenc = Encoding.UTF8;
var destenc = Encoding.UTF8;
using (var instm = new MemoryStream(srcenc.GetBytes(srcstr)))
using (var outstm = new MemoryStream())
{
dotnet_text_encoder.TextConverter.ConvertStream(instm, srcenc, outstm, destenc, true, nl);
var actual = destenc.GetString(outstm.ToArray());
Assert.Equal(expectedstr, actual);
}
}
static bool WriteBytesToStream(Stream output, ReadOnlySpan <char> input, Newline nl, bool prevcr, Encoding outputEncoding, ref byte[] wbuf)
{
int off = 0;
Span <byte> wspan = wbuf.AsSpan();
for (int i = 0; i < input.Length; i++)
{
if (prevcr)
{
if (input[i] == LfValue)
{
// CRLF has come
WriteNewline(output, nl, CrLf.AsSpan());
off = i + 1;
prevcr = false;
continue;
}
else
{
// single CR has come
WriteNewline(output, nl, Cr.AsSpan());
prevcr = false;
}
}
if (input[i] == CrValue)
{
// CR has come, but cannot decide to CRLF or single CR,
var sp = input.Slice(off, i - off);
wspan = EncodeAndWriteStream(sp, output, wspan, nl, outputEncoding, ref wbuf);
off = i + 1;
prevcr = true;
}
else if (input[i] == LfValue)
{
// single LF has come
var sp = input.Slice(off, i - off);
wspan = EncodeAndWriteStream(sp, output, wspan, nl, outputEncoding, ref wbuf);
WriteNewline(output, nl, Lf.AsSpan());
off = i + 1;
}
}
// write remaining buffer
if (off < input.Length)
{
var sp = input.Slice(off);
EncodeAndWriteStream(sp, output, wspan, nl, outputEncoding, ref wbuf);
}
return(prevcr);
}
public void CrLfConvertRepeat(string srcstr, string expectedstr, Newline nl)
{
var original = new string(Enumerable.Range(0, 1024).SelectMany(i => srcstr).ToArray());
var expected = new string(Enumerable.Range(0, 1024).SelectMany(i => expectedstr).ToArray());
var srcenc = Encoding.Unicode;
var destenc = Encoding.UTF8;
using (var instm = new MemoryStream(srcenc.GetBytes(original)))
using (var outstm = new MemoryStream())
{
dotnet_text_encoder.TextConverter.ConvertStream(instm, srcenc, outstm, destenc, true, nl);
var actual = destenc.GetString(outstm.ToArray());
Assert.Equal(expected, actual);
}
}
public void LongStringCrLf(Newline nl, string originalnl, string expectednl)
{
var len = 1024;
var linenum = 10;
var rnd = new Random();
var srcstr = new string(Enumerable.Range(0, linenum).SelectMany((idx) =>
Enumerable.Range(0, len).Select(x => (char)rnd.Next((int)'a', (int)'z')).Concat(originalnl)).ToArray())
;
var expected = srcstr.Replace(originalnl, expectednl);
var srcenc = Encoding.UTF8;
var destenc = Encoding.UTF8;
using (var instm = new MemoryStream(srcenc.GetBytes(srcstr)))
using (var outstm = new MemoryStream())
{
dotnet_text_encoder.TextConverter.ConvertStream(instm, srcenc, outstm, destenc, true, nl);
var actual = destenc.GetString(outstm.ToArray());
Assert.Equal(expected, actual);
}
}
static void WriteNewline(Stream stm, Newline nl, ReadOnlySpan <byte> defaultValue)
{
switch (nl)
{
case Newline.Cr:
stm.Write(Cr.AsSpan());
break;
case Newline.Lf:
stm.Write(Lf.AsSpan());
break;
case Newline.Crlf:
stm.Write(CrLf.AsSpan());
break;
default:
stm.Write(defaultValue);
break;
}
}
public Task Run(string cimpath, string project, IEnumerable <string> files, bool dynamic)
{
return(Task.Run(() =>
{
foreach (var s in files)
{
var si = new ProcessStartInfo(cimpath + $"\\clie.exe", $"import \"{s}\"{(dynamic?" -y":"")}");
si.UseShellExecute = false;
si.CreateNoWindow = true;
si.RedirectStandardOutput = true;
si.WorkingDirectory = project + "\\master";
si.EnvironmentVariables.Add("PRCNAM", "SymbolFrontend");
si.EnvironmentVariables.Add("PROJECT", project.Substring(project.LastIndexOf("\\") + 1) + ".gef");
si.EnvironmentVariables.Add("SITE_ROOT", project + "\\");
Newline?.Invoke(this, $"Import souboru {Path.GetFileName(s)}...");
try
{
using (var p = Process.Start(si))
{
using (StreamReader reader = p.StandardOutput)
{
while (!reader.EndOfStream)
{
string result = reader.ReadLine();
Newline?.Invoke(this, result);
}
}
}
}
catch (Exception ex)
{
Newline?.Invoke(this, ex.Message);
}
}
}));
}
public override string ToString()
{
string name = "";
if (Type == Encoding.UTF7)
{
name = "UTF-7";
}
else if (Type == Encoding.UTF8)
{
name = "UTF-8";
}
else if (Type == Encoding.Unicode)
{
name = "UTF-16 LE";
}
else if (Type == Encoding.BigEndianUnicode)
{
name = "UTF-16 BE";
}
else if (Type == Encoding.UTF32)
{
name = "UTF-32";
}
else if (Type == Encoding.Default)
{
name = Type.WebName;
}
if (Bom)
{
name += " BOM";
}
name += $" ({Newline.ToString()})";
return(name);
}
public static void ConvertStream(Stream input, Encoding inputEncoding, Stream output, Encoding outputEncoding, bool noPreamble, Newline nl = Newline.None)
{
using (var sr = new StreamReader(input, inputEncoding))
{
var buf = ArrayPool <char> .Shared.Rent(4096);
bool first = true;
var wbuf = ArrayPool <byte> .Shared.Rent(4096);
bool prevcr = false;
try
{
while (true)
{
var charread = sr.Read(buf, 0, buf.Length);
if (charread == 0)
{
break;
}
if (first && !noPreamble)
{
var pre = outputEncoding.GetPreamble();
if (pre != null && pre.Length != 0)
{
output.Write(pre, 0, pre.Length);
}
}
first = false;
prevcr = WriteBytesToStream(output, buf.AsSpan(0, charread), nl, prevcr, outputEncoding, ref wbuf);
}
if (prevcr)
{
// treat last cr
WriteNewline(output, nl, Cr.AsSpan());
}
}
finally
{
ArrayPool <char> .Shared.Return(buf);
ArrayPool <byte> .Shared.Return(wbuf);
}
}
}
static Span <byte> EncodeAndWriteStream(ReadOnlySpan <char> sp, Stream output, Span <byte> wspan, Newline nl, Encoding outputEncoding, ref byte[] wbuf)
{
var wlen = outputEncoding.GetByteCount(sp);
if (wlen > wspan.Length)
{
wspan = Span <byte> .Empty;
ArrayPool <byte> .Shared.Return(wbuf);
wbuf = ArrayPool <byte> .Shared.Rent(wlen);
wspan = wbuf.AsSpan();
}
outputEncoding.GetBytes(sp, wspan);
var tmp = wspan.Slice(0, wlen);
output.Write(tmp);
return(wspan);
}
public static void NewlineParserIncorrect()
=> AssertThat(() => Newline.Parse("xyz")).ThrowsExactlyException <ParseException>();
public static void NewlineCorrect(string input) => AssertThat(Newline.Parse(input)).IsEqualTo(input);
/*
* private Token attributes() { Attribute tok = new Attribute(null, lineno);
* var matcher = JadeScanner.getMatcherForPattern("^\\("); if
* (matcher.find(0)) { consume(matcher[0].Length); attributeMode = true; } else
* { return null; }
*
* StringBuilder sb = new StringBuilder(); String regexp =
* "^[, ]*?([-_\\w]+)? *?= *?(\"[^\"]*?\"|'[^']*?'|[.-_\\w]+)"; matcher =
* JadeScanner.getMatcherForPattern(regexp); if (matcher.find(0)) { while
* (matcher.find(0)) { String name = matcher[0].Groups[1].Value; String value =
* matcher[0].Groups[2].Value; tok.addAttribute(name, value);
* sb.append(matcher.group(0)); consume(matcher[0].Length); matcher =
* JadeScanner.getMatcherForPattern(regexp); } tok.setValue(sb.toString()); }
* else { return null; }
*
* matcher = JadeScanner.getMatcherForPattern("^ *?\\)"); if (matcher.find(0)) {
* consume(matcher[0].Length); attributeMode = false; } else { throw new
* JadeLexerException
* ("Error while parsing attribute. Missing closing bracket ", filename,
* getLineno(), JadeScanner.getInput()); } return tok; }
*/
private Token indent()
{
MatchCollection matcher;
String re;
if (indentRe != null)
{
matcher = _jadeScanner.getMatcherForPattern(indentRe);
}
else
{
// tabs
re = "^\\n(\\t*) *";
String indentType = "tabs";
matcher = _jadeScanner.getMatcherForPattern(re);
// spaces
if (matcher.Count > 0 && matcher[0].Groups[1].Value.Length == 0)
{
re = "^\\n( *)";
indentType = "spaces";
matcher = _jadeScanner.getMatcherForPattern(re);
}
// established
if (matcher.Count > 0 && matcher[0].Groups[1].Value.Length > 0)
{
this.indentRe = re;
}
this.indentType = indentType;
}
if (matcher.Count > 0 && matcher[0].Success && matcher[0].Groups.Count > 0)
{
Token tok;
int indents = matcher[0].Groups[1].Value.Length;
if (lastIndents <= 0 && indents > 0)
{
lastIndents = indents;
}
lineno++;
consume(indents + 1);
if ((indents > 0 && lastIndents > 0 && indents % lastIndents != 0) || _jadeScanner.isIntendantionViolated())
{
throw new JadeLexerException("invalid indentation; expecting " + indents + " " + indentType, filename, getLineno(), templateLoader);
}
// blank line
if (_jadeScanner.isBlankLine())
{
return(new Newline("newline", lineno));
}
// outdent
if (indentStack.Count > 0 && indents < indentStack.First.Value)
{
while (indentStack.Count > 0 && indentStack.First.Value > indents)
{
indentStack.poll();
}
stash.AddLast(new Outdent("outdent", lineno));
tok = this.stash.pollLast();
// indent
}
else if (indents > 0 && (indentStack.Count < 1 || indents != indentStack.First.Value))
{
indentStack.AddLast(indents);
tok = new Indent("indent", lineno);
tok.setIndents(indents);
// newline
}
else
{
tok = new Newline("newline", lineno);
}
return(tok);
}
return(null);
}
private static Parser <SyntaxToken, JassCompilationUnitSyntax> GetCompilationUnitParser()
{
return(Newline.Many().Then(DeclarationParser.Many()).Then(FunctionParser.Many(),
(declarations, functions) => new JassCompilationUnitSyntax(declarations.ToImmutableArray(), functions.ToImmutableArray()))
.Before(End));
}
private static Parser <SyntaxToken, IDeclaration> GetGlobalDeclarationListParser()
{
return(Keyword(SyntaxTokenType.GlobalsKeyword).Then(Newline.AtLeastOnce()).Then(GlobalDeclarationParser.Many()).Before(Keyword(SyntaxTokenType.EndglobalsKeyword))
.Select <IDeclaration>(globals => new JassGlobalDeclarationListSyntax(globals.ToImmutableArray())));
}
