private string PreprocessTemplate(string template)
{
if (options.IncludeResolver == null)
{
return(template);
}
Regex includePattern = new Regex("\\{include:([^\\}]+)\\}",
RegexOptions.Multiline | RegexOptions.CultureInvariant);
return(includePattern.Replace(template, delegate(Match include)
{
string includedScript = string.Empty;
if (include.Groups.Count == 2)
{
string includePath = include.Groups[1].Value;
IStreamSource includeSource = options.IncludeResolver.Resolve(includePath);
if (includeSource != null)
{
Stream includeStream = includeSource.GetStream();
StreamReader reader = new StreamReader(includeStream);
includedScript = reader.ReadToEnd();
includeSource.CloseStream(includeStream);
}
}
return includedScript;
}));
}
private void ProcessIncludeInstruction(string instructionLine, bool skipPreprocessing)
{
Debug.Assert(_includeStream == null);
Debug.Assert(_includeTextReader == null);
bool included = false;
int beginQuoteIndex = instructionLine.IndexOf('"');
int endQuoteIndex = 0;
if (beginQuoteIndex > 0)
{
endQuoteIndex = instructionLine.IndexOf('"', beginQuoteIndex + 1);
}
if (endQuoteIndex > 0)
{
Exception resolveException = null;
string includePath = instructionLine.Substring(beginQuoteIndex + 1, endQuoteIndex - beginQuoteIndex - 1);
try {
_include = _includeResolver.ResolveInclude(_source, includePath);
if (_include != null)
{
_includeStream = _include.GetStream();
if (_includeStream != null)
{
_includeTextReader = new StreamReader(_includeStream);
_includeLineCounter = 0;
if (skipPreprocessing)
{
string includeText = _includeTextReader.ReadToEnd();
_skipPreprocessingWriter.Write(includeText);
_include.CloseStream(_includeStream);
_includeTextReader = null;
_includeStream = null;
included = true;
}
}
}
}
catch (Exception e) {
resolveException = e;
}
if ((included == false) && (_includeTextReader == null))
{
RaiseError("Unable to resolve or open included file '" + includePath + "'", resolveException);
}
}
else
{
RaiseError("Invalid #include instruction.");
}
}
private char[] GetBuffer(IStreamSource source) {
char[] buffer = null;
Stream stream = source.GetStream();
if (stream != null) {
StreamReader reader = new StreamReader(stream);
string text = reader.ReadToEnd();
buffer = text.ToCharArray();
source.CloseStream(stream);
}
return buffer;
}
public void Save(ITweetDTO dto)
{
var textItem = cleanup.Cleanup(dto.Text);
var text = $"{dto.CreatedAt}\t{dto.Id}\t{dto.CreatedBy.Id}\t{dto.Retweeted}\t{textItem}\r\n";
lock (syncRoot)
{
var stream = streamSource.GetStream();
var data = Encoding.UTF8.GetBytes(text);
stream.Write(data, 0, data.Length);
stream.Flush();
}
}
private string GetMarkup(IStreamSource source) {
string markup = null;
Stream stream = source.GetStream();
try {
StreamReader reader = new StreamReader(stream);
markup = reader.ReadToEnd();
}
finally {
if (stream != null) {
source.CloseStream(stream);
stream = null;
}
}
return markup;
}
private char[] GetBuffer(IStreamSource source)
{
char[] buffer = null;
Stream stream = source.GetStream();
if (stream != null)
{
StreamReader reader = new StreamReader(stream);
string text = reader.ReadToEnd();
buffer = text.ToCharArray();
source.CloseStream(stream);
}
return(buffer);
}
private string GetMarkup(IStreamSource source)
{
string markup = null;
Stream stream = source.GetStream();
try {
StreamReader reader = new StreamReader(stream);
markup = reader.ReadToEnd();
}
finally {
if (stream != null)
{
source.CloseStream(stream);
stream = null;
}
}
return(markup);
}
private void SaveInternal(ITweet message, double?sentiment)
{
var text = message.Text.Replace("\r\n", " ").Replace("\n", " ");
lock (syncRoot)
{
var stream = streamSource.GetStream();
using (var streamOut = new StreamWriter(stream, Encoding.UTF8, 4096, true))
using (var csvDataTarget = new CsvWriter(streamOut))
{
csvDataTarget.Configuration.Delimiter = "\t";
csvDataTarget.WriteField(message.CreatedAt);
csvDataTarget.WriteField(message.Id);
csvDataTarget.WriteField(message.CreatedBy.Id);
string type = "Message";
if (message.RetweetedTweet != null)
{
csvDataTarget.WriteField(message.RetweetedTweet.Id);
csvDataTarget.WriteField(message.RetweetedTweet.CreatedBy.Id);
type = "Retweet";
}
else
{
csvDataTarget.WriteField(message.QuotedTweet?.Id);
csvDataTarget.WriteField(message.QuotedTweet?.CreatedBy.Id);
if (message.QuotedTweet != null)
{
type = "Quote";
}
}
csvDataTarget.WriteField(type);
csvDataTarget.WriteField(message.Language);
csvDataTarget.WriteField(sentiment);
csvDataTarget.WriteField(text);
csvDataTarget.NextRecord();
streamOut.Flush();
}
stream.Flush();
}
}
public void Save(ITweetDTO tweet)
{
if (tweet == null)
{
throw new ArgumentNullException(nameof(tweet));
}
try
{
var data = new RawTweetData();
data.Data = Encoding.UTF8.GetBytes(tweet.ToJson()).Zip();
lock (syncRoot)
{
var stream = streamSource.GetStream();
Serializer.SerializeWithLengthPrefix(stream, data, PrefixStyle.Base128, 1);
stream.Flush();
}
}
catch (Exception ex)
{
log.LogError(ex, "Error");
}
}
private void ProcessIncludeInstruction(string instructionLine, bool skipPreprocessing) {
Debug.Assert(_includeStream == null);
Debug.Assert(_includeTextReader == null);
bool included = false;
int beginQuoteIndex = instructionLine.IndexOf('"');
int endQuoteIndex = 0;
if (beginQuoteIndex > 0) {
endQuoteIndex = instructionLine.IndexOf('"', beginQuoteIndex + 1);
}
if (endQuoteIndex > 0) {
Exception resolveException = null;
string includePath = instructionLine.Substring(beginQuoteIndex + 1, endQuoteIndex - beginQuoteIndex - 1);
try {
_include = _includeResolver.ResolveInclude(_source, includePath);
if (_include != null) {
_includeStream = _include.GetStream();
if (_includeStream != null) {
_includeTextReader = new StreamReader(_includeStream);
_includeLineCounter = 0;
if (skipPreprocessing) {
string includeText = _includeTextReader.ReadToEnd();
_skipPreprocessingWriter.Write(includeText);
_include.CloseStream(_includeStream);
_includeTextReader = null;
_includeStream = null;
included = true;
}
}
}
}
catch (Exception e) {
resolveException = e;
}
if ((included == false) && (_includeTextReader == null)) {
RaiseError("Unable to resolve or open included file '" + includePath + "'", resolveException);
}
}
else {
RaiseError("Invalid #include instruction.");
}
}
