public void TestOnLogFileProduced()
{
string logFile = Path.Combine(Path.GetTempPath(), GetRandomFileName());
int maxFileSize = 1; // max file size of 1 means every line will go to a new file
int messageCount = 5; // number of messages to log
var reportedLogFiles = new List <string>();
using (var log = new CsvFileLog(logFile, new[] { CsvFileLog.ColumnKind.Message }, maxFileSize: maxFileSize))
{
log.OnLogFileProduced += (path) => reportedLogFiles.Add(path);
for (int i = 0; i < messageCount; i++)
{
log.Write(TestTimestamp, TestThreadId, TestSeverity, i.ToString());
}
}
// assert that a log file was reported for each line
reportedLogFiles.Count.Should().Be(messageCount);
// assert the content of each log file
for (int i = 0; i < messageCount; i++)
{
var expected = $"\"{i}\"{Environment.NewLine}";
File.ReadAllText(reportedLogFiles[i]).Should().BeEquivalentTo(expected);
}
}
private string RenderMessage(CsvFileLog log, string message)
{
var sb = new StringBuilder();
log.RenderMessage(sb, TestTimestamp, TestThreadId, TestSeverity, message);
return(sb.ToString());
}
public void TestRenderSchema(CsvFileLog.ColumnKind column, int columnIndex, int columnCount)
{
string logFile = Path.Combine(Path.GetTempPath(), GetRandomFileName());
// schema:
// <empty>,...,<empty>,<column>,<empty>,...,<empty>
// where
// - <column> is at position 'columnIndex'
// - total number of columns is 'columnCount'
var schema =
Repeat(CsvFileLog.ColumnKind.Empty, columnIndex - 1)
.Concat(new[] { column })
.Concat(Repeat(CsvFileLog.ColumnKind.Empty, columnCount - columnIndex - 1));
using (var log = new CsvFileLog(logFile, schema, serviceName: "CsvFileLogTests"))
{
var actual = RenderMessage(log, TestMessage);
var expected = string.Join(",", schema
.Select(col => RenderColumn(log, col, TestMessage))
.Select(str => '"' + str + '"'));
actual.Should().BeEquivalentTo(expected);
}
IEnumerable <CsvFileLog.ColumnKind> Repeat(CsvFileLog.ColumnKind col, int count)
{
return(Enumerable.Range(0, Math.Max(0, count)).Select(_ => col));
}
}
Logger()
{
int limit;
int.TryParse(ConfigurationManager.AppSettings["Log Limit"], out limit);
string logTypeValue = ConfigurationManager.AppSettings["Log Type"];
switch (logTypeValue)
{
case "CSV":
Log = new CsvFileLog(limit);
break;
case "Encrypted CSV":
Log = new EncryptedCsvFileLog(limit);
break;
case "Event Log":
Log = new EventLog();
break;
default:
throw new NoLogDefinedException("The log file is not defined in the config file");
}
}
public void TestDontRenderConstColumns(string schema, string expectedFileSchema, string expectedConstSchema)
{
var constCol = CsvFileLog.ColumnKind.BuildId;
var dynCol = CsvFileLog.ColumnKind.Message;
var csvLog = new CsvFileLog(
GetRandomLogFile(),
schema: TranslateSchema(schema),
renderConstColums: false);
csvLog.IsConstValueColumn(constCol).Should().BeTrue();
csvLog.IsConstValueColumn(dynCol).Should().BeFalse();
csvLog.FileSchema.Should().BeEquivalentTo(TranslateSchema(expectedFileSchema));
csvLog.ConstSchema.Should().BeEquivalentTo(TranslateSchema(expectedConstSchema));
CsvFileLog.ColumnKind[] TranslateSchema(string s)
{
return(s
.Split(',')
.Select(c => c.Trim())
.Where(c => !string.IsNullOrEmpty(c))
.Select(TranslateCol)
.ToArray());
}
CsvFileLog.ColumnKind TranslateCol(string c)
{
Assert.True(c == "D" || c == "C", $"Column specified must be either 'C' or 'D', but is '{c}'");
return(c == "D" ? dynCol : constCol);
}
}
[InlineData("1\n3", "\"1\n3\"")] // a new line may appear in a message
public void TestRenderMessageColumn(string message, string expected)
{
string logFile = Path.Combine(Path.GetTempPath(), GetRandomFileName());
using (var log = new CsvFileLog(logFile, new[] { CsvFileLog.ColumnKind.Message }))
{
var actual = RenderMessage(log, message);
actual.Should().BeEquivalentTo(expected);
}
}
private static async Task WithLoggerAsync(Func <ILogger, Task> action, string?logFilePath)
{
CsvFileLog?csvFileLog = null;
ConsoleLog?consoleLog = null;
Logger? logger = null;
try
{
if (!string.IsNullOrEmpty(logFilePath))
{
// Needed to satisfy the type checker
Contract.AssertNotNull(logFilePath);
if (string.IsNullOrEmpty(Path.GetDirectoryName(logFilePath)))
{
var cwd = Directory.GetCurrentDirectory();
logFilePath = Path.Combine(cwd, logFilePath);
}
csvFileLog = new CsvFileLog(logFilePath, new List <CsvFileLog.ColumnKind>()
{
CsvFileLog.ColumnKind.PreciseTimeStamp,
CsvFileLog.ColumnKind.ProcessId,
CsvFileLog.ColumnKind.ThreadId,
CsvFileLog.ColumnKind.LogLevel,
CsvFileLog.ColumnKind.LogLevelFriendly,
CsvFileLog.ColumnKind.Message,
});
}
consoleLog = new ConsoleLog(useShortLayout: false, printSeverity: true);
var logs = new ILog?[] { csvFileLog, consoleLog };
logger = new Logger(logs.Where(log => log != null).Cast <ILog>().ToArray());
await action(logger);
}
finally
{
logger?.Dispose();
csvFileLog?.Dispose();
consoleLog?.Dispose();
}
}
Logger()
{
int limit;
int.TryParse(ConfigurationManager.AppSettings["Log Limit"], out limit);
string logTypeValue = ConfigurationManager.AppSettings["Log Type"];
switch (logTypeValue)
{
case "CSV":
Log = new CsvFileLog(limit);
break;
case "Encrypted CSV":
Log = new EncryptedCsvFileLog(limit);
break;
case "Event Log":
Log = new EventLog();
break;
default:
throw new NoLogDefinedException("The log file is not defined in the config file");
}
}
private void EnableRemoteTelemetryIfNeeded(string logFilePath)
{
if (!_enableRemoteTelemetry)
{
return;
}
if (_kustoUploader == null)
{
_logger.Warning
(
"Remote telemetry flag is enabled but no Kusto connection string was found in environment variable '{0}'",
KustoConnectionStringEnvVarName
);
return;
}
_csvFileLog = new CsvFileLog
(
logFilePath: logFilePath + TmpCsvLogFileExt,
serviceName: ServiceName,
schema: KustoTableCsvSchema,
severity: _fileLogSeverity,
maxFileSize: _csvLogMaxFileSize
);
// Every time a log file written to disk and closed, we rename it and upload it to Kusto.
// The last log file will be produced when _csvFileLog is disposed, so _kustUploader better
// not be disposed before _csvFileLog.
_csvFileLog.OnLogFileProduced += (path) =>
{
string newPath = Path.ChangeExtension(path, CsvLogFileExt);
File.Move(path, newPath);
_kustoUploader.PostFileForUpload(newPath, _csvFileLog.BuildId);
};
_logger.AddLog(_csvFileLog);
_logger.Always("Remote telemetry enabled");
}
private void EnableRemoteTelemetryIfNeeded(string logFilePath)
{
if (!_enableRemoteTelemetry)
{
return;
}
var kustoConnectionString = Environment.GetEnvironmentVariable(KustoConnectionStringEnvVarName);
if (string.IsNullOrWhiteSpace(kustoConnectionString))
{
_logger.Warning
(
"Remote telemetry flag is enabled but no Kusto connection string was found in environment variable '{0}'",
KustoConnectionStringEnvVarName
);
return;
}
_csvFileLog = new CsvFileLog
(
logFilePath: logFilePath + TmpCsvLogFileExt,
serviceName: ServiceName,
schema: KustoTableCsvSchema,
renderConstColums: false,
severity: _fileLogSeverity,
maxFileSize: _csvLogMaxFileSize
);
var indexedColumns = _csvFileLog.FileSchema.Select((col, idx) => new CsvColumnMapping {
ColumnName = col.ToString(), Ordinal = idx
});
var constColumns = _csvFileLog.ConstSchema.Select(col => new CsvColumnMapping {
ColumnName = col.ToString(), ConstValue = _csvFileLog.RenderConstColumn(col)
});
var csvMapping = indexedColumns.Concat(constColumns).ToArray();
var csvMappingStr = string.Join("", csvMapping.Select(col => $"{Environment.NewLine} Name: '{col.ColumnName}', ConstValue: '{col.ConstValue}', Ordinal: {col.Ordinal}"));
_logger.Always("Using csv mapping:{0}", csvMappingStr);
_kustoUploader = new KustoUploader
(
kustoConnectionString,
database: KustoDatabase,
table: KustoTable,
csvMapping: csvMapping,
deleteFilesOnSuccess: true,
checkForIngestionErrors: true,
log: _consoleLog
);
// Every time a log file written to disk and closed, we rename it and upload it to Kusto.
// The last log file will be produced when _csvFileLog is disposed, so _kustUploader better
// not be disposed before _csvFileLog.
_csvFileLog.OnLogFileProduced += (path) =>
{
string newPath = Path.ChangeExtension(path, CsvLogFileExt);
File.Move(path, newPath);
_kustoUploader.PostFileForUpload(newPath, _csvFileLog.BuildId);
};
_logger.AddLog(_csvFileLog);
_logger.Always("Remote telemetry enabled");
}
private string RenderColumn(CsvFileLog log, CsvFileLog.ColumnKind col, string message)
{
return(log.RenderColumn(col, TestTimestamp, TestThreadId, TestSeverity, message));
}
public void TestParseTableSchema(string schema, CsvFileLog.ColumnKind[] expected)
{
var actual = CsvFileLog.ParseTableSchema(schema);
actual.Should().BeEquivalentTo(expected);
}
