public async Task Test_FileProcessor_FrontMatter()
{
var root = "/a/b/c";
var optionsMock = new SiteInfo();
var loggerMock = new Mock <ILogger <FileProcessor> >();
var fileProviderMock = new Mock <IFileProvider>()
.SetupFileProviderMock(
root,
new List <FakeDirectory>()
{
new FakeDirectory(string.Empty, new FakeFile[] {
// File without any frontmatter
new FakeFile("a.txt", Encoding.UTF8.GetBytes(string.Empty)),
// File with empty frontmatter
new FakeFile("b.txt", Encoding.UTF8.GetBytes(
CreateFrontMatter()
)),
// File with frontmatter but only new keys
new FakeFile("c.txt", Encoding.UTF8.GetBytes(
CreateFrontMatter(new Dictionary <string, object> {
{ "tags", new string[] { "A" } }
})
)),
// File writh frontmatter, but overwritten keys
new FakeFile("d.txt", Encoding.UTF8.GetBytes(
CreateFrontMatter(new Dictionary <string, object> {
})
)),
/*new FakeFile("2021-01-01-article.txt", Encoding.UTF8.GetBytes(
* CreateFrontMatter(new Dictionary<string, object>{
* { "PublishedTime", "18:00" }
* })
* ))*/
})
}
);
var metadataProviderMock = new YamlFrontMatterMetadataProvider(new YamlParser());
var fileSystem = new FileSystem(fileProviderMock.Object);
var fileMetadataParserMock = new FileMetadataParser(new Mock <ILogger <FileMetadataParser> >().Object, metadataProviderMock, new MetadataParserOptions());
var sut = new FileProcessor(fileSystem, loggerMock.Object, new IContentPreprocessorStrategy[] { }, optionsMock, fileMetadataParserMock);
var result = await sut.Process(new FileFilterCriteria
{
DirectoriesToSkip = new string[] { },
FileExtensionsToTarget = new string[] { ".txt" }
});
result.Should().NotBeEmpty();
result.Count().Should().Be(4);
var fileA = result.Single(x => x.Name.Equals("a.txt"));
var fileB = result.Single(x => x.Name.Equals("b.txt"));
var fileC = result.Single(x => x.Name.Equals("c.txt"));
var fileD = result.Single(x => x.Name.Equals("d.txt"));
fileA.MetaData.Count.Should().Be(1);
fileB.MetaData.Count.Should().Be(1);
fileC.MetaData.Count.Should().Be(2, "Default keys + one new keys equals 2");
fileD.MetaData.Count.Should().Be(1);
}
public void Test_FilemetadataParser_EmptyFileWithoutConfigOnlyGetsDefaultValues()
{
// Arange
var optionsMock = new MetadataParserOptions();
var metadataProviderMock = new Mock <IMetadataProvider>();
metadataProviderMock
.Setup(x => x.Retrieve <FileMetaData>(It.Is <string>(p => p.Equals(string.Empty))))
.Returns(new Metadata <FileMetaData> {
});
var loggerMock = new LoggerMock <FileMetadataParser>();
var sut = new FileMetadataParser(loggerMock.Object, metadataProviderMock.Object, optionsMock);
var criteria = new MetadataCriteria
{
Content = string.Empty,
FileName = "file.html"
};
// Act
var result = sut.Parse(criteria);
// Assert
result.Should().NotBeNull();
result.Data.Should().NotBeNull();
result.Data.Count.Should().Be(1, "Only URI is added by default");
result.Data.Uri.Should().NotBeNull();
result.Data.Uri.Should().Be("file.html");
}
public async Task Test_FileProcessor_Subdirectories()
{
var root = "/a/b/c";
var optionsMock = new SiteInfo();
var loggerMock = new Mock <ILogger <FileProcessor> >();
var fileProviderMock = new Mock <IFileProvider>()
.SetupFileProviderMock(
root,
new List <FakeDirectory>()
{
new FakeDirectory("_subdir", new FakeFile[] {
new FakeFile(Path.Combine(root, "test.txt"))
})
});
var metadataProviderMock = new YamlFrontMatterMetadataProvider(new YamlParser());
var fileSystem = new FileSystem(fileProviderMock.Object);
var fileMetadataParserMock = new FileMetadataParser(new Mock <ILogger <FileMetadataParser> >().Object, metadataProviderMock, new MetadataParserOptions());
var sut = new FileProcessor(fileSystem, loggerMock.Object, new IContentPreprocessorStrategy[] { }, optionsMock, fileMetadataParserMock);
var result = await sut.Process(new FileFilterCriteria
{
DirectoriesToSkip = new string[] { },
FileExtensionsToTarget = new string[] { ".txt" }
});
result.Should().NotBeEmpty();
result.Count().Should().Be(1);
var testFile = result.Single(x => x.Name.Equals("test.txt"));
//testFile.MetaData.Collection.Should().Be("subdir");
}
public async Task Test_FileProcessor_ChangedFileExtension()
{
var root = "/a/b/c";
var optionsMock = new SiteInfo();
var loggerMock = new Mock <ILogger <FileProcessor> >();
var fileProviderMock = new Mock <IFileProvider>()
.SetupFileProviderMock(
root,
new List <FakeDirectory>()
{
new FakeDirectory(string.Empty, new FakeFile[] {
new FakeFile("test.md")
})
});
var metadataProviderMock = new YamlFrontMatterMetadataProvider(new YamlParser());
var fileSystem = new FileSystem(fileProviderMock.Object);
var fileMetadataParserMock = new FileMetadataParser(new Mock <ILogger <FileMetadataParser> >().Object, metadataProviderMock,
new MetadataParserOptions()
{
ExtensionMapping = new Dictionary <string, string> {
{ ".md", ".html" }
}
}
);
var sut = new FileProcessor(fileSystem, loggerMock.Object, new IContentPreprocessorStrategy[] { }, optionsMock, fileMetadataParserMock);
var result = await sut.Process(new FileFilterCriteria
{
DirectoriesToSkip = new string[] { },
FileExtensionsToTarget = new string[] { ".md" }
});
result.Should().NotBeEmpty();
result.Count().Should().Be(1);
var testFile = result.Single(x => x.Name.Equals("test.html"));
}
public async Task Test_FileProcessor_WithFilter_Should_ReturnMatchingFiles()
{
var root = "/a/b/c";
var optionsMock = new SiteInfo();
var loggerMock = new Mock <ILogger <FileProcessor> >();
var fileProviderMock = new Mock <IFileProvider>()
.SetupFileProviderMock(
root,
new List <FakeDirectory>()
{
new FakeDirectory(string.Empty, new FakeFile[] {
new FakeFile("index.html"),
new FakeFile("other.png")
})
}
);
var metadataProviderMock = new YamlFrontMatterMetadataProvider(new YamlParser());
var fileSystem = new FileSystem(fileProviderMock.Object);
var fileMetadataParserMock = new FileMetadataParser(new Mock <ILogger <FileMetadataParser> >().Object, metadataProviderMock, new MetadataParserOptions());
var sut = new FileProcessor(fileSystem, loggerMock.Object, new IContentPreprocessorStrategy[] { }, optionsMock, fileMetadataParserMock);
var result = await sut.Process(new FileFilterCriteria
{
DirectoriesToSkip = new string[] { },
FileExtensionsToTarget = new string[] { ".html" }
});
result.Should().NotBeEmpty();
result.Count().Should().Be(1);
}
public void Test_FilemetadataParser_EmptyFileIfMultipleConfigurationsApplyLastOneWins()
{
// Arange
var options = new MetadataParserOptions
{
Defaults = new DefaultMetadatas {
new DefaultMetadata {
Path = string.Empty,
Values = new FileMetaData {
Layout = "default.html"
}
},
new DefaultMetadata {
Path = "test",
Values = new FileMetaData {
Layout = "other.html",
Collection = "test"
}
}
}
};
var metadataProviderMock = new Mock <IMetadataProvider>();
metadataProviderMock
.Setup(x => x.Retrieve <FileMetaData>(It.Is <string>(p => p.Equals(string.Empty))))
.Returns(new Metadata <FileMetaData> {
});
var loggerMock = new LoggerMock <FileMetadataParser>();
var sut = new FileMetadataParser(loggerMock.Object, metadataProviderMock.Object, options);
var criteria = new MetadataCriteria
{
Content = string.Empty,
FileName = Path.Combine("test", "file.html"),
Permalink = Path.Combine("test", ":name:ext")
};
// Act
var result = sut.Parse(criteria);
// Assert
result.Should().NotBeNull();
result.Data.Should().NotBeNull();
result.Data.Count.Should().Be(3, "Defaults = 1 + Applied Config = 2, Makes 3 values");
result.Data.Uri.Should().NotBeNull();
result.Data.Uri.Should().Be(Path.Combine("test", "file.html"));
result.Data.Layout.Should().NotBeNull();
result.Data.Layout.Should().Be("other.html");
result.Data.Collection.Should().NotBeNull();
result.Data.Collection.Should().Be("test");
}
public void Test_FilemetadataParser_MultipleLayers()
{
// Arange
var options = new MetadataParserOptions
{
Defaults = new DefaultMetadatas {
new DefaultMetadata {
Path = string.Empty,
Values = new FileMetaData {
Layout = "default.html"
}
},
new DefaultMetadata {
Path = "test",
Values = new FileMetaData {
Collection = "test"
}
}
}
};
var metadataProviderMock = new Mock <IMetadataProvider>();
metadataProviderMock
.Setup(x => x.Retrieve <FileMetaData>(It.Is <string>(p => p.Equals(string.Empty))))
.Returns(new Metadata <FileMetaData> {
});
var loggerMock = new Mock <ILogger <FileMetadataParser> >();
var sut = new FileMetadataParser(loggerMock.Object, metadataProviderMock.Object, options);
var criteria = new MetadataCriteria
{
Content = string.Empty,
FileName = "posts/2021/file.html",
Permalink = "posts/2021/:name:ext"
};
// Act
var result = sut.Parse(criteria);
// Assert
result.Should().NotBeNull();
result.Data.Should().NotBeNull();
result.Data.Count.Should().Be(2, "Defaults = 1 + Applied Config = 1, Makes 2 values");
result.Data.Uri.Should().NotBeNull();
result.Data.Uri.Should().Be("posts/2021/file.html");
result.Data.Layout.Should().NotBeNull();
result.Data.Layout.Should().Be("default.html");
}
private void PrepareFile(DataSet ds, bool append)
{
if (append)
{
if (!_output.CanSeek)
{
throw new IOException("destination stream must be seekable for append operations.");
}
ValidateFile();
Thrift.FileMetaData fileMeta = ReadMetadata();
_meta.SetMeta(fileMeta);
Schema existingSchema = new FileMetadataParser(fileMeta).ParseSchema(_formatOptions);
if (!ds.Schema.Equals(existingSchema))
{
throw new ParquetException($"{nameof(DataSet)} schema does not match existing file schema");
}
GoBeforeFooter();
}
else
{
if (_existingSchema == null)
{
_existingSchema = ds.Schema;
//file starts with magic
WriteMagic();
_meta.AddSchema(ds);
}
else
{
if (!_existingSchema.Equals(ds.Schema))
{
throw new ParquetException($"expeted schema {_existingSchema} but found {ds.Schema}.");
}
}
}
}
public async Task Test_FileProcessor_WithoutFiles_Should_ReturnEmptyList()
{
var root = "/a/b/c";
var optionsMock = new SiteInfo();
var loggerMock = new Mock <ILogger <FileProcessor> >();
var fileProviderMock = new Mock <IFileProvider>()
.SetupFileProviderMock(
root,
new List <FakeDirectory>()
{
}
);
var metadataProviderMock = new YamlFrontMatterMetadataProvider(new YamlParser());
var fileSystem = new FileSystem(fileProviderMock.Object);
var fileMetadataParserMock = new FileMetadataParser(new Mock <ILogger <FileMetadataParser> >().Object, metadataProviderMock, new MetadataParserOptions());
var sut = new FileProcessor(fileSystem, loggerMock.Object, new IContentPreprocessorStrategy[] { }, optionsMock, fileMetadataParserMock);
var result = await sut.Process(new FileFilterCriteria
{
DirectoriesToSkip = new string[] { },
FileExtensionsToTarget = new string[] { }
});
result.Should().BeEmpty();
}
/// <summary>
/// Test read, to be defined
/// </summary>
public DataSet Read()
{
_readerOptions.Validate();
_meta = ReadMetadata();
var metaParser = new FileMetadataParser(_meta);
Schema schema = metaParser.ParseSchema(_formatOptions);
var pathToValues = new Dictionary <string, IList>();
long pos = 0;
long rowsRead = 0;
foreach (Thrift.RowGroup rg in _meta.Row_groups)
{
//check whether to skip RG completely
if ((_readerOptions.Count != -1 && rowsRead >= _readerOptions.Count) ||
(_readerOptions.Offset > pos + rg.Num_rows - 1))
{
pos += rg.Num_rows;
continue;
}
long offset = Math.Max(0, _readerOptions.Offset - pos);
long count = _readerOptions.Count == -1 ? rg.Num_rows : Math.Min(_readerOptions.Count - rowsRead, rg.Num_rows);
for (int icol = 0; icol < rg.Columns.Count; icol++)
{
Thrift.ColumnChunk cc = rg.Columns[icol];
SchemaElement se = schema[cc];
var p = new ColumnReader(cc, se, _input, ThriftStream, _formatOptions);
try
{
IList chunkValues = p.Read(offset, count);
if (!pathToValues.TryGetValue(se.Path, out IList allValues))
{
pathToValues[se.Path] = chunkValues;
}
else
{
allValues.AddRange(chunkValues);
}
if (icol == 0)
{
//todo: this may not work
rowsRead += chunkValues.Count;
}
}
catch (Exception ex)
{
throw new ParquetException($"fatal error reading column '{se}'", ex);
}
}
pos += rg.Num_rows;
}
return(new DataSet(schema, pathToValues, _meta.Num_rows, _meta.Created_by));
}
