public async Task Test_FileProcessor_FrontMatter() { var root = "/a/b/c"; var optionsMock = new SiteInfo(); var loggerMock = new Mock <ILogger <FileProcessor> >(); var fileProviderMock = new Mock <IFileProvider>() .SetupFileProviderMock( root, new List <FakeDirectory>() { new FakeDirectory(string.Empty, new FakeFile[] { // File without any frontmatter new FakeFile("a.txt", Encoding.UTF8.GetBytes(string.Empty)), // File with empty frontmatter new FakeFile("b.txt", Encoding.UTF8.GetBytes( CreateFrontMatter() )), // File with frontmatter but only new keys new FakeFile("c.txt", Encoding.UTF8.GetBytes( CreateFrontMatter(new Dictionary <string, object> { { "tags", new string[] { "A" } } }) )), // File writh frontmatter, but overwritten keys new FakeFile("d.txt", Encoding.UTF8.GetBytes( CreateFrontMatter(new Dictionary <string, object> { }) )), /*new FakeFile("2021-01-01-article.txt", Encoding.UTF8.GetBytes( * CreateFrontMatter(new Dictionary<string, object>{ * { "PublishedTime", "18:00" } * }) * ))*/ }) } ); var metadataProviderMock = new YamlFrontMatterMetadataProvider(new YamlParser()); var fileSystem = new FileSystem(fileProviderMock.Object); var fileMetadataParserMock = new FileMetadataParser(new Mock <ILogger <FileMetadataParser> >().Object, metadataProviderMock, new MetadataParserOptions()); var sut = new FileProcessor(fileSystem, loggerMock.Object, new IContentPreprocessorStrategy[] { }, optionsMock, fileMetadataParserMock); var result = await sut.Process(new FileFilterCriteria { DirectoriesToSkip = new string[] { }, FileExtensionsToTarget = new string[] { ".txt" } }); result.Should().NotBeEmpty(); result.Count().Should().Be(4); var fileA = result.Single(x => x.Name.Equals("a.txt")); var fileB = result.Single(x => x.Name.Equals("b.txt")); var fileC = result.Single(x => x.Name.Equals("c.txt")); var fileD = result.Single(x => x.Name.Equals("d.txt")); fileA.MetaData.Count.Should().Be(1); fileB.MetaData.Count.Should().Be(1); fileC.MetaData.Count.Should().Be(2, "Default keys + one new keys equals 2"); fileD.MetaData.Count.Should().Be(1); }
public void Test_FilemetadataParser_EmptyFileWithoutConfigOnlyGetsDefaultValues() { // Arange var optionsMock = new MetadataParserOptions(); var metadataProviderMock = new Mock <IMetadataProvider>(); metadataProviderMock .Setup(x => x.Retrieve <FileMetaData>(It.Is <string>(p => p.Equals(string.Empty)))) .Returns(new Metadata <FileMetaData> { }); var loggerMock = new LoggerMock <FileMetadataParser>(); var sut = new FileMetadataParser(loggerMock.Object, metadataProviderMock.Object, optionsMock); var criteria = new MetadataCriteria { Content = string.Empty, FileName = "file.html" }; // Act var result = sut.Parse(criteria); // Assert result.Should().NotBeNull(); result.Data.Should().NotBeNull(); result.Data.Count.Should().Be(1, "Only URI is added by default"); result.Data.Uri.Should().NotBeNull(); result.Data.Uri.Should().Be("file.html"); }
public async Task Test_FileProcessor_Subdirectories() { var root = "/a/b/c"; var optionsMock = new SiteInfo(); var loggerMock = new Mock <ILogger <FileProcessor> >(); var fileProviderMock = new Mock <IFileProvider>() .SetupFileProviderMock( root, new List <FakeDirectory>() { new FakeDirectory("_subdir", new FakeFile[] { new FakeFile(Path.Combine(root, "test.txt")) }) }); var metadataProviderMock = new YamlFrontMatterMetadataProvider(new YamlParser()); var fileSystem = new FileSystem(fileProviderMock.Object); var fileMetadataParserMock = new FileMetadataParser(new Mock <ILogger <FileMetadataParser> >().Object, metadataProviderMock, new MetadataParserOptions()); var sut = new FileProcessor(fileSystem, loggerMock.Object, new IContentPreprocessorStrategy[] { }, optionsMock, fileMetadataParserMock); var result = await sut.Process(new FileFilterCriteria { DirectoriesToSkip = new string[] { }, FileExtensionsToTarget = new string[] { ".txt" } }); result.Should().NotBeEmpty(); result.Count().Should().Be(1); var testFile = result.Single(x => x.Name.Equals("test.txt")); //testFile.MetaData.Collection.Should().Be("subdir"); }
public async Task Test_FileProcessor_ChangedFileExtension() { var root = "/a/b/c"; var optionsMock = new SiteInfo(); var loggerMock = new Mock <ILogger <FileProcessor> >(); var fileProviderMock = new Mock <IFileProvider>() .SetupFileProviderMock( root, new List <FakeDirectory>() { new FakeDirectory(string.Empty, new FakeFile[] { new FakeFile("test.md") }) }); var metadataProviderMock = new YamlFrontMatterMetadataProvider(new YamlParser()); var fileSystem = new FileSystem(fileProviderMock.Object); var fileMetadataParserMock = new FileMetadataParser(new Mock <ILogger <FileMetadataParser> >().Object, metadataProviderMock, new MetadataParserOptions() { ExtensionMapping = new Dictionary <string, string> { { ".md", ".html" } } } ); var sut = new FileProcessor(fileSystem, loggerMock.Object, new IContentPreprocessorStrategy[] { }, optionsMock, fileMetadataParserMock); var result = await sut.Process(new FileFilterCriteria { DirectoriesToSkip = new string[] { }, FileExtensionsToTarget = new string[] { ".md" } }); result.Should().NotBeEmpty(); result.Count().Should().Be(1); var testFile = result.Single(x => x.Name.Equals("test.html")); }
public async Task Test_FileProcessor_WithFilter_Should_ReturnMatchingFiles() { var root = "/a/b/c"; var optionsMock = new SiteInfo(); var loggerMock = new Mock <ILogger <FileProcessor> >(); var fileProviderMock = new Mock <IFileProvider>() .SetupFileProviderMock( root, new List <FakeDirectory>() { new FakeDirectory(string.Empty, new FakeFile[] { new FakeFile("index.html"), new FakeFile("other.png") }) } ); var metadataProviderMock = new YamlFrontMatterMetadataProvider(new YamlParser()); var fileSystem = new FileSystem(fileProviderMock.Object); var fileMetadataParserMock = new FileMetadataParser(new Mock <ILogger <FileMetadataParser> >().Object, metadataProviderMock, new MetadataParserOptions()); var sut = new FileProcessor(fileSystem, loggerMock.Object, new IContentPreprocessorStrategy[] { }, optionsMock, fileMetadataParserMock); var result = await sut.Process(new FileFilterCriteria { DirectoriesToSkip = new string[] { }, FileExtensionsToTarget = new string[] { ".html" } }); result.Should().NotBeEmpty(); result.Count().Should().Be(1); }
public void Test_FilemetadataParser_EmptyFileIfMultipleConfigurationsApplyLastOneWins() { // Arange var options = new MetadataParserOptions { Defaults = new DefaultMetadatas { new DefaultMetadata { Path = string.Empty, Values = new FileMetaData { Layout = "default.html" } }, new DefaultMetadata { Path = "test", Values = new FileMetaData { Layout = "other.html", Collection = "test" } } } }; var metadataProviderMock = new Mock <IMetadataProvider>(); metadataProviderMock .Setup(x => x.Retrieve <FileMetaData>(It.Is <string>(p => p.Equals(string.Empty)))) .Returns(new Metadata <FileMetaData> { }); var loggerMock = new LoggerMock <FileMetadataParser>(); var sut = new FileMetadataParser(loggerMock.Object, metadataProviderMock.Object, options); var criteria = new MetadataCriteria { Content = string.Empty, FileName = Path.Combine("test", "file.html"), Permalink = Path.Combine("test", ":name:ext") }; // Act var result = sut.Parse(criteria); // Assert result.Should().NotBeNull(); result.Data.Should().NotBeNull(); result.Data.Count.Should().Be(3, "Defaults = 1 + Applied Config = 2, Makes 3 values"); result.Data.Uri.Should().NotBeNull(); result.Data.Uri.Should().Be(Path.Combine("test", "file.html")); result.Data.Layout.Should().NotBeNull(); result.Data.Layout.Should().Be("other.html"); result.Data.Collection.Should().NotBeNull(); result.Data.Collection.Should().Be("test"); }
public void Test_FilemetadataParser_MultipleLayers() { // Arange var options = new MetadataParserOptions { Defaults = new DefaultMetadatas { new DefaultMetadata { Path = string.Empty, Values = new FileMetaData { Layout = "default.html" } }, new DefaultMetadata { Path = "test", Values = new FileMetaData { Collection = "test" } } } }; var metadataProviderMock = new Mock <IMetadataProvider>(); metadataProviderMock .Setup(x => x.Retrieve <FileMetaData>(It.Is <string>(p => p.Equals(string.Empty)))) .Returns(new Metadata <FileMetaData> { }); var loggerMock = new Mock <ILogger <FileMetadataParser> >(); var sut = new FileMetadataParser(loggerMock.Object, metadataProviderMock.Object, options); var criteria = new MetadataCriteria { Content = string.Empty, FileName = "posts/2021/file.html", Permalink = "posts/2021/:name:ext" }; // Act var result = sut.Parse(criteria); // Assert result.Should().NotBeNull(); result.Data.Should().NotBeNull(); result.Data.Count.Should().Be(2, "Defaults = 1 + Applied Config = 1, Makes 2 values"); result.Data.Uri.Should().NotBeNull(); result.Data.Uri.Should().Be("posts/2021/file.html"); result.Data.Layout.Should().NotBeNull(); result.Data.Layout.Should().Be("default.html"); }
private void PrepareFile(DataSet ds, bool append) { if (append) { if (!_output.CanSeek) { throw new IOException("destination stream must be seekable for append operations."); } ValidateFile(); Thrift.FileMetaData fileMeta = ReadMetadata(); _meta.SetMeta(fileMeta); Schema existingSchema = new FileMetadataParser(fileMeta).ParseSchema(_formatOptions); if (!ds.Schema.Equals(existingSchema)) { throw new ParquetException($"{nameof(DataSet)} schema does not match existing file schema"); } GoBeforeFooter(); } else { if (_existingSchema == null) { _existingSchema = ds.Schema; //file starts with magic WriteMagic(); _meta.AddSchema(ds); } else { if (!_existingSchema.Equals(ds.Schema)) { throw new ParquetException($"expeted schema {_existingSchema} but found {ds.Schema}."); } } } }
public async Task Test_FileProcessor_WithoutFiles_Should_ReturnEmptyList() { var root = "/a/b/c"; var optionsMock = new SiteInfo(); var loggerMock = new Mock <ILogger <FileProcessor> >(); var fileProviderMock = new Mock <IFileProvider>() .SetupFileProviderMock( root, new List <FakeDirectory>() { } ); var metadataProviderMock = new YamlFrontMatterMetadataProvider(new YamlParser()); var fileSystem = new FileSystem(fileProviderMock.Object); var fileMetadataParserMock = new FileMetadataParser(new Mock <ILogger <FileMetadataParser> >().Object, metadataProviderMock, new MetadataParserOptions()); var sut = new FileProcessor(fileSystem, loggerMock.Object, new IContentPreprocessorStrategy[] { }, optionsMock, fileMetadataParserMock); var result = await sut.Process(new FileFilterCriteria { DirectoriesToSkip = new string[] { }, FileExtensionsToTarget = new string[] { } }); result.Should().BeEmpty(); }
/// <summary> /// Test read, to be defined /// </summary> public DataSet Read() { _readerOptions.Validate(); _meta = ReadMetadata(); var metaParser = new FileMetadataParser(_meta); Schema schema = metaParser.ParseSchema(_formatOptions); var pathToValues = new Dictionary <string, IList>(); long pos = 0; long rowsRead = 0; foreach (Thrift.RowGroup rg in _meta.Row_groups) { //check whether to skip RG completely if ((_readerOptions.Count != -1 && rowsRead >= _readerOptions.Count) || (_readerOptions.Offset > pos + rg.Num_rows - 1)) { pos += rg.Num_rows; continue; } long offset = Math.Max(0, _readerOptions.Offset - pos); long count = _readerOptions.Count == -1 ? rg.Num_rows : Math.Min(_readerOptions.Count - rowsRead, rg.Num_rows); for (int icol = 0; icol < rg.Columns.Count; icol++) { Thrift.ColumnChunk cc = rg.Columns[icol]; SchemaElement se = schema[cc]; var p = new ColumnReader(cc, se, _input, ThriftStream, _formatOptions); try { IList chunkValues = p.Read(offset, count); if (!pathToValues.TryGetValue(se.Path, out IList allValues)) { pathToValues[se.Path] = chunkValues; } else { allValues.AddRange(chunkValues); } if (icol == 0) { //todo: this may not work rowsRead += chunkValues.Count; } } catch (Exception ex) { throw new ParquetException($"fatal error reading column '{se}'", ex); } } pos += rg.Num_rows; } return(new DataSet(schema, pathToValues, _meta.Num_rows, _meta.Created_by)); }