public void GetAsyncQueryResolver_NoParams_CallsParamVersionWithUnmodifedQuery()
{
var mockExecuter = new Mock <IAsyncQueryResolver <TestEntity> >(MockBehavior.Strict);
mockExecuter
.Setup(e => e.AnyAsync(It.IsAny <Expression <Func <TestEntity, bool> > >(), It.IsAny <CancellationToken>()))
.ReturnsAsync(true);
var expectedPipeline = new ProcessorPipeline(null);
var mockSubject = new Mock <Repository <TestEntity> >(MockBehavior.Strict, expectedPipeline);
mockSubject
.Setup(r => r.GetAsyncQueryResolver(It.IsAny <Func <IQueryable <TestEntity>, IQueryable <TestEntity> > >()))
.Returns(mockExecuter.Object)
.Callback <Func <IQueryable <TestEntity>, IQueryable <TestEntity> > >(f =>
{
// Check to make sure the function just returns the same queryable, as-is.
var expectedQueryable = new List <TestEntity>().AsQueryable();
var result1 = f(expectedQueryable);
Assert.Same(expectedQueryable, result1);
});
var result = mockSubject.Object.GetAsyncQueryResolver();
Assert.Same(result, mockExecuter.Object);
}
public FsuEngine(IOutput output)
{
Output = output;
Tokenizer = new Tokenizer <TokenType>(TokenType.Invalid, TokenType.Eol, TokenType.Eof, false);
Tokenizer.OnDefinitionAdded += (sender, definition) => {
output.WriteLine(Level.Fine, $"Added Token: '&-e;{definition.Regex}&-^;' &-9;{definition.Precedence}&-^; (&-a;{definition.TokenType}&-^;)");
};
Parser = new TokenParser <TokenType, IProcessor>(TokenType.Comment, TokenType.Eol, TokenType.Eof, TokenType.Invalid);
Parser.OnGrammarAdded += (sender, grammer) => {
output.WriteLine(Level.Fine, $"Added Grammer: '&-b;{grammer.TriggerTokenValuePattern}&-^;' (&-a;{string.Join(", ", grammer.TriggerTokens)}&-^;) with {grammer.Rules.Count} rule(s)");
};
Parser.OnTokenError += (sender, token) => {
output.WriteLine(Level.Error, $"&-c;ERROR - Unexpected token: '{token.Value}' ({token.TokenType}) {token.Location}&-^;");
};
Parser.OnTokenParsing += (sender, token) => {
token.WriteLine(output, Level.Debug);
};
interpreter = new Interpreter <TokenType, IProcessor>(Tokenizer, Parser);
PropertyProviders = new PropertyProviderList(new BasicFilePropertyProvider());
PropertyStore = new PropertyStore();
Pipeline = new ProcessorPipeline(Output, PropertyProviders, PropertyStore, interpreter);
Init();
}
public void Ctor_()
{
var mockProcessorProvider = new Mock <IProcessorProvider>(MockBehavior.Strict);
var expectedPipeline = new ProcessorPipeline(mockProcessorProvider.Object);
var subject = new TestRepository(expectedPipeline);
Assert.Same(expectedPipeline, subject.TestGetInternalPipeline());
}
public async Task UpdateAsync_DoesNothingWhenNoChanges()
{
var expectedPipeline = new ProcessorPipeline(null);
var mockSubject = new Mock <Repository <TestEntity> >(MockBehavior.Strict, expectedPipeline);
var expectedEntity = new TestEntity();
await mockSubject.Object.UpdateAsync(expectedEntity, CancellationToken.None);
}
public async Task ProcessAsync_UsesProvider()
{
var expectedProcessors = new List <IProcessor <TestDataPackage> >();
var mockProvider = new Mock <IProcessorProvider>(MockBehavior.Strict);
mockProvider.Setup(p => p.GetProcessors <TestDataPackage>()).Returns(expectedProcessors);
var subject = new ProcessorPipeline(mockProvider.Object);
await subject.ProcessAsync(new TestDataPackage());
mockProvider.Verify(p => p.GetProcessors <TestDataPackage>(), Times.Once);
}
public async Task ProcessAsync_ExecutesProvidedProcessors()
{
var mock2Called = false;
var mockProcessor1 = new Mock <IProcessor <TestDataPackage> >(MockBehavior.Strict);
mockProcessor1.SetupGet(p => p.Stage).Returns(new NumericPriorityStage(0));
mockProcessor1
.Setup(p => p.ProcessAsync(It.IsAny <TestDataPackage>(), It.IsAny <CancellationToken>()))
.Returns(Task.CompletedTask)
.Callback(() => Assert.True(mock2Called));
var mockProcessor2 = new Mock <IProcessor <TestDataPackage> >(MockBehavior.Strict);
mockProcessor2.SetupGet(p => p.Stage).Returns(new NumericPriorityStage(100));
mockProcessor2
.Setup(p => p.ProcessAsync(It.IsAny <TestDataPackage>(), It.IsAny <CancellationToken>()))
.Returns(Task.CompletedTask)
.Callback <TestDataPackage, CancellationToken>((dp, ct) =>
{
Assert.Same(dp[ProcessorPipeline.ProcessorKey], mockProcessor2.Object);
mock2Called = true;
});
var expectedProcessors = new List <IProcessor <TestDataPackage> >
{
mockProcessor1.Object,
mockProcessor2.Object
};
var mockProvider = new Mock <IProcessorProvider>(MockBehavior.Strict);
mockProvider.Setup(p => p.GetProcessors <TestDataPackage>()).Returns(expectedProcessors);
var expectedData = new TestDataPackage();
var subject = new ProcessorPipeline(mockProvider.Object);
await subject.ProcessAsync(expectedData);
mockProcessor1.Verify(p => p.ProcessAsync(
It.Is <TestDataPackage>(v => ReferenceEquals(v, expectedData)),
It.IsAny <CancellationToken>()),
Times.Once);
mockProcessor2.Verify(p => p.ProcessAsync(
It.Is <TestDataPackage>(v => ReferenceEquals(v, expectedData)),
It.IsAny <CancellationToken>()),
Times.Once);
}
private static int Main(string[] args)
{
void Configuration(ParserSettings with)
{
with.EnableDashDash = true;
with.CaseInsensitiveEnumValues = true;
with.AutoHelp = true;
with.HelpWriter = null;
}
var parser = new Parser(Configuration);
var parsed = parser.ParseArguments <HtmlDumpOptions, JsonDumpOptions, MediaDumpOptions>(args);
var result = 0;
try
{
var resolveMedia = false;
BaseOptions baseOpts = null;
var processors = new List <IProcessor>();
parsed
.WithParsed <HtmlDumpOptions>(opts =>
{
processors.Add(new HtmlGeneratorProcessor(
opts.OutputDirectory,
!opts.DontIncludeCustomPosterNames,
!opts.DontUseColoredNames,
!opts.DontIncludeSameFagCount,
opts.Style,
opts.Prettify));
baseOpts = opts;
resolveMedia = !opts.DontResolveMedia;
})
.WithParsed <JsonDumpOptions>(opts =>
{
processors.Add(new JsonSerializerProcessor(opts.OutputDirectory, opts.Formatted));
baseOpts = opts;
resolveMedia = !opts.DontResolveMedia;
})
.WithParsed <MediaDumpOptions>(opts =>
{
baseOpts = opts;
resolveMedia = true;
})
.WithNotParsed(errs =>
{
var helpText = HelpText.AutoBuild(parsed, h =>
{
h = HelpText.DefaultParsingErrorsHandler(parsed, h);
h.Copyright = "";
h.Heading = "";
h.AutoVersion = false;
h.AddDashesToOption = true;
return(h);
}, e => e, true, 190);
Console.WriteAscii("Mitsuba Archivizer", Color.DodgerBlue);
Console.WriteFormatted("commit {0} @ {1}",
new Formatter(ThisAssembly.Git.Commit, Color.Crimson),
new Formatter(ThisAssembly.Git.Branch, Color.Chartreuse), Color.White);
Console.Write(helpText);
result = 1;
});
if (result != 1)
{
var postCount = 0;
var cursorBackupTop1 = -1;
var cursorBackupLeft1 = -1;
ThreadParser.OnPostCount += count => postCount = count;
ThreadParser.OnPostParsing += idx =>
{
if (cursorBackupTop1 == -1 || cursorBackupLeft1 == -1)
{
cursorBackupTop1 = Console.CursorTop;
cursorBackupLeft1 = Console.CursorLeft;
}
Console.SetCursorPosition(cursorBackupLeft1, cursorBackupTop1);
Console.Write(new string(' ', Console.WindowWidth));
Console.SetCursorPosition(cursorBackupLeft1, cursorBackupTop1);
Console.WriteLine("Parsing post: ({0}/{1})", idx + 1, postCount);
};
var resolver = new MediaResolver(
resolveMedia,
baseOpts.OutputDirectory,
!baseOpts.DontSeperateMediaByExtension,
!baseOpts.DontDownloadThumbnails,
baseOpts.AllowedMediaExtensions,
baseOpts.AllowedThumbnailExtensions);
var mediaCount = 0;
var cursorBackupTop2 = -1;
var cursorBackupLeft2 = -1;
resolver.OnPostWithMediaCount += count => mediaCount = count;
resolver.OnProcessingPostMedia += (post, idx) =>
{
if (cursorBackupTop2 == -1 || cursorBackupLeft2 == -1)
{
cursorBackupTop2 = Console.CursorTop;
cursorBackupLeft2 = Console.CursorLeft;
}
Console.SetCursorPosition(cursorBackupLeft2, cursorBackupTop2);
Console.Write(new string(' ', Console.WindowWidth));
Console.SetCursorPosition(cursorBackupLeft2, cursorBackupTop2);
Console.WriteLine("Downloading media: {0} ({1}/{2})", post.File.FileName, idx + 1, mediaCount);
};
var pipeline = new ProcessorPipeline(processors, resolver);
pipeline.OnProcessorInvoked += name => Console.WriteLine("{0} is working...", name);
foreach (var input in baseOpts.InputSeq)
{
var thread = ThreadParser.TryParse(input).Result;
pipeline.Process(thread).Wait();
}
Console.WriteLine("Done!");
}
}
catch (Exception ex)
{
Console.Error.WriteLine(ex);
result = 1;
}
return(result);
}
public TestRepository(ProcessorPipeline processorPipeline)
: base(processorPipeline)
{
}
public async Task AddAsync_PipelineExecuted()
{
var addingCalled = false;
var addedCalled = false;
var executeAddCalled = false;
var mockProcessorAdding = new Mock <IProcessor <EntityAddingPackage <TestEntity> > >(MockBehavior.Strict);
mockProcessorAdding.SetupGet(p => p.Stage).Returns(new NumericPriorityStage(0));
mockProcessorAdding
.Setup(p => p.ProcessAsync(It.IsAny <EntityAddingPackage <TestEntity> >(), It.IsAny <CancellationToken>()))
.Returns(Task.CompletedTask)
.Callback(() =>
{
Assert.False(addedCalled);
Assert.False(executeAddCalled);
addingCalled = true;
});
var mockProcessorAdded = new Mock <IProcessor <EntityAddedPackage <TestEntity> > >(MockBehavior.Strict);
mockProcessorAdded.SetupGet(p => p.Stage).Returns(new NumericPriorityStage(0));
mockProcessorAdded
.Setup(p => p.ProcessAsync(It.IsAny <EntityAddedPackage <TestEntity> >(), It.IsAny <CancellationToken>()))
.Returns(Task.CompletedTask)
.Callback(() =>
{
Assert.True(addingCalled);
Assert.True(executeAddCalled);
addedCalled = true;
});
var mockProcessorProvider = new Mock <IProcessorProvider>(MockBehavior.Strict);
mockProcessorProvider
.Setup(p => p.GetProcessors <EntityAddingPackage <TestEntity> >())
.Returns(new List <IProcessor <EntityAddingPackage <TestEntity> > > {
mockProcessorAdding.Object
});
mockProcessorProvider
.Setup(p => p.GetProcessors <EntityAddedPackage <TestEntity> >())
.Returns(new List <IProcessor <EntityAddedPackage <TestEntity> > > {
mockProcessorAdded.Object
});
var expectedPipeline = new ProcessorPipeline(mockProcessorProvider.Object);
var mockSubject = new Mock <Repository <TestEntity> >(MockBehavior.Strict, expectedPipeline);
mockSubject
.Protected()
.Setup <Task>("ExecuteAddAsync", ItExpr.IsAny <TestEntity>(), It.IsAny <CancellationToken>())
.Returns(Task.CompletedTask)
.Callback(() =>
{
Assert.True(addingCalled);
Assert.False(addedCalled);
executeAddCalled = true;
});
var expectedEntity = new TestEntity();
await mockSubject.Object.AddAsync(expectedEntity, CancellationToken.None);
mockProcessorProvider.Verify(p => p.GetProcessors <EntityAddingPackage <TestEntity> >(), Times.Once);
mockProcessorProvider.Verify(p => p.GetProcessors <EntityAddedPackage <TestEntity> >(), Times.Once);
mockProcessorAdding.Verify(p => p.ProcessAsync(
It.Is <EntityAddingPackage <TestEntity> >(v => ReferenceEquals(v.Entity, expectedEntity)),
It.IsAny <CancellationToken>()),
Times.Once());
mockProcessorAdded.Verify(p => p.ProcessAsync(
It.Is <EntityAddedPackage <TestEntity> >(v => ReferenceEquals(v.Entity, expectedEntity)),
It.IsAny <CancellationToken>()),
Times.Once());
mockSubject
.Protected()
.Verify("ExecuteAddAsync",
Times.Once(),
ItExpr.Is <TestEntity>(v => ReferenceEquals(v, expectedEntity)),
ItExpr.IsAny <CancellationToken>());
}
public async Task UpdateAsync_PipelineExecuted()
{
var updatingCalled = false;
var updatedCalled = false;
var executeUpdateCalled = false;
var mockProcessorUpdating = new Mock <IProcessor <EntityUpdatingPackage <TestEntity> > >(MockBehavior.Strict);
mockProcessorUpdating.SetupGet(p => p.Stage).Returns(new NumericPriorityStage(0));
mockProcessorUpdating
.Setup(p => p.ProcessAsync(It.IsAny <EntityUpdatingPackage <TestEntity> >(), It.IsAny <CancellationToken>()))
.Returns(Task.CompletedTask)
.Callback(() =>
{
Assert.False(updatedCalled);
Assert.False(executeUpdateCalled);
updatingCalled = true;
});
var mockProcessorUpdated = new Mock <IProcessor <EntityUpdatedPackage <TestEntity> > >(MockBehavior.Strict);
mockProcessorUpdated.SetupGet(p => p.Stage).Returns(new NumericPriorityStage(0));
mockProcessorUpdated
.Setup(p => p.ProcessAsync(It.IsAny <EntityUpdatedPackage <TestEntity> >(), It.IsAny <CancellationToken>()))
.Returns(Task.CompletedTask)
.Callback(() =>
{
Assert.True(updatingCalled);
Assert.True(executeUpdateCalled);
updatedCalled = true;
});
var mockProcessorProvider = new Mock <IProcessorProvider>(MockBehavior.Strict);
mockProcessorProvider
.Setup(p => p.GetProcessors <EntityUpdatingPackage <TestEntity> >())
.Returns(new List <IProcessor <EntityUpdatingPackage <TestEntity> > > {
mockProcessorUpdating.Object
});
mockProcessorProvider
.Setup(p => p.GetProcessors <EntityUpdatedPackage <TestEntity> >())
.Returns(new List <IProcessor <EntityUpdatedPackage <TestEntity> > > {
mockProcessorUpdated.Object
});
ChangeSet expectedChangeSet = null;
var expectedPipeline = new ProcessorPipeline(mockProcessorProvider.Object);
var mockSubject = new Mock <Repository <TestEntity> >(MockBehavior.Strict, expectedPipeline);
mockSubject
.Protected()
.Setup <Task <ChangeSet> >("ExecuteUpdateAsync", ItExpr.IsAny <TestEntity>(), ItExpr.IsAny <ChangeSet>(), It.IsAny <CancellationToken>())
.Callback <TestEntity, ChangeSet, CancellationToken>((e, cs, ct) =>
{
expectedChangeSet = cs;
Assert.True(updatingCalled);
Assert.False(updatedCalled);
executeUpdateCalled = true;
})
.Returns(Task.FromResult(expectedChangeSet));
var expectedEntity = ChangeTrackingObject.CreateTrackable <TestEntity>();
expectedEntity.StringValue = "NewValue";
var result = await mockSubject.Object.UpdateAsync(expectedEntity, CancellationToken.None);
mockProcessorProvider.Verify(p => p.GetProcessors <EntityUpdatingPackage <TestEntity> >(), Times.Once);
mockProcessorProvider.Verify(p => p.GetProcessors <EntityUpdatedPackage <TestEntity> >(), Times.Once);
mockProcessorUpdating.Verify(p => p.ProcessAsync(
It.Is <EntityUpdatingPackage <TestEntity> >(v => ReferenceEquals(v.Entity, expectedEntity)),
It.IsAny <CancellationToken>()),
Times.Once());
mockProcessorUpdated.Verify(p => p.ProcessAsync(
It.Is <EntityUpdatedPackage <TestEntity> >(v => ReferenceEquals(v.Entity, expectedEntity)),
It.IsAny <CancellationToken>()),
Times.Once());
mockSubject
.Protected()
.Verify("ExecuteUpdateAsync",
Times.Once(),
ItExpr.Is <TestEntity>(v => ReferenceEquals(v, expectedEntity)),
ItExpr.Is <ChangeSet>(v => ReferenceEquals(v, expectedChangeSet)),
ItExpr.IsAny <CancellationToken>());
}
/// <summary>
/// Prepares the application for use
/// </summary>
/// <param name="config">Command arguments</param>
/// <param name="contextFactory">Generates the contexts passed to the processor pipeline</param>
/// <param name="processorPipeline">Executes a series of processes that build the site, piece by piece</param>
public App(Config config, ContextFactory contextFactory, ProcessorPipeline processorPipeline)
{
_config = config;
_contextFactory = contextFactory;
_processorPipeline = processorPipeline;
}
/// <summary>
/// Initializes a new instance of the <see cref="Repository{T}"/> class.
/// </summary>
/// <param name="processorPipeline">The processor pipeline.</param>
protected Repository(ProcessorPipeline processorPipeline)
{
ProcessorPipeline = processorPipeline;
}
private async void archivizeButton_Click(object sender, EventArgs e)
{
if (!threadListTextBox.Lines.Any())
{
return;
}
threadListTextBox.Enabled = false;
mediaGroupBox.Enabled = false;
jsonGroupBox.Enabled = false;
htmlGroupBox.Enabled = false;
archivizeButton.Enabled = false;
outputDirectoryButton.Enabled = false;
var processors = new List <IProcessor>();
if (jsonEnabledCheckbox.Checked)
{
processors.Add(new JsonSerializerProcessor(outputDirTextbox.Text, formattedCheckbox.Checked));
}
if (htmlEnabledCheckbox.Checked)
{
processors.Add(new HtmlGeneratorProcessor(outputDirTextbox.Text, namesCheckbox.Checked, coloredNamesCheckbox.Checked, samefagCheckbox.Checked, (string)styleComboBox.SelectedItem, prettifyCheckbox.Checked));
}
MediaResolver resolver;
if (mediaEnabledCheckbox.Checked)
{
var exts = mediaExtTextbox.Text.Split(',');
var thumbExts = thumbnailExtTextBox.Text.Split(',');
resolver = new MediaResolver(true, outputDirTextbox.Text, groupByExtensionsCheckbox.Checked, thumbnailsEnabledCheckbox.Checked, exts, thumbExts);
resolver.OnPostWithMediaCount += Resolver_OnPostWithMediaCount;
resolver.OnProcessingPostMedia += Resolver_OnProcessingPostMedia;
}
else
{
resolver = new MediaResolver();
}
var pipeline = new ProcessorPipeline(processors, resolver);
pipeline.OnProcessorInvoked += Pipeline_OnProcessorInvoked;
foreach (var line in threadListTextBox.Lines)
{
try
{
statusTextLabel.Text = "Parsing posts...";
var thread = await ThreadParser.TryParse(line);
if (mediaEnabledCheckbox.Checked)
{
statusTextLabel.Text = "Resolving media...";
foreach (var post in thread.Posts)
{
if (post.File != null)
{
_total++;
if (resolver.DownloadThumbnails && !string.IsNullOrEmpty(post.File.FileThumbUrl))
{
_total++;
}
}
}
}
await pipeline.Process(thread);
}
catch (Exception ex)
{
MessageBox.Show(ex.ToString(), $"Exception thrown while processing {line}", MessageBoxButtons.OK,
MessageBoxIcon.Error);
}
}
Reset();
}
public async Task ProcessAsync_PreviousChainWorks()
{
var expectedProcessors = new List <object>();
var mockProvider = new Mock <IProcessorProvider>(MockBehavior.Strict);
mockProvider.Setup(p => p.GetProcessors <TestDataPackage>()).Returns(expectedProcessors.OfType <IProcessor <TestDataPackage> >());
mockProvider.Setup(p => p.GetProcessors <TestDataPackage2>()).Returns(expectedProcessors.OfType <IProcessor <TestDataPackage2> >());
var expectedData = new TestDataPackage();
var expectedData2 = new TestDataPackage2();
var subject = new ProcessorPipeline(mockProvider.Object);
var mockProcessor1 = new Mock <IProcessor <TestDataPackage> >(MockBehavior.Strict);
mockProcessor1.SetupGet(p => p.Stage).Returns(new NumericPriorityStage(0));
mockProcessor1
.Setup(p => p.ProcessAsync(It.IsAny <TestDataPackage>(), It.IsAny <CancellationToken>()))
.Callback <TestDataPackage, CancellationToken>((data, token) => Assert.Null(data.Previous))
.Returns(Task.CompletedTask);
var mockProcessor2 = new Mock <IProcessor <TestDataPackage> >(MockBehavior.Strict);
mockProcessor2.SetupGet(p => p.Stage).Returns(new NumericPriorityStage(100));
mockProcessor2
.Setup(p => p.ProcessAsync(It.IsAny <TestDataPackage>(), It.IsAny <CancellationToken>()))
.Callback <TestDataPackage, CancellationToken>(async(data, token) =>
{
Assert.Null(data.Previous);
await subject.ProcessAsync(expectedData2);
})
.Returns(Task.CompletedTask);
var mockProcessor3 = new Mock <IProcessor <TestDataPackage2> >(MockBehavior.Strict);
mockProcessor3.SetupGet(p => p.Stage).Returns(new NumericPriorityStage(0));
mockProcessor3
.Setup(p => p.ProcessAsync(It.IsAny <TestDataPackage2>(), It.IsAny <CancellationToken>()))
.Callback <TestDataPackage2, CancellationToken>((data, token) => Assert.Same(expectedData, data.Previous))
.Returns(Task.CompletedTask);
expectedProcessors.Add(mockProcessor1.Object);
expectedProcessors.Add(mockProcessor2.Object);
expectedProcessors.Add(mockProcessor3.Object);
await subject.ProcessAsync(expectedData);
mockProcessor1.Verify(p => p.ProcessAsync(
It.Is <TestDataPackage>(v => ReferenceEquals(v, expectedData)),
It.IsAny <CancellationToken>()),
Times.Once);
mockProcessor2.Verify(p => p.ProcessAsync(
It.Is <TestDataPackage>(v => ReferenceEquals(v, expectedData)),
It.IsAny <CancellationToken>()),
Times.Once);
mockProcessor3.Verify(p => p.ProcessAsync(
It.Is <TestDataPackage2>(v => ReferenceEquals(v, expectedData2)),
It.IsAny <CancellationToken>()),
Times.Once);
}
