protected override async Task <StageResultList <TOut, string, TransformStageCache <TInCache> > > DoInternal([AllowNull] TransformStageCache <TInCache>?cache, OptionToken options)
{
var input = await this.input(cache?.ParentCache, options).ConfigureAwait(false);
var task = LazyTask.Create(async() =>
{
var inputList = await input.Perform;
var list = await Task.WhenAll(inputList.result.Select(async subInput =>
{
if (subInput.HasChanges)
{
var subResult = await subInput.Perform;
var transformed = await this.transform(subResult.result).ConfigureAwait(false);
bool hasChanges = true;
if (cache != null && cache.Transformed.TryGetValue(transformed.Id, out var oldHash))
{
hasChanges = oldHash == transformed.Hash;
}
return(result: StageResult.Create(transformed, transformed.Hash, hasChanges, transformed.Id), inputId: subInput.Id, outputHash: transformed.Hash);
}
else
{
if (cache == null || !cache.InputToOutputId.TryGetValue(subInput.Id, out var oldOutputId) || !cache.Transformed.TryGetValue(subInput.Id, out var oldOutputHash))
{
throw this.Context.Exception("No changes, so old value should be there.");
}
return(result: StageResult.Create(LazyTask.Create(async() =>
{
var newSource = await subInput.Perform;
var transformed = await this.transform(newSource.result).ConfigureAwait(false);
return (transformed, transformed.Hash);
}), false, oldOutputId),
inputId: subInput.Id,
outputHash: oldOutputHash
);
}
})).ConfigureAwait(false);
var newCache = new TransformStageCache <TInCache>()
{
InputToOutputId = list.ToDictionary(x => x.inputId, x => x.result.Id),
OutputIdOrder = list.Select(x => x.result.Id).ToArray(),
ParentCache = inputList.cache,
Transformed = list.ToDictionary(x => x.result.Id, x => x.outputHash)
};
return(result: list.Select(x => x.result).ToImmutableList(), cache: newCache);
});
bool hasChanges = input.HasChanges;
var newCache = cache;
if (input.HasChanges || newCache == null)
{
var(list, c) = await task;
newCache = c;
if (!hasChanges && list.Count != cache?.OutputIdOrder.Length)
{
hasChanges = true;
}
if (!hasChanges && cache != null)
{
for (int i = 0; i < cache.OutputIdOrder.Length && !hasChanges; i++)
{
if (list[i].Id != cache.OutputIdOrder[i])
{
hasChanges = true;
}
if (list[i].HasChanges)
{
hasChanges = true;
}
}
}
}
return(StageResultList.Create(task, hasChanges, newCache.OutputIdOrder.ToImmutableList()));
}
protected override async Task <StageResultList <TOut, TInItemCache, WhereStageCache <TInCache> > > DoInternal([AllowNull] WhereStageCache <TInCache>?cache, OptionToken options)
{
var input = await this.input(cache?.ParentCache, options).ConfigureAwait(false);
var task = LazyTask.Create(async() =>
{
var inputList = await input.Perform;
var list = (await Task.WhenAll(inputList.result.Select(async subInput =>
{
bool pass;
if (subInput.HasChanges)
{
var(result, itemCache) = await subInput.Perform;
pass = await this.predicate(result).ConfigureAwait(false);
}
else
{
if (cache is null)
{
throw new InvalidOperationException("This shoudl not happen. if item has no changes, ther must be a child cache.");
}
// since HasChanges if false, it was present in the last invocation
// if it is passed this it was added to thec cache otherwise not.
pass = cache.OutputIdOrder.Contains(subInput.Id);
}
if (pass)
{
return(subInput);
}
else
{
return(null);
}
})).ConfigureAwait(false)).Where(x => x != null);
var newCache = new WhereStageCache <TInCache>()
{
OutputIdOrder = list.Select(x => x.Id).ToArray(),
ParentCache = inputList.cache
};
return(result: list.ToImmutableList(), cache: newCache);
});
bool hasChanges = input.HasChanges;
var newCache = cache;
if (input.HasChanges || newCache == null)
{
var(list, c) = await task;
newCache = c;
if (!hasChanges && list.Count != cache?.OutputIdOrder.Length)
{
hasChanges = true;
}
if (!hasChanges && cache != null)
{
for (int i = 0; i < cache.OutputIdOrder.Length && !hasChanges; i++)
{
if (list[i].Id != cache.OutputIdOrder[i])
{
hasChanges = true;
}
if (list[i].HasChanges)
{
hasChanges = true;
}
}
}
}
return(StageResultList.Create(task, hasChanges, newCache.OutputIdOrder.ToImmutableList()));
}
protected override async Task <StageResultList <Stream, string, TransformStageCache <TInCache> > > DoInternal([AllowNull] TransformStageCache <TInCache>?cache, OptionToken options)
{
var input = await this.input(cache?.ParentCache, options).ConfigureAwait(false);
var task = LazyTask.Create(async() =>
{
var inputList = await input.Perform;
var sidecarLookup = inputList.result.Where(x => Path.GetExtension(x.Id) == this.SidecarExtension)
.ToDictionary(x => Path.Combine(Path.GetDirectoryName(x.Id) ?? string.Empty, Path.GetFileNameWithoutExtension(x.Id)));
var files = inputList.result.Where(x => Path.GetExtension(x.Id) != this.SidecarExtension);
var list = await Task.WhenAll(files.Select(async file =>
{
if (sidecarLookup.TryGetValue(file.Id, out var sidecar) && (file.HasChanges || sidecar.HasChanges))
{
var(fileResult, fileCache) = await file.Perform;
var(sidecarResult, sidecarCache) = await sidecar.Perform;
var deserializer = new YamlDotNet.Serialization.DeserializerBuilder()
.WithNamingConvention(YamlDotNet.Serialization.NamingConventions.CamelCaseNamingConvention.Instance)
.Build();
var oldMetadata = fileResult.Metadata;
MetadataContainer?newMetadata;
try
{
using var stream = sidecarResult.Value;
using var reader = new StreamReader(stream);
var metadata = deserializer.Deserialize <TMetadata>(reader);
if (metadata != null)
{
if (this.update != null)
{
newMetadata = oldMetadata.AddOrUpdate(metadata.GetType(), metadata, (oldValue, newValue) => this.update((TMetadata)oldValue !/*AllowNull is set, so why the warnign?*/, (TMetadata)newValue));
}
else
{
newMetadata = oldMetadata.Add(metadata.GetType(), metadata);
}
}
else
{
newMetadata = null;
}
}
catch (YamlDotNet.Core.YamlException e) when(e.InnerException is null) // Hope that only happens when it does not match.
{
newMetadata = null;
}
if (newMetadata != null)
{
fileResult = fileResult.With(newMetadata);
}
var hasChanges = true;
if (cache != null && cache.Transformed.TryGetValue(fileResult.Id, out var oldHash))
{
hasChanges = oldHash != fileResult.Hash;
}
return(result: StageResult.Create <Stream, string>(fileResult, fileResult.Hash, hasChanges, fileResult.Id), inputId: file.Id, outputHash: fileResult.Hash);
}
else if (file.HasChanges)
{
var(fileResult, fileCache) = await file.Perform;
var hasChanges = true;
if (cache != null && cache.Transformed.TryGetValue(fileResult.Id, out var oldHash))
{
hasChanges = oldHash != fileResult.Hash;
}
System.Diagnostics.Debug.Assert(hasChanges); // if the original file had changes so must this have.
return(result: StageResult.Create <Stream, string>(fileResult, fileResult.Hash, hasChanges, fileResult.Id), inputId: file.Id, outputHash: fileResult.Hash);
}
else
{
if (cache == null || !cache.InputToOutputId.TryGetValue(file.Id, out var oldOutputId) || !cache.Transformed.TryGetValue(file.Id, out var oldOutputHash))
{
throw this.Context.Exception("No changes, so old value should be there.");
}
var task = LazyTask.Create(async() =>
{
var(fileResult, fileCache) = await file.Perform;
return(fileResult, fileResult.Hash);
});
return(result: StageResult.Create(task, false, oldOutputId), inputId: file.Id, outputHash: oldOutputHash);
}
})).ConfigureAwait(false);
var newCache = new TransformStageCache <TInCache>()
{
InputToOutputId = list.ToDictionary(x => x.inputId, x => x.result.Id),
OutputIdOrder = list.Select(x => x.result.Id).ToArray(),
ParentCache = inputList.cache,
Transformed = list.ToDictionary(x => x.result.Id, x => x.outputHash)
};
return(result: list.Select(x => x.result).ToImmutableList(), cache: newCache);
});
bool hasChanges = input.HasChanges;
var newCache = cache;
if (input.HasChanges || newCache == null)
{
var(list, c) = await task;
newCache = c;
if (!hasChanges && list.Count != cache?.OutputIdOrder.Length)
{
hasChanges = true;
}
if (!hasChanges && cache != null)
{
for (int i = 0; i < cache.OutputIdOrder.Length && !hasChanges; i++)
{
if (list[i].Id != cache.OutputIdOrder[i])
{
hasChanges = true;
}
if (list[i].HasChanges)
{
hasChanges = true;
}
}
}
}
return(StageResultList.Create(task, hasChanges, newCache.OutputIdOrder.ToImmutableList()));
}