private IEnumerable<ICommand> GenerateDiff(IDataIterator sourceIterator, IDataIterator targetIterator)
{
var commands = new List<ICommand>();
var sourceDataItem = sourceIterator.Next();
var targetDataItem = targetIterator.Next();
while (sourceDataItem != null || targetDataItem != null)
{
var compareResult = Compare(sourceDataItem, targetDataItem);
commands.AddRange((sourceDataItem ?? targetDataItem).GenerateDiff(sourceDataItem, targetDataItem, compareResult));
if (compareResult < 0)
{
sourceDataItem = sourceIterator.Next();
}
else
{
if (compareResult > 0)
{
targetDataItem = targetIterator.Next();
}
else
{
if (compareResult == 0)
{
targetDataItem = targetIterator.Next();
sourceDataItem = sourceIterator.Next();
}
}
}
}
return commands;
}
/// <summary>
/// Generates the diff.
/// </summary>
/// <param name="sourceIterator">The source iterator.</param>
/// <param name="targetIterator">The target iterator.</param>
/// <returns>
/// The diff.
/// </returns>
protected static IList<ICommand> GenerateDiff(IDataIterator sourceIterator, IDataIterator targetIterator)
{
List<ICommand> commands = new List<ICommand>();
IDataItem sourceDataItem = sourceIterator.Next();
IDataItem targetDataItem = targetIterator.Next();
while (sourceDataItem != null || targetDataItem != null)
{
int compareResult = Compare(sourceDataItem, targetDataItem);
commands.AddRange((sourceDataItem ?? targetDataItem).GenerateDiff(sourceDataItem, targetDataItem, compareResult));
if (compareResult < 0)
{
sourceDataItem = sourceIterator.Next();
}
else
{
if (compareResult > 0)
{
targetDataItem = targetIterator.Next();
}
else
{
if (compareResult == 0)
{
targetDataItem = targetIterator.Next();
sourceDataItem = sourceIterator.Next();
}
}
}
}
return commands;
}
public static List <ICommand> GetDiffCommands(HashSet <string> targetItemIDs, CollisionBehavior collisionBehavior,
IDataIterator sourceDataIterator, IDataIterator targetDataIterator, DataEngine engine)
{
var commands = new List <ICommand>();
commands.AddRange(GenerateDiff(sourceDataIterator, targetDataIterator));
//if an item is found to be deleted AND added, we can be sure it's a move
var deleteCommands = commands.OfType <DeleteItemCommand>();
var shouldBeUpdateCommands =
commands.OfType <AddItemCommand>()
.Select(a => new
{
Added = a,
Deleted = deleteCommands.FirstOrDefault(d => d.ItemID == a.ItemID)
}).Where(u => u.Deleted != null).ToList();
foreach (var command in shouldBeUpdateCommands)
{
commands.AddRange(command.Deleted.GenerateUpdateCommand(command.Added));
commands.Remove(command.Added);
commands.Remove(command.Deleted);
//now, this one is an assumption, but would go wrong without the assumption anyway: this assumption is in fact safer
//if the itempath of a delete command starts with this delete command, it will be moved along to the new node, not deleted, just leave it alone
//but we will skip items which are not in the target folder
commands.RemoveAll(c =>
c is DeleteItemCommand &&
((DeleteItemCommand)c).ItemPath.StartsWith(command.Deleted.ItemPath) &&
targetItemIDs.Contains(((DeleteItemCommand)c).ItemID));
}
commands.ForEach(_ => _.CollisionBehavior = collisionBehavior);
return(commands);
}
protected static IList <ICommand> GenerateDiff(IDataIterator sourceIterator, IDataIterator targetIterator)
{
var commands = new List <ICommand>();
var sourceDataItem = sourceIterator.Next();
var targetDataItem = targetIterator.Next();
while (sourceDataItem != null || targetDataItem != null)
{
var compareResult = Compare(sourceDataItem, targetDataItem);
commands.AddRange((sourceDataItem ?? targetDataItem).GenerateDiff(sourceDataItem, targetDataItem,
compareResult));
if (compareResult < 0)
{
sourceDataItem = sourceIterator.Next();
}
else
{
if (compareResult > 0)
{
targetDataItem = targetIterator.Next();
}
else
{
if (compareResult == 0)
{
targetDataItem = targetIterator.Next();
sourceDataItem = sourceIterator.Next();
}
}
}
}
return(commands);
}
/// <summary>
/// Maps an iterator into a Dictionary with unique keys for each item.
/// </summary>
/// <param name="iterator"></param>
/// <returns></returns>
protected static Dictionary <string, IDataItem> Map(IDataIterator iterator)
{
var dict = new Dictionary <string, IDataItem>();
while (true)
{
var item = iterator.Next();
if (item != null)
{
var key = CreateUniqueKey(item);
try
{
dict.Add(key, item);
}
catch (ArgumentException)
{
//This should not happen unless the dataset is corrupted, which would mean it would not give a proper result
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine($"Two items got the same unique key. This suggests invalid data. Check {GetItemPath(item)} vs {GetItemPath(dict[key])}");
Console.ResetColor();
}
}
else
{
return(dict);
}
}
}
public static List <ICommand> GetCommands(IDataIterator sourceDataIterator, IDataIterator targetDataIterator)
{
var source = Map(sourceDataIterator);
var target = Map(targetDataIterator);
var commands = GetCommands(source, target);
return(commands);
}
/// <inheritdoc />
public void AddNamedDataIterator(string name, IDataIterator iterator)
{
if (_additionalNameDataIterators.ContainsKey(name))
{
throw new ArgumentException($"Named data iterator with name {name} already registered in this trainer ({Name}).");
}
_additionalNameDataIterators.Add(name, iterator);
}
public static void TimingRandom <T>(ITitanicArray <T> arr, IDataIterator <T> iter)
{
string name = "TimingRand " + arr.GetType().Name + "-" + iter.GetType().Name;
var s1 = Stopwatch.StartNew();
WriteRandom(arr, iter);
Debug.WriteLine(name + "\tw=" + s1.ElapsedMilliseconds);
ReadAndTestRandom(arr, iter);
Debug.WriteLine(name + "\tt=" + s1.ElapsedMilliseconds);
}
static void WriteRandom <T>(ITitanicArray <T> arr, IDataIterator <T> iter)
{
long len = arr.Length;
var seq = LinearFeedbackShiftRegister.SequenceLength(lfsrStart, (ulong)len).GetEnumerator();
iter.Reset();
for (long i = 0; i < len; i++)
{
seq.MoveNext();
long x = (long)seq.Current;
T next = iter.GetNext();
arr[x] = next;
}
}
static void ReadAndTestRandom <T>(ITitanicArray <T> arr, IDataIterator <T> iter)
{
long len = arr.Length;
var seq = LinearFeedbackShiftRegister.SequenceLength(lfsrStart, (ulong)len).GetEnumerator();
iter.Reset();
for (long i = 0; i < len; i++)
{
seq.MoveNext();
long x = (long)seq.Current;
T next = iter.GetNext();
Assert.IsTrue(iter.AreEqual(arr[x], next));
}
}
public static void ReadWriteTest <T>(ITitanicArray <T> arr, IDataIterator <T> iter)
{
long len = arr.Length;
iter.Reset();
for (long i = 0; i < len; i++)
{
arr[i] = iter.GetNext();
}
iter.Reset();
for (long i = 0; i < len; i++)
{
Assert.IsTrue(iter.AreEqual(arr[i], iter.GetNext()));
}
}
/// <summary>
/// Invoke this hook with a certain parameter registry.
/// </summary>
/// <param name="registry">The registry containing the required values for this hook's execution.</param>
/// <param name="resolver">A helper resolver for complex registry entries (automatically cached).</param>
public override void SubInvoke(IRegistry registry, IRegistryResolver resolver)
{
INetwork network = resolver.ResolveGetSingle <INetwork>("network.self");
ITrainer trainer = resolver.ResolveGetSingle <ITrainer>("trainer.self");
string validationIteratorName = ParameterRegistry.Get <string>("validation_iterator_name");
string finalExternalOutputAlias = ParameterRegistry.Get <string>("final_external_output_alias");
string activationsAlias = ParameterRegistry.Get <string>("output_activations_alias");
string targetsAlias = ParameterRegistry.Get <string>("targets_alias");
if (!trainer.AdditionalNameDataIterators.ContainsKey(validationIteratorName))
{
throw new InvalidOperationException($"Additional named data iterator for validation with name \"{validationIteratorName}\" does not exist in referenced trainer {trainer} but is required.");
}
IDataIterator validationIterator = trainer.AdditionalNameDataIterators[validationIteratorName];
ScoreBegin(registry, resolver);
foreach (var block in validationIterator.Yield(Operator.Handler, Operator.Sigma))
{
trainer.ProvideExternalInputData(network, block);
network.Run(Operator.Handler, trainingPass: false);
INDArray finalOutputPredictions = null;
foreach (ILayerBuffer layerBuffer in network.YieldExternalOutputsLayerBuffers())
{
foreach (string outputAlias in layerBuffer.ExternalOutputs)
{
if (outputAlias.Equals(finalExternalOutputAlias))
{
finalOutputPredictions = Operator.Handler.ClearTrace(layerBuffer.Outputs[outputAlias].Get <INDArray>(activationsAlias));
goto FoundOutput;
}
}
;
}
throw new InvalidOperationException($"Cannot find final output with alias \"{finalExternalOutputAlias}\" in the current network (but is required to score validation).");
FoundOutput:
ScoreIntermediate(finalOutputPredictions, block[targetsAlias], Operator.Handler);
}
ScoreEnd(registry, resolver);
}
/// <returns></returns>
public static List <ICommand> GetDiffCommands(string sourcePath, string targetPath, CollisionBehavior collisionBehavior = CollisionBehavior.Undefined)
{
var sourceManager = Factory.Instance.GetSourceDataManager();
var targetManager = Factory.Instance.GetTargetDataManager();
sourceManager.SerializationPath = sourcePath;
targetManager.SerializationPath = targetPath;
IDataIterator sourceDataIterator = sourceManager.ItemIterator ?? new EmptyIterator();
IDataIterator targetDataIterator = targetManager.ItemIterator;
var commands = GetCommands(sourceDataIterator, targetDataIterator);
var engine = new DataEngine();
engine.ProcessCommands(ref commands);
return(commands);
}
public static void TimingTest <T>(ITitanicArray <T> arr, IDataIterator <T> iter)
{
string name = "TimingTest " + arr.GetType().Name + "-" + iter.GetType().Name;
var s1 = Stopwatch.StartNew();
long len = arr.Length;
iter.Reset();
for (long i = 0; i < len; i++)
{
arr[i] = iter.GetNext();
}
Debug.WriteLine(name + "\tw=" + s1.ElapsedMilliseconds);
iter.Reset();
for (long i = 0; i < len; i++)
{
Assert.IsTrue(iter.AreEqual(arr[i], iter.GetNext()));
}
Debug.WriteLine(name + "\tt=" + s1.ElapsedMilliseconds);
}
public static List <ICommand> GetDiffCommands(string sourcePath, string targetPath, CollisionBehavior collisionBehavior = CollisionBehavior.Undefined)
{
var targetManager = Factory.Instance.GetSourceDataManager();
var sourceManager = Factory.Instance.GetTargetDataManager();
sourceManager.SerializationPath = sourcePath;
targetManager.SerializationPath = targetPath;
IDataIterator sourceDataIterator = sourceManager.ItemIterator;
IDataIterator targetDataIterator = targetManager.ItemIterator;
var engine = new DataEngine();
var commands = new List <ICommand>();
commands.AddRange(GenerateDiff(sourceDataIterator, targetDataIterator));
//if an item is found to be deleted AND added, we can be sure it's a move
var deleteCommands = commands.OfType <DeleteItemCommand>();
var shouldBeUpdateCommands =
commands.OfType <AddItemCommand>()
.Select(a => new
{
Added = a,
Deleted = deleteCommands.FirstOrDefault(d => d.ItemID == a.ItemID)
}).Where(u => u.Deleted != null).ToList();
foreach (var command in shouldBeUpdateCommands)
{
commands.AddRange(command.Deleted.GenerateUpdateCommand(command.Added));
commands.Remove(command.Added);
commands.Remove(command.Deleted);
//now, this one is an assumption, but would go wrong without the assumption anyway: this assumption is in fact safer
//if the itempath of a delete command starts with this delete command, it will be moved along to the new node, not deleted, just leave it alone
commands.RemoveAll(c => c is DeleteItemCommand && ((DeleteItemCommand)c).ItemPath.StartsWith(command.Deleted.ItemPath));
}
commands.ForEach(_ => _.CollisionBehavior = collisionBehavior);
engine.ProcessCommands(ref commands);
return(commands);
}
private static HashSet <string> GetItemIDs(string path)
{
var itemIDs = new HashSet <string>();
if (string.IsNullOrEmpty(path))
{
return(itemIDs);
}
var dataManager = Factory.Instance.GetTargetDataManager();
dataManager.SerializationPath = path;
IDataIterator dataIterator = dataManager.ItemIterator;
var item = dataIterator.Next();
while (item != null && item is QuickContentDataItem)
{
itemIDs.Add((item as QuickContentDataItem).ItemID);
item = dataIterator.Next();
}
return(itemIDs);
}
public static void ReadWriteRandom <T>(ITitanicArray <T> arr, IDataIterator <T> iter)
{
WriteRandom(arr, iter);
ReadAndTestRandom(arr, iter);
}
