protected override bool MoveNextCore()
{
// Iterate sub cursor or move to the next file.
while (_subCursor == null || !_subCursor.MoveNext())
{
// Cleanup old sub cursor
if (_subCursor != null)
{
_subCursor.Dispose();
_subCursor = null;
}
if (!TryGetNextPathAndValues(out string path, out string relativePath, out List <string> values))
{
return(false);
}
ILegacyDataLoader loader = null;
try
{
// Load the sub cursor and reset the data.
loader = _parent.CreateLoaderFromBytes(_parent._subLoaderBytes, new MultiFileSource(path));
}
catch (Exception e)
{
Ch.Warning($"Failed to load file {path} due to a loader exception. Moving on to the next file. Ex: {e.Message}");
continue;
}
_subCursor = loader.GetRowCursor(_subActivecolumnsNeeded);
try
{
UpdateSubGetters();
UpdateColumnValues(relativePath, values);
}
catch (InvalidOperationException e)
{
// Failed to load this file so skip.
Ch.Warning(MessageSensitivity.Schema, e.Message);
if (_subCursor != null)
{
_subCursor.Dispose();
_subCursor = null;
}
}
}
return(true);
}
/// <summary>
/// Parse all column values from the directory path.
/// </summary>
/// <param name="path">The directory path to parse for name/value pairs.</param>
/// <param name="results">The resulting name value pairs.</param>
/// <returns>true if the parsing was successfull.</returns>
private bool TryParseValuesFromPath(string path, out List <string> results)
{
Contracts.CheckNonWhiteSpace(path, nameof(path));
results = null;
try
{
results = _parent._pathParser.ParseValues(path).ToList();
return(true);
}
catch (InvalidOperationException e)
{
Ch.Warning($"Could not parse column values from the path {path}. Ex: {e.Message}");
results = null;
return(false);
}
}
/// <summary>
/// Truncate path to the specified number of trailing directories.
/// </summary>
/// <param name="dirCount">Number of directories to retain.</param>
/// <param name="path">Path to truncate.</param>
/// <param name="truncPath">The resulting truncated path.</param>
/// <returns>true if the truncation was successful.</returns>
private bool TryTruncatePath(int dirCount, string path, out string truncPath)
{
truncPath = null;
// Remove directories that shouldn't be parsed.
var segments = PartitionedPathUtils.SplitDirectories(path);
segments = segments.Skip(segments.Count() - dirCount - 1);
if (segments.Count() < dirCount - 1)
{
Ch.Warning($"Path {path} did not have {dirCount} directories necessary for parsing.");
return false;
}
// Rejoin segments to create a valid path.
truncPath = String.Join(Path.DirectorySeparatorChar.ToString(), segments);
return true;
}
private bool TryGetNextPathAndValues(out string path, out string relativePath, out List<string> values)
{
path = null;
relativePath = null;
values = null;
do
{
// No more files to load.
if (!_fileOrder.MoveNext())
{
return false;
}
// Get next file and parse the column values from the file path.
string curPath = _parent._files.GetPathOrNull(_fileOrder.Current);
if (String.IsNullOrEmpty(curPath))
{
Ch.Warning($"File at index {_fileOrder.Current} is missing a path. Loading of file is being skipped.");
continue;
}
if (!TryTruncatePath(_parent._tailingDirCount, curPath, out relativePath))
{
continue;
}
if (!TryParseValuesFromPath(relativePath, out values))
{
continue;
}
path = curPath;
} while (String.IsNullOrEmpty(path));
return true;
}