public static IEnumerable <DocumentToken> GetTokens(DocumentSource source, object sourceData)
{
var visit = ContentFactory.Default.CreateFrom(sourceData).Visit();
var visitWhere = ContentFactory.Default.CreateFrom(sourceData).Visit().Where(p => p.Value is IContentPrimitive);
var pairs = ContentFactory.Default.CreateFrom(sourceData).Visit()
.Where(p => p.Value is IContentPrimitive)
.Select(p => new ContentPathValue(
ContentPath.Root
.Append(p.Path
.Select(n =>
n is ContentPath.ListIndex ? ContentPath.AnyIndex : n)), p.Value));
ContentPath lastPath = null;
var offset = 0;
// filter pairs
var filteredPairs = NullFilter(TokenFilter(TokenizePairs(pairs)))
// order by path
.OrderBy(p => p.Path.ToString());
// build tokens from pairs
foreach (var pair in filteredPairs)
{
// assign incremental offsets for similar paths
offset = pair.Path.Equals(lastPath) ? offset + 1 : 0;
var path = new DocumentSourcePath(pair.Path, offset);
lastPath = pair.Path;
yield return(new DocumentToken(source, sourceData, path, pair.Value, pair.Value));
}
}
public DocumentToken(
DocumentSource source,
object sourceData,
DocumentSourcePath sourcePath,
IContentNode raw,
IContentNode normalized,
CultureInfo locale = null)
{
Source = source ?? throw new ArgumentNullException(nameof(source));
SourcePath = sourcePath ?? throw new ArgumentNullException(nameof(sourcePath));
SourceData = sourceData ?? throw new ArgumentNullException(nameof(sourceData));
Raw = raw ?? throw new ArgumentNullException(nameof(raw));
Normalized = normalized ?? throw new ArgumentNullException(nameof(normalized));
Locale = locale;
}