private List <string> ApplySubFilters(string mapValue, XElement absoluteRoot, XElement relativeRoot)
{
List <string> result = new List <string>();
XElement targetRoot = subFilter.pathType switch {
EOnlineInterpreterPathType.relative => relativeRoot,
EOnlineInterpreterPathType.absolute => absoluteRoot,
_ => throw new InvalidOperationException("pathType " + subFilter.pathType + " is not supported by DownloaderInterpreterConfig.cs")
};
List <XElement> basePathElements = XMLUtils.FindNodesAtPath(targetRoot, subFilter.basePath.Split(':'));
List <string> filterValues = new List <string>();
foreach (XElement element in basePathElements)
{
filterValues.AddRange(XMLUtils.FindNodeValuesAtPath(element, subFilter.valuePath.Split(':')).Select(value => value.Trim()));
}
if (subFilter.optional && (basePathElements.Count == 0 || basePathElements.Count > filterValues.Count))
{
//this file has no filtered basePathElements
//or this file has some filtered basePathElements without a filterValue
result.Add(mapValue);
}
var fileNameModifierRegex = new Regex(subFilter.fileNameModifierRegex);
result.AddRange(filterValues
.Where(value => CustomRegex.AllMatching(value, subFilter.valueRegexFilters))
.Select(value =>
fileNameModifierRegex.Replace(mapValue, subFilter.fileNameModifierReplacement.Replace("${value}", value))
));
return(result);
}
public Dictionary <string, Dictionary <string, List <string> > > BuildXMLContent(Queue <string> fileUrls)
{
Dictionary <string, Dictionary <string, List <string> > > result = new Dictionary <string, Dictionary <string, List <string> > >();
foreach (string fileUrl in fileUrls)
{
Console.WriteLine("loading xml data from url: " + fileUrl);
XDocument document = XMLUtils.LoadDocumentFromURL(fileUrl);
if (!XMLUtils.IsMatchingPathConstraints(document.Root, pathConstraints))
{
Console.WriteLine("skipping file, because of nonMatching path-constraints: " + fileUrl);
continue;
}
List <XElement> primaryConfigElements = XMLUtils.FindNodesAtPath(document.Root, configPath.Split(':'));
Console.WriteLine("found " + primaryConfigElements.Count + " elements at path: " + configPath);
foreach (XElement primaryElement in primaryConfigElements)
{
List <string> primaryMapValues = mapConfigBy.GetMapValues(fileUrl, document, primaryElement);
if (primaryMapValues.Count != 1)
{
Console.WriteLine("Could not uniquely identify node at mapConfigBy.path");
throw new InvalidOperationException("make sure your mapConfigBy.path can uniquely identify a node when loading from xml files!");
}
Dictionary <string, List <string> > secondaryValues = new Dictionary <string, List <string> >();
foreach (OnlineInterpreterFilter filter in configFilter)
{
List <string> filterValues = new List <string>();
switch (filter.pathType)
{
case EOnlineInterpreterPathType.relative:
filterValues.AddRange(XMLUtils.FindNodeValuesAtPath(primaryElement, filter.path.Split(':'))
.Select(value => ResolveTranslationValue(value.Trim(), filter.outputName)));
break;
case EOnlineInterpreterPathType.absolute:
filterValues.AddRange(XMLUtils.FindNodeValuesAtPath(document.Root, filter.path.Split(':'))
.Select(value => ResolveTranslationValue(value.Trim(), filter.outputName)));
break;
default:
throw new InvalidOperationException("pathType " + filter.pathType + " is not supported by OnlineSourceInterpreterConfig.cs");
}
secondaryValues[filter.outputName] = filterValues;
}
result[primaryMapValues[0]] = secondaryValues;
}
}
return(result);
}
public List <string> BuildXMLContent(Queue <string> fileUrls)
{
List <string> result = new List <string>();
foreach (string fileUrl in fileUrls)
{
Console.WriteLine("loading xml data from url: " + fileUrl);
XDocument document = XMLUtils.LoadDocumentFromURL(fileUrl);
if (!XMLUtils.IsMatchingPathConstraints(document.Root, pathConstraints))
{
Console.WriteLine("skipping file, because of nonMatching pathConstraints: " + fileUrl);
continue;
}
List <XElement> primaryConfigElements = XMLUtils.FindNodesAtPath(document.Root, configPath.Split(':'));
Console.WriteLine("found " + primaryConfigElements.Count + " elements at path: " + configPath);
foreach (XElement primaryElement in primaryConfigElements)
{
List <string> primaryMapValues = XMLUtils.FindNodeValuesAtPath(primaryElement, fileNamePath.Split(':')).ToList();
if (primaryMapValues.Count != 1)
{
Console.WriteLine("Could not uniquely identify node at mappingPath: " + fileNamePath);
throw new InvalidOperationException("make sure your mappingPath can uniquely identify a node when loading from xml files!");
}
string mapValue = primaryMapValues[0].Trim();
if (!CustomRegex.AllMatching(mapValue, fileNameRegexFilters))
{
continue;
}
ApplySubFilters(mapValue, document.Root, primaryElement).ForEach(value => {
if (!result.Contains(value))
{
result.Add(value);
}
});
}
}
return(result);
}
Queue <string> IOnlineSourcesConfig.GetDataFileURLs(IOnlineUrlHolder urlHolder, IOnlineSourceHolder sourceHolder)
{
Queue <string> result = new Queue <string>();
Console.WriteLine("gathering online source addresses...");
XDocument document = XMLUtils.LoadDocumentFromURL(GetAssetInfoUrl(urlHolder, sourceHolder));
foreach (
string actualFileName in
from fileName in XMLUtils.FindNodeValuesAtPath(document.Root, assetFileNamePath.Split(':'))
where CustomRegex.AllMatching(fileName, dataContainerRegexFilters)
let addressKey = addressDictSupplier.GetOnlineAddressDict().First(kvp => fileName.StartsWith(kvp.Key))
select addressKey.Value + fileName.Substring(addressKey.Key.Length))
{
result.Enqueue(GetOnlineBaseUrl(urlHolder, sourceHolder) + "/" + actualFileName);
}
return(result);
}
List <string> IMappingValue.GetMapValues(string fileUrl, XDocument document, XElement targetElement)
{
return(XMLUtils.FindNodeValuesAtPath(targetElement, path.Split(':')).ToList());
}