public async Task <ApiLoaderWorkItem> ProcessAsync(ApiLoaderWorkItem resourceWorkItem)
{
var contextPrefix = LogContext.BuildContextPrefix(resourceWorkItem);
var count = Interlocked.Increment(ref _count);
_log.Debug($"{contextPrefix} #{count} submitting");
var refreshToken = false;
while (true)
{
using (var response = await _poster.PostResource(
resourceWorkItem.Json, resourceWorkItem.ElementName,
resourceWorkItem.ResourceSchemaName, refreshToken).ConfigureAwait(false))
{
if (response.StatusCode.Equals(HttpStatusCode.Unauthorized) && !refreshToken)
{
_log.Info("Expired token detected, refreshing and retrying request.");
refreshToken = true;
continue;
}
// if we refreshed the token we should then disable refresh token until it is needed again.
if (refreshToken)
{
refreshToken = false;
}
resourceWorkItem.AddSubmissionResult(response, count);
}
return(resourceWorkItem);
}
}
public bool Process(FileContext fileContext, Stream stream)
{
var contextPrefix = LogContext.BuildContextPrefix(Path.GetFileNameWithoutExtension(fileContext.FileName));
if (fileContext.NumberOfIdRefs == 0)
{
_log.Debug($"{contextPrefix} No file references are found. Skipping preload of references.");
return(true);
}
var sw = new Stopwatch();
sw.Start();
var referenceCache = _referenceCacheProvider.GetXmlReferenceCache(fileContext.FileName);
using (var reader = new XmlTextReader(stream))
{
while (reader.Read())
{
if (reader.NodeType != XmlNodeType.Element)
{
continue;
}
var id = reader.GetAttribute("id");
if (string.IsNullOrEmpty(id))
{
continue;
}
using (var r = reader.ReadSubtree())
{
var referenceSource = XElement.Load(r);
referenceCache.PreloadReferenceSource(id, referenceSource);
}
}
_log.Info($"{contextPrefix} {referenceCache.NumberOfLoadedReferences} references preloaded.");
}
sw.Stop();
_log.Debug($"{contextPrefix} finished preloading in {sw.Elapsed.TotalSeconds} seconds.");
return(true);
}
public async Task <ApiLoaderWorkItem> ProcessAsync(ApiLoaderWorkItem resourceWorkItem)
{
var contextPrefix = LogContext.BuildContextPrefix(resourceWorkItem);
var sw = Stopwatch.StartNew();
var count = Interlocked.Increment(ref _count);
var results = await _next.ProcessAsync(resourceWorkItem).ConfigureAwait(false);
sw.Stop();
_log.Debug(
$"{contextPrefix} #{count} completed in {Math.Round((decimal) sw.Elapsed.TotalMilliseconds, 3)} milliseconds");
_resourceStatistic.AddOrUpdate(resourceWorkItem, sw.ElapsedMilliseconds);
return(results);
}
public bool Process(FileContext fileContext, Stream stream)
{
var contextPrefix = LogContext.BuildContextPrefix(Path.GetFileNameWithoutExtension(fileContext.FileName));
if (fileContext.NumberOfIdRefs == 0)
{
_log.Debug($"{contextPrefix} No file references are found. Skipping finding references.");
return(true);
}
var sw = new Stopwatch();
sw.Start();
var total = 0;
var referenceCache = _referenceCacheProvider.GetXmlReferenceCache(fileContext.FileName);
using (var reader = new XmlTextReader(stream))
{
while (reader.Read())
{
if (reader.NodeType != XmlNodeType.Element)
{
continue;
}
var refId = reader.GetAttribute("ref");
if (!string.IsNullOrEmpty(refId))
{
referenceCache.LoadReference(refId);
total++;
}
}
_log.Info($"{contextPrefix} {total} references to {referenceCache.NumberOfReferences} resources found");
}
sw.Stop();
_log.Debug($"Finished finding references in {sw.Elapsed.TotalSeconds} seconds.");
return(true);
}
private IEnumerable <FileContext> CreateFileContexts()
{
var fileContexts = new HashSet <FileContext>();
foreach (string file in Directory.GetFiles(_dataConfiguration.Folder, "*.xml", SearchOption.AllDirectories))
{
var contextPrefix = LogContext.BuildContextPrefix(Path.GetFileName(file));
Stopwatch sw = null;
if (_log.IsDebugEnabled)
{
sw = new Stopwatch();
sw.Start();
}
_log.Debug($"{contextPrefix} Start processing");
try
{
var fileContext = CreateFileContext(file, contextPrefix);
fileContexts.Add(fileContext);
}
catch (Exception e)
{
_log.Error(e);
}
if (_log.IsDebugEnabled)
{
sw.Stop();
_log.Debug($"{contextPrefix} Finished processing in {sw.Elapsed.TotalSeconds} seconds.");
}
}
return(fileContexts);
FileContext CreateFileContext(string fileName, string contextPrefix)
{
var fileContext = new FileContext
{
FileName = fileName,
Resources = new HashSet <string>(),
IsValid = true,
};
var xmlReaderSettings = new XmlReaderSettings {
CloseInput = true
};
if (!_xsdConfiguration.DoNotValidateXml)
{
_log.Debug($"{contextPrefix} XSD validation is enabled.");
xmlReaderSettings.Schemas = _xmlSchemaSet;
xmlReaderSettings.ValidationType = ValidationType.Schema;
xmlReaderSettings.ValidationFlags = XmlSchemaValidationFlags.AllowXmlAttributes |
XmlSchemaValidationFlags.ReportValidationWarnings |
XmlSchemaValidationFlags.ProcessIdentityConstraints;
xmlReaderSettings.ValidationEventHandler += (s, e) =>
{
_log.Error($"{e.Message}");
fileContext.IsValid = false;
};
}
using (var stream = new FileStream(fileName, FileMode.Open, FileAccess.Read))
{
if (stream.Length == 0)
{
_log.Warn($"{contextPrefix} Empty file.");
return(fileContext);
}
try
{
using (var reader = XmlReader.Create(stream, xmlReaderSettings))
{
// get the interchange name
if (reader.MoveToContent() == XmlNodeType.Element)
{
var match = _interchangeNameRegex.Match(reader.Name);
if (match.Success)
{
fileContext.FileType = match.Groups["InterchangeType"].Value;
}
else
{
_log.Warn($"{contextPrefix} Unknown file type");
return(fileContext);
}
}
_log.Debug($"{contextPrefix} Checking for reference identities and resources");
// loop through and count the ids refs
while (reader.Read())
{
if (reader.NodeType != XmlNodeType.Element)
{
continue;
}
if (reader.Depth == 1)
{
fileContext.Resources.Add(reader.Name);
}
string refId = reader.GetAttribute("ref");
if (!string.IsNullOrEmpty(refId))
{
fileContext.NumberOfIdRefs++;
}
}
}
}
catch (XmlException e)
{
_log.Error($"Unexpected error in {fileName}", e);
fileContext.IsValid = false;
}
}
if (fileContext.NumberOfIdRefs > 0)
{
_log.Debug($"{contextPrefix} Reference identities exist");
}
_log.Debug($"{contextPrefix} Finished processing.");
if (_xsdConfiguration.DoNotValidateXml)
{
_log.Debug($"{contextPrefix} was not validated with XSD validation.");
return(fileContext);
}
if (fileContext.IsValid)
{
_log.Info($"{contextPrefix} passed XSD validation.");
}
else
{
_log.Warn($"{contextPrefix} failed XSD validation.");
_bulkLoadClientResult.ExitCode = 1;
}
return(fileContext);
}
}
