public async Task <ApiLoaderWorkItem> ProcessAsync(ApiLoaderWorkItem resourceWorkItem) { var contextPrefix = LogContext.BuildContextPrefix(resourceWorkItem); var count = Interlocked.Increment(ref _count); _log.Debug($"{contextPrefix} #{count} submitting"); var refreshToken = false; while (true) { using (var response = await _poster.PostResource( resourceWorkItem.Json, resourceWorkItem.ElementName, resourceWorkItem.ResourceSchemaName, refreshToken).ConfigureAwait(false)) { if (response.StatusCode.Equals(HttpStatusCode.Unauthorized) && !refreshToken) { _log.Info("Expired token detected, refreshing and retrying request."); refreshToken = true; continue; } // if we refreshed the token we should then disable refresh token until it is needed again. if (refreshToken) { refreshToken = false; } resourceWorkItem.AddSubmissionResult(response, count); } return(resourceWorkItem); } }
public bool Process(FileContext fileContext, Stream stream) { var contextPrefix = LogContext.BuildContextPrefix(Path.GetFileNameWithoutExtension(fileContext.FileName)); if (fileContext.NumberOfIdRefs == 0) { _log.Debug($"{contextPrefix} No file references are found. Skipping preload of references."); return(true); } var sw = new Stopwatch(); sw.Start(); var referenceCache = _referenceCacheProvider.GetXmlReferenceCache(fileContext.FileName); using (var reader = new XmlTextReader(stream)) { while (reader.Read()) { if (reader.NodeType != XmlNodeType.Element) { continue; } var id = reader.GetAttribute("id"); if (string.IsNullOrEmpty(id)) { continue; } using (var r = reader.ReadSubtree()) { var referenceSource = XElement.Load(r); referenceCache.PreloadReferenceSource(id, referenceSource); } } _log.Info($"{contextPrefix} {referenceCache.NumberOfLoadedReferences} references preloaded."); } sw.Stop(); _log.Debug($"{contextPrefix} finished preloading in {sw.Elapsed.TotalSeconds} seconds."); return(true); }
public async Task <ApiLoaderWorkItem> ProcessAsync(ApiLoaderWorkItem resourceWorkItem) { var contextPrefix = LogContext.BuildContextPrefix(resourceWorkItem); var sw = Stopwatch.StartNew(); var count = Interlocked.Increment(ref _count); var results = await _next.ProcessAsync(resourceWorkItem).ConfigureAwait(false); sw.Stop(); _log.Debug( $"{contextPrefix} #{count} completed in {Math.Round((decimal) sw.Elapsed.TotalMilliseconds, 3)} milliseconds"); _resourceStatistic.AddOrUpdate(resourceWorkItem, sw.ElapsedMilliseconds); return(results); }
public bool Process(FileContext fileContext, Stream stream) { var contextPrefix = LogContext.BuildContextPrefix(Path.GetFileNameWithoutExtension(fileContext.FileName)); if (fileContext.NumberOfIdRefs == 0) { _log.Debug($"{contextPrefix} No file references are found. Skipping finding references."); return(true); } var sw = new Stopwatch(); sw.Start(); var total = 0; var referenceCache = _referenceCacheProvider.GetXmlReferenceCache(fileContext.FileName); using (var reader = new XmlTextReader(stream)) { while (reader.Read()) { if (reader.NodeType != XmlNodeType.Element) { continue; } var refId = reader.GetAttribute("ref"); if (!string.IsNullOrEmpty(refId)) { referenceCache.LoadReference(refId); total++; } } _log.Info($"{contextPrefix} {total} references to {referenceCache.NumberOfReferences} resources found"); } sw.Stop(); _log.Debug($"Finished finding references in {sw.Elapsed.TotalSeconds} seconds."); return(true); }
private IEnumerable <FileContext> CreateFileContexts() { var fileContexts = new HashSet <FileContext>(); foreach (string file in Directory.GetFiles(_dataConfiguration.Folder, "*.xml", SearchOption.AllDirectories)) { var contextPrefix = LogContext.BuildContextPrefix(Path.GetFileName(file)); Stopwatch sw = null; if (_log.IsDebugEnabled) { sw = new Stopwatch(); sw.Start(); } _log.Debug($"{contextPrefix} Start processing"); try { var fileContext = CreateFileContext(file, contextPrefix); fileContexts.Add(fileContext); } catch (Exception e) { _log.Error(e); } if (_log.IsDebugEnabled) { sw.Stop(); _log.Debug($"{contextPrefix} Finished processing in {sw.Elapsed.TotalSeconds} seconds."); } } return(fileContexts); FileContext CreateFileContext(string fileName, string contextPrefix) { var fileContext = new FileContext { FileName = fileName, Resources = new HashSet <string>(), IsValid = true, }; var xmlReaderSettings = new XmlReaderSettings { CloseInput = true }; if (!_xsdConfiguration.DoNotValidateXml) { _log.Debug($"{contextPrefix} XSD validation is enabled."); xmlReaderSettings.Schemas = _xmlSchemaSet; xmlReaderSettings.ValidationType = ValidationType.Schema; xmlReaderSettings.ValidationFlags = XmlSchemaValidationFlags.AllowXmlAttributes | XmlSchemaValidationFlags.ReportValidationWarnings | XmlSchemaValidationFlags.ProcessIdentityConstraints; xmlReaderSettings.ValidationEventHandler += (s, e) => { _log.Error($"{e.Message}"); fileContext.IsValid = false; }; } using (var stream = new FileStream(fileName, FileMode.Open, FileAccess.Read)) { if (stream.Length == 0) { _log.Warn($"{contextPrefix} Empty file."); return(fileContext); } try { using (var reader = XmlReader.Create(stream, xmlReaderSettings)) { // get the interchange name if (reader.MoveToContent() == XmlNodeType.Element) { var match = _interchangeNameRegex.Match(reader.Name); if (match.Success) { fileContext.FileType = match.Groups["InterchangeType"].Value; } else { _log.Warn($"{contextPrefix} Unknown file type"); return(fileContext); } } _log.Debug($"{contextPrefix} Checking for reference identities and resources"); // loop through and count the ids refs while (reader.Read()) { if (reader.NodeType != XmlNodeType.Element) { continue; } if (reader.Depth == 1) { fileContext.Resources.Add(reader.Name); } string refId = reader.GetAttribute("ref"); if (!string.IsNullOrEmpty(refId)) { fileContext.NumberOfIdRefs++; } } } } catch (XmlException e) { _log.Error($"Unexpected error in {fileName}", e); fileContext.IsValid = false; } } if (fileContext.NumberOfIdRefs > 0) { _log.Debug($"{contextPrefix} Reference identities exist"); } _log.Debug($"{contextPrefix} Finished processing."); if (_xsdConfiguration.DoNotValidateXml) { _log.Debug($"{contextPrefix} was not validated with XSD validation."); return(fileContext); } if (fileContext.IsValid) { _log.Info($"{contextPrefix} passed XSD validation."); } else { _log.Warn($"{contextPrefix} failed XSD validation."); _bulkLoadClientResult.ExitCode = 1; } return(fileContext); } }