private static void LogResource(ApiLoaderWorkItem resource)
{
var contextPrefix = LogContext.BuildContextPrefix(resource);
var successfulResponse = resource.Responses.FirstOrDefault(r => r.IsSuccess);
if (successfulResponse != null)
{
_log.Info($"{contextPrefix} #{successfulResponse.RequestNumber} {successfulResponse.Message} - {(int)successfulResponse.StatusCode}");
return;
}
int responseCount = resource.Responses.Count;
for (int i = 0; i < responseCount; ++i)
{
var response = resource.Responses[i];
if (i == responseCount - 1)
{
_log.Error($"{contextPrefix} #{response.RequestNumber} {response.Message} - {(int)response.StatusCode} - {response.Content}");
}
else
{
_log.Warn($"{contextPrefix} #{response.RequestNumber} {response.Message} - {(int)response.StatusCode} - {response.Content}");
}
if (_log.IsDebugEnabled)
{
_log.Debug(
$"{Environment.NewLine}{resource.XElement}{Environment.NewLine}{LogContext.JsonPrettify(resource.Json)}");
}
}
}
public bool Process(ApiLoaderWorkItem resourceWorkItem)
{
var hash = _xmlHashProvider.Hash(resourceWorkItem.XElement.ToString());
resourceWorkItem.SetHash(hash);
return(true);
}
public bool Process(ApiLoaderWorkItem resourceWorkItem)
{
var values = new[]
{
$"{resourceWorkItem.ElementName}Extension", resourceWorkItem.ElementName
};
var map = _mappings.SingleOrDefault(m => values.Contains(m.SourceName));
if (map == null)
{
return false;
}
var jsonXElement = new XElement(map.TargetName);
foreach (var element in resourceWorkItem.XElement.Elements())
{
var path = element.Name.LocalName;
PerformElementMapping(map, element, path, jsonXElement);
}
resourceWorkItem.SetJsonXElement(jsonXElement);
return true;
}
public async Task <ApiLoaderWorkItem> ProcessAsync(ApiLoaderWorkItem resourceWorkItem)
{
var contextPrefix = LogContext.BuildContextPrefix(resourceWorkItem);
var count = Interlocked.Increment(ref _count);
_log.Debug($"{contextPrefix} #{count} submitting");
var refreshToken = false;
while (true)
{
using (var response = await _poster.PostResource(
resourceWorkItem.Json, resourceWorkItem.ElementName,
resourceWorkItem.ResourceSchemaName, refreshToken).ConfigureAwait(false))
{
if (response.StatusCode.Equals(HttpStatusCode.Unauthorized) && !refreshToken)
{
_log.Info("Expired token detected, refreshing and retrying request.");
refreshToken = true;
continue;
}
// if we refreshed the token we should then disable refresh token until it is needed again.
if (refreshToken)
{
refreshToken = false;
}
resourceWorkItem.AddSubmissionResult(response, count);
}
return(resourceWorkItem);
}
}
private void SetNonZeroExitCode(ApiLoaderWorkItem resource)
{
if (resource.IsSuccess)
{
return;
}
_bulkLoadClientResult.ExitCode = 1;
}
public bool Process(ApiLoaderWorkItem resourceWorkItem)
{
if (_hashCache.Exists(resourceWorkItem.Hash))
{
return(false);
}
_hashCache.Visited(resourceWorkItem.Hash);
return(true);
}
public void Should_map_to_deep_elements()
{
const string data = "value";
var sourceElement = XElement.Parse($"<A><C>{data}</C></A>");
var resourceWorkItem = new ApiLoaderWorkItem("source.xml", 1, sourceElement, 1);
Assert.IsTrue(_step.Process(resourceWorkItem));
dynamic result = JObject.Parse(resourceWorkItem.Json);
Assert.AreEqual(data, result.C.D.E.F.Value);
}
public async Task <ApiLoaderWorkItem> ProcessAsync(ApiLoaderWorkItem resourceWorkItem)
{
var contextPrefix = LogContext.BuildContextPrefix(resourceWorkItem);
var sw = Stopwatch.StartNew();
var count = Interlocked.Increment(ref _count);
var results = await _next.ProcessAsync(resourceWorkItem).ConfigureAwait(false);
sw.Stop();
_log.Debug(
$"{contextPrefix} #{count} completed in {Math.Round((decimal) sw.Elapsed.TotalMilliseconds, 3)} milliseconds");
_resourceStatistic.AddOrUpdate(resourceWorkItem, sw.ElapsedMilliseconds);
return(results);
}
public void AddOrUpdate(ApiLoaderWorkItem resourceWorkItem, double milliSeconds)
{
if (_statisticsByResource.ContainsKey(resourceWorkItem.ElementName))
{
var statistic = _statisticsByResource[resourceWorkItem.ElementName];
statistic.TotalMilliseconds += milliSeconds;
statistic.NumberOfResources++;
}
else
{
var statistic = new Statistic
{
NumberOfResources = 1,
TotalMilliseconds = milliSeconds
};
_statisticsByResource[resourceWorkItem.ElementName] = statistic;
}
}
public void Setup()
{
((Hierarchy)LogManager.GetRepository()).Root.AddAppender(this);
_logs = new List <LoggingEvent>();
_cacheProvider = new TestCacheProvider(new XmlReferenceCache(CreateEmptyMetadata()));
_metadata = new XmlModelMetadata
{
Model = "A", Property = "B", Type = "string"
};
var element = XElement.Parse("<A><ResourceReference ref='1'/></A>");
_step = new ResolveReferenceStep(
_cacheProvider, new[]
{
_metadata
});
_resourceWorkItem = new ApiLoaderWorkItem("filename", 1, element, 1);
}
public bool Process(ApiLoaderWorkItem resourceWorkItem)
{
var values = new[]
{
$"{resourceWorkItem.ElementName}Extension", resourceWorkItem.ElementName
};
var xmlModelMetadata = _metadata.FirstOrDefault(x => values.Contains(x.Model));
if (xmlModelMetadata == null)
{
return(false);
}
var targetModel = xmlModelMetadata.Model;
var referenceCache =
new Lazy <IXmlReferenceCache>(
() =>
_referenceCacheProvider.GetXmlReferenceCache(resourceWorkItem.SourceFileName));
ResolveReferences(referenceCache, resourceWorkItem.XElement, targetModel);
return(true);
}
public static string BuildContextPrefix(ApiLoaderWorkItem resourceWorkItem)
{
return(BuildContextPrefix(
resourceWorkItem.SourceFileName, resourceWorkItem.ElementName, resourceWorkItem.LineNumber,
resourceWorkItem.Level));
}
