public async Task <IMessage> ProvideAsync(CancellationToken cancellationToken)
{
var fileContentCache = (Cache <string>)_messageCache;
if (string.IsNullOrEmpty(fileContentCache.Item))
{
Stream fileStream = await _streamProvider.Provide(cancellationToken);
if (fileStream == null)
{
return(null);
}
fileStream.Seek(0, SeekOrigin.Begin);
UTF8Encoding utF8Encoding = new UTF8Encoding(false, true);
using (StreamReader reader = new StreamReader(fileStream, utF8Encoding, true, 1024, true))
{
fileContentCache.Item = reader.ReadToEnd();
}
fileStream.Seek(0, SeekOrigin.Begin);
return(_xmlSerializationService.Deserialize <Message>(fileStream));
}
return(_xmlSerializationService.Deserialize <Message>(fileContentCache.Item));
}
private async Task <object> ProvideXmlAsync <T>(string fileName, string container, CancellationToken cancellationToken)
{
using (var fileStream = await _fileService.OpenReadStreamAsync(fileName, container, cancellationToken))
{
return(_xmlSerializationService.Deserialize <T>(fileStream));
}
}
public T RetrieveDataFromSyndicationItem(SyndicationItem syndicationItem)
{
using (var stringWriter = new StringWriter())
{
using (var xmlWriter = XmlWriter.Create(stringWriter))
{
syndicationItem.Content.WriteTo(xmlWriter, "temp", "temp");
}
var fundingClaimFeedItem = XDocument.Parse(stringWriter.ToString()).Elements().Descendants();
using (var memoryStream = new MemoryStream())
{
using (var xmlWriter = XmlWriter.Create(memoryStream))
{
fundingClaimFeedItem.First().WriteTo(xmlWriter);
}
memoryStream.Seek(0, SeekOrigin.Begin);
var model = _xmlserializationService.Deserialize <T>(memoryStream);
return(model);
}
}
}
public void Load()
{
var xml = XElement.Load(_configuration.CalculationMethodRepositoryFile);
int version = _xmlSerializationService.VersionFrom(xml);
_xmlSerializationService.Deserialize <ICoreCalculationMethodRepository>(xml, _context.CurrentProject, version).All().Each(_calculationMethodRepository.AddCalculationMethod);
}
public (Guid syndicationItemId, contract contract) RetrieveContractFromSyndicationItem(SyndicationItem syndicationItem)
{
using (var stringWriter = new StringWriter())
{
using (var xmlWriter = XmlWriter.Create(stringWriter))
{
syndicationItem.Content.WriteTo(xmlWriter, "temp", "temp");
}
var contract = XDocument.Parse(stringWriter.ToString()).Descendants().Where(x => x.Name.LocalName == "contract").ToList();
using (var memoryStream = new MemoryStream())
{
using (var xmlWriter = XmlWriter.Create(memoryStream))
{
contract.First().WriteTo(xmlWriter);
}
memoryStream.Seek(0, SeekOrigin.Begin);
var deserializedContract = _xmlserializationService.Deserialize <contract>(memoryStream);
return(Guid.Parse(syndicationItem.Id.Remove(0, 5)), deserializedContract);
}
}
}
public async Task <IMessage> ProvideAsync(IReferenceDataContext referenceDataContext, CancellationToken cancellationToken)
{
using (var stream = await _fileService.OpenReadStreamAsync(referenceDataContext.FileReference, referenceDataContext.Container, cancellationToken))
{
stream.Position = 0;
return(_xmlSerializationService.Deserialize <Message>(stream));
}
}
public async Task <IMessage> ProvideAsync(CancellationToken cancellationToken)
{
using (var stream = await _streamProvider.Provide(cancellationToken))
{
stream.Seek(0, SeekOrigin.Begin);
return(_xmlSerializationService.Deserialize <Message>(stream));
}
}
public async Task <IMessage> ProvideAsync(IFundingServiceContext fundingServiceContext, CancellationToken cancellationToken)
{
Message message;
using (var fileStream = await _fileService.OpenReadStreamAsync(fundingServiceContext.FileReference, fundingServiceContext.Container, cancellationToken))
{
message = _xmlSerializationService.Deserialize <Message>(fileStream);
}
return(message);
}
public async Task <Message> ProvideAsync(IFileValidationContext fileValidationContext, CancellationToken cancellationToken)
{
using (var stream = await _fileService.OpenReadStreamAsync(fileValidationContext.FileReference, fileValidationContext.Container, cancellationToken))
{
_xsdValidationService.Validate(stream);
stream.Position = 0;
return(_xmlSerializationService.Deserialize <Message>(stream));
}
}
public async Task <Message> ProvideAsync(IDataStoreContext dataStoreContext, CancellationToken cancellationToken)
{
try
{
using (var stream = await _fileService.OpenReadStreamAsync(dataStoreContext.Filename, dataStoreContext.Container, cancellationToken))
{
return(_xmlSerializationService.Deserialize <Message>(stream));
}
}
catch (Exception ex)
{
_logger.LogError("Failed to retrieve and deserialise message", ex);
throw;
}
}
public async Task <AmalgamationRoot> ProvideAsync(string filepath, CancellationToken cancellationToken)
{
string filename = Path.GetFileName(filepath);
string container = Path.GetDirectoryName(filepath);
using (var stream = await _fileService.OpenReadStreamAsync(filename, container, cancellationToken))
{
var message = _xmlSerializationService.Deserialize <Message>(stream);
return(new AmalgamationRoot()
{
Filename = filename,
Message = message
});
}
}
public async Task <Scene2Ds> LoadScene2DAsync()
{
try
{
var path = GetScene2DPath();
var fileContent = await _fileStorage.ReadAsStringAsync(path);
return(_xmlSerializationService.Deserialize <Scene2Ds>(fileContent));
}
catch (DirectoryNotFoundException)
{
return(null);
}
catch (FileNotFoundException)
{
return(null);
}
}
public IEnumerable <T> LoadMany <T>(string fileName, bool resetIds = false)
{
_projectConverterLogger.Clear();
XElement xelRoot = XElement.Load(fileName);
if (string.IsNullOrEmpty(fileName))
{
return(Enumerable.Empty <T>());
}
var possibleElementsToDeserialize = retrieveElementsToDeserializeFromFile <T>(xelRoot, fileName).ToList();
var selection = possibleElementsToDeserialize.Count > 1
? selectToDeserialize(possibleElementsToDeserialize, _objectTypeResolver.TypeFor <T>())
: possibleElementsToDeserialize;
if (!selection.Any())
{
return(Enumerable.Empty <T>());
}
var deserializedObjects = new List <T>();
int version = _xmlSerializationService.VersionFrom(xelRoot);
var currentProject = _context.CurrentProject;
selection.Each(element => deserializedObjects.Add(_xmlSerializationService.Deserialize <T>(element, currentProject, version)));
var notificationMessages = _projectConverterLogger.AllMessages().ToList();
if (notificationMessages.Any())
{
_context.PublishEvent(new ShowNotificationsEvent(notificationMessages));
}
_postSerializationSteps.PerformPostDeserializationFor(deserializedObjects, version, resetIds);
return(deserializedObjects);
}
public async Task <IMessage> Provide(string fileName)
{
Message message = null;
try
{
using (MemoryStream ms = new MemoryStream())
{
await _blobStoragePersistenceService.GetAsync(fileName, ms).ConfigureAwait(false);
ms.Seek(0, SeekOrigin.Begin);
message = _xmlSerializationService.Deserialize <Message>(ms);
}
}
catch (Exception ex)
{
_logger.LogError($"Failed to retrieve and deserialise ILR file {fileName}", ex);
throw;
}
return(message);
}
public async Task <bool> MapFileAsync(string sourceFileReference, string sourceFileContainer, string targetFileContainer, CancellationToken cancellationToken)
{
const int taskCount = 7;
var currentTask = 0;
_logger.LogInfo($"Mapping {sourceFileReference} into {targetFileContainer}");
var timer = new Stopwatch();
timer.Start();
try
{
_messengerService.Send(new TaskProgressMessage("Starting", currentTask++, taskCount));
// Generate new filename
var targetFileReference = GenerateOutputName(sourceFileReference);
_logger.LogInfo($"Updated filename from {sourceFileReference} is {targetFileReference}");
// Need to do extract from zip (if relevant) here
Loose.Previous.Message sourceMessage = null;
using (var sourceStream = await _fileService.OpenReadStreamAsync(sourceFileReference, sourceFileContainer, new System.Threading.CancellationToken()))
{
_logger.LogVerbose($"Read in {timer.ElapsedMilliseconds}ms");
timer.Restart();
// Validation against XSD
if (!ValidateSchema(sourceStream))
{
return(false);
}
_logger.LogVerbose($"Schema validated in {timer.ElapsedMilliseconds}ms");
_messengerService.Send(new TaskProgressMessage("Schema validated", currentTask++, taskCount));
timer.Restart();
sourceMessage = _xmlSerializationService.Deserialize <Loose.Previous.Message>(sourceStream);
_logger.LogVerbose($"Deserialize in {timer.ElapsedMilliseconds}ms");
_messengerService.Send(new TaskProgressMessage("File loaded", currentTask++, taskCount));
timer.Restart();
}
// Map from previous year to current year structure via automapper
var targetMessage = _mapper.Map(sourceMessage);
_logger.LogVerbose($"Mapped in {timer.ElapsedMilliseconds}ms");
_messengerService.Send(new TaskProgressMessage("Mapped to current year structure", currentTask++, taskCount));
timer.Restart();
// Uplift any relevant values in the current year structure
var upliftedMessage = _yearUplifter.Process(targetMessage);
_logger.LogVerbose($"Uplifted in {timer.ElapsedMilliseconds}ms");
_messengerService.Send(new TaskProgressMessage("Values uplifted for current year", currentTask++, taskCount));
timer.Restart();
// Anonymise any PII information in the current year structure
var anonymisedMessage = _anonymiser.Process(upliftedMessage);
_logger.LogVerbose($"Anonymised in {timer.ElapsedMilliseconds}ms");
_messengerService.Send(new TaskProgressMessage("Anonymised for current year", currentTask++, taskCount));
timer.Restart();
// Write out the current year structure
using (var targetStream = await _fileService.OpenWriteStreamAsync(targetFileReference, targetFileContainer, new System.Threading.CancellationToken()))
{
_logger.LogVerbose($"Get Out Stream in {timer.ElapsedMilliseconds}ms");
timer.Restart();
_xmlSerializationService.Serialize <Loose.Message>(anonymisedMessage, targetStream);
_logger.LogVerbose($"Serialize in {timer.ElapsedMilliseconds}ms");
timer.Restart();
await targetStream.FlushAsync();
_logger.LogVerbose($"Flush in {timer.ElapsedMilliseconds}ms");
timer.Restart();
}
// Write out the anonymisation lookup details (LRN and ULN)
if (_anonymiseLog?.Log.Any() == true)
{
using (var targetStream = await _fileService.OpenWriteStreamAsync(
targetFileReference + ".CSV",
targetFileContainer,
new System.Threading.CancellationToken()))
{
var newLineBytes = Encoding.ASCII.GetBytes(Environment.NewLine);
foreach (var logEntry in _anonymiseLog.Log)
{
var reportLine = $"{logEntry.FieldName} {logEntry.OldValue} {logEntry.NewValue}";
var reportLineBytes = Encoding.ASCII.GetBytes(reportLine);
targetStream.Write(reportLineBytes, 0, reportLineBytes.Length);
targetStream.Write(newLineBytes, 0, newLineBytes.Length);
}
await targetStream.FlushAsync();
_anonymiseLog.Clear();
}
}
_messengerService.Send(_validationErrorHandler.ErrorRaised ?
new TaskProgressMessage("File saved - Completed with XML Validation warnings - Please check logs", currentTask++, taskCount) :
new TaskProgressMessage("File saved - Completed", currentTask++, taskCount));
}
catch (Exception ex)
{
_logger.LogFatal($"Failed mapping {sourceFileReference} into {targetFileContainer}", ex);
return(false);
}
return(true);
}
private T deserializeContent <T>(MetaDataContent content, SerializationContext serializationContext)
{
return(_serializationService.Deserialize <T>(content.Data, _project, serializationContext));
}
