/// <summary>
/// Removes a spatial stream from the persistent store identified by its spatial descriptor attributes
/// </summary>
public FileSystemErrorStatus RemoveSpatialStreamFromPersistentStore(Guid dataModelId,
string streamName,
int subGridX, int subGridY,
long segmentStartDateTicks,
long segmentEndDateTicks,
long version,
FileSystemStreamType streamType)
{
try
{
var cacheKey = new SubGridSpatialAffinityKey(version, dataModelId, subGridX, subGridY, segmentStartDateTicks, segmentEndDateTicks);
try
{
SpatialCache(streamType).Remove(cacheKey);
}
catch (KeyNotFoundException)
{
return(FileSystemErrorStatus.GranuleDoesNotExist);
}
ImmutableProxy?.RemoveSpatialStreamFromPersistentStore(dataModelId, streamName, subGridX, subGridY, segmentStartDateTicks, segmentEndDateTicks, version, streamType);
return(FileSystemErrorStatus.OK);
}
catch (Exception e)
{
_log.LogError(e, "Exception occurred:");
return(FileSystemErrorStatus.UnknownFailureRemovingFileFromFS);
}
}
/// <summary>
/// Supports removing a named stream from the persistent store via the grid cache
/// </summary>
public FileSystemErrorStatus RemoveStreamFromPersistentStore(Guid dataModelId,
FileSystemStreamType streamType,
string streamName)
{
try
{
var cacheKey = ComputeNamedStreamCacheKey(dataModelId, streamName);
if (_log.IsTraceEnabled())
{
_log.LogInformation($"Removing key:{cacheKey}");
}
// Remove item from both immutable and mutable caches
try
{
NonSpatialCache(streamType).Remove(cacheKey);
}
catch (KeyNotFoundException e)
{
_log.LogError(e, "Exception occurred:");
}
ImmutableProxy?.RemoveStreamFromPersistentStore(dataModelId, streamType, streamName);
return(FileSystemErrorStatus.OK);
}
catch (Exception e)
{
_log.LogError(e, $"Exception removing stream {streamName} from persistent store");
return(FileSystemErrorStatus.UnknownErrorWritingToFS);
}
}
/// <summary>
/// Primary method for performing mutability conversion to immutable. It accepts either
/// a) a sourceObject, from which the immutable stream can be built directly, or
/// b) a stream, which must be de-serialized into a source object, from which the immutable stream can be built
/// i.e. either mutableStream, or source are null
/// </summary>
/// <param name="streamType"></param>
/// <param name="mutableStream"></param>
/// <param name="source"></param>
/// <param name="immutableStream"></param>
/// <returns></returns>
public bool ConvertToImmutable(FileSystemStreamType streamType, MemoryStream mutableStream, object source, out MemoryStream immutableStream)
{
immutableStream = null;
if (mutableStream == null && source == null)
{
throw new TRexException("Unable to determine a single valid source for immutability conversion.");
}
bool result;
switch (streamType)
{
case FileSystemStreamType.SubGridDirectory:
{
result = source == null
? ConvertSubGridDirectoryToImmutable(mutableStream, out immutableStream)
: ConvertSubGridDirectoryToImmutable(source, out immutableStream);
break;
}
case FileSystemStreamType.SubGridSegment:
{
result = source == null
? ConvertSubGridSegmentToImmutable(mutableStream, out immutableStream)
: ConvertSubGridSegmentToImmutable(source, out immutableStream);
break;
}
case FileSystemStreamType.Events:
{
result = source == null
? ConvertEventListToImmutable(mutableStream, out immutableStream)
: ConvertEventListToImmutable(source, out immutableStream);
break;
}
default:
{
// EG: Sub grid existence map etc
immutableStream = mutableStream;
result = true;
break;
}
}
if (mutableStream != null && immutableStream != null && Log.IsTraceEnabled())
{
Log.LogInformation($"Mutability conversion: Type:{streamType}, Initial Size: {mutableStream.Length}, Final Size: {immutableStream.Length}, Ratio: {(immutableStream.Length/(1.0*mutableStream.Length)) * 100}%");
}
return(result);
}
public static string SpatialCacheName(StorageMutability mutability, FileSystemStreamType streamType)
{
return(streamType switch
{
FileSystemStreamType.ProductionDataXML => SiteModelsCacheName(mutability),
FileSystemStreamType.SubGridDirectory => SpatialSubGridDirectoryCacheName(mutability),
FileSystemStreamType.SubGridSegment => SpatialSubGridSegmentCacheName(mutability),
FileSystemStreamType.SubGridExistenceMap => ProductionDataExistenceMapCacheName(mutability),
_ => string.Empty,
});
public void FromBinary(IBinaryRawReader reader)
{
var version = VersionSerializationHelper.CheckVersionByte(reader, VERSION_NUMBER);
if (version == 1)
{
ProjectUID = reader.ReadGuid() ?? Guid.Empty;
AssetUID = reader.ReadGuid() ?? Guid.Empty;
StreamType = (FileSystemStreamType)reader.ReadInt();
}
}
/// <summary>
/// Supports writing a named data stream to the persistent store via the grid cache.
/// </summary>
public FileSystemErrorStatus WriteStreamToPersistentStore(Guid dataModelId,
string streamName,
FileSystemStreamType streamType,
MemoryStream mutableStream,
object source)
{
try
{
var cacheKey = ComputeNamedStreamCacheKey(dataModelId, streamName);
using (var compressedStream = MemoryStreamCompression.Compress(mutableStream))
{
if (_log.IsTraceEnabled())
{
_log.LogInformation($"Putting key:{cacheKey} in {NonSpatialCache(streamType).Name}, size:{mutableStream.Length} -> {compressedStream.Length}, ratio:{(compressedStream.Length / (1.0 * mutableStream.Length)) * 100}%");
}
NonSpatialCache(streamType).Put(cacheKey, new SerialisedByteArrayWrapper(compressedStream.ToArray()));
}
try
{
// Create the immutable stream from the source data
if (Mutability == StorageMutability.Mutable && ImmutableProxy != null)
{
if (!PerformNonSpatialImmutabilityConversion(mutableStream, ImmutableProxy.NonSpatialCache(streamType), cacheKey, streamType, source))
{
_log.LogError("Unable to project an immutable stream");
return(FileSystemErrorStatus.MutableToImmutableConversionError);
}
}
}
catch (Exception e)
{
_log.LogError(e, $"Exception performing mutability conversion in {nameof(WriteStreamToPersistentStore)}");
return(FileSystemErrorStatus.MutableToImmutableConversionError);
}
return(FileSystemErrorStatus.OK);
}
catch (Exception e)
{
_log.LogError(e, $"Exception writing stream {streamName} to persistent store");
return(FileSystemErrorStatus.UnknownErrorWritingToFS);
}
}
/// <summary>
/// Supports writing a spatial data stream to the persistent store via the grid cache.
/// </summary>
public FileSystemErrorStatus WriteSpatialStreamToPersistentStore(Guid dataModelId,
string streamName,
int subGridX, int subGridY,
long segmentStartDateTicks, long segmentEndDateTicks,
long version,
FileSystemStreamType streamType,
MemoryStream mutableStream,
object source)
{
try
{
var cacheKey = new SubGridSpatialAffinityKey(version, dataModelId, subGridX, subGridY, segmentStartDateTicks, segmentEndDateTicks);
using (var compressedStream = MemoryStreamCompression.Compress(mutableStream))
{
var spatialCache = SpatialCache(streamType);
if (_log.IsTraceEnabled())
{
_log.LogInformation($"Putting key:{cacheKey} in {spatialCache.Name}, size:{mutableStream.Length} -> {compressedStream.Length}, ratio:{(compressedStream.Length / (1.0 * mutableStream.Length)) * 100}%");
}
spatialCache.Put(cacheKey, new SerialisedByteArrayWrapper(compressedStream.ToArray()));
}
// Convert the stream to the immutable form and write it to the immutable storage proxy
try
{
if (Mutability == StorageMutability.Mutable && ImmutableProxy != null)
{
PerformSpatialImmutabilityConversion(mutableStream, ImmutableProxy.SpatialCache(streamType), cacheKey, streamType, source);
}
}
catch (Exception e)
{
_log.LogError(e, $"Exception performing mutability conversion in {nameof(WriteSpatialStreamToPersistentStore)}");
return(FileSystemErrorStatus.MutableToImmutableConversionError);
}
return(FileSystemErrorStatus.OK);
}
catch (Exception e)
{
_log.LogError(e, $"Exception writing spatial stream {streamName} to persistent store");
return(FileSystemErrorStatus.UnknownErrorWritingToFS);
}
}
public void ExtractSiteModelFile(string fileName, FileSystemStreamType streamType)
{
var readResult = _siteModel.PrimaryStorageProxy.ReadStreamFromPersistentStore(_siteModel.ID, fileName, streamType, out var MS);
if (readResult != FileSystemErrorStatus.OK || MS == null)
{
Log.LogInformation($"Failed to read file {fileName} of type {streamType}, (readResult = {readResult}), or stream is null");
Console.WriteLine($"Failed to read file {fileName} of type {streamType}, (readResult = {readResult}), or stream is null");
Console.WriteLine($"Failed to read existence map (readResult = {readResult}), or stream is null");
}
else
{
using (MS)
{
var basePath = Path.Combine(_projectOutputPath);
Directory.CreateDirectory(basePath);
File.WriteAllBytes(Path.Combine(basePath, fileName), MS.ToArray());
}
}
}
/// <summary>
/// Supports reading a stream of spatial data from the persistent store via the grid cache
/// </summary>
public FileSystemErrorStatus ReadSpatialStreamFromPersistentStore(Guid dataModelId,
string streamName,
int subGridX, int subGridY,
long segmentStartDateTicks,
long segmentEndDateTicks,
long version,
FileSystemStreamType streamType,
out MemoryStream stream)
{
stream = null;
try
{
var cacheKey = new SubGridSpatialAffinityKey(version, dataModelId, subGridX, subGridY, segmentStartDateTicks, segmentEndDateTicks);
//Log.LogInformation($"Getting key:{streamName}");
try
{
using var ms = new MemoryStream(SpatialCache(streamType).Get(cacheKey).Bytes);
stream = MemoryStreamCompression.Decompress(ms);
stream.Position = 0;
}
catch (KeyNotFoundException)
{
return(FileSystemErrorStatus.GranuleDoesNotExist);
}
return(FileSystemErrorStatus.OK);
}
catch (Exception e)
{
_log.LogError(e, "Exception occurred:");
stream = null;
return(FileSystemErrorStatus.UnknownErrorReadingFromFS);
}
}
FileSystemErrorStatus IStorageProxy.WriteStreamToPersistentStore(Guid dataModelID, string streamName, FileSystemStreamType streamType,
MemoryStream mutablestream, object source)
{
throw new NotImplementedException();
}
/// <summary>
/// TAG File Buffer Queue key constructor taking project, asset and filename
/// </summary>
/// <param name="projectUID"></param>
/// <param name="assetUID"></param>
/// <param name="streamType"></param>
public SiteModelMachineAffinityKey(Guid projectUID, Guid assetUID, FileSystemStreamType streamType)
{
ProjectUID = projectUID;
AssetUID = assetUID;
StreamType = streamType;
}
/// <summary>
/// Determines the correct cache to read/write particular types of information from/to
/// </summary>
public IStorageProxyCache <INonSpatialAffinityKey, ISerialisedByteArrayWrapper> NonSpatialCache(FileSystemStreamType streamType)
{
return(streamType switch
{
FileSystemStreamType.ProductionDataXML => siteModelCache,
FileSystemStreamType.SubGridExistenceMap => spatialDataExistenceMapCache,
FileSystemStreamType.DesignTopologyExistenceMap => designTopologyExistenceMapsCache,
_ => generalNonSpatialCache
});
public IStorageProxyCache <ISiteModelMachineAffinityKey, ISerialisedByteArrayWrapper> ProjectMachineCache(FileSystemStreamType streamType)
{
throw new NotImplementedException();
}
public FileSystemErrorStatus RemoveSpatialStreamFromPersistentStore(Guid dataModelID, string streamName, int subGridX, int subGridY, long segmentStartDateTicks, long segmentEndDateTicks, long version, FileSystemStreamType streamType)
{
throw new NotImplementedException();
}
FileSystemErrorStatus IStorageProxy.RemoveStreamFromPersistentStore(Guid dataModelID, FileSystemStreamType streamType, string streamName)
{
throw new NotImplementedException();
}
FileSystemErrorStatus IStorageProxy.ReadSpatialStreamFromPersistentStore(Guid dataModelID, string streamName, int subGridX, int subGridY, long segmentStartDateTicks, long segmentEndDateTicks, long version, FileSystemStreamType streamType, out MemoryStream stream)
{
throw new NotImplementedException();
}
FileSystemErrorStatus IStorageProxy.WriteSpatialStreamToPersistentStore(Guid dataModelID, string streamName, int subGridX, int subGridY, long segmentStartDateTicks, long segmentEndDateTicks,
long version, FileSystemStreamType streamType, MemoryStream mutableStream, object source)
{
throw new NotImplementedException();
}
/// <summary>
/// Supports reading a named stream from the persistent store via the grid cache
/// </summary>
public FileSystemErrorStatus ReadStreamFromPersistentStore(Guid dataModelId, string streamName, FileSystemStreamType streamType, out MemoryStream stream)
{
stream = null;
try
{
var cacheKey = ComputeNamedStreamCacheKey(dataModelId, streamName);
//Log.LogInformation($"Getting key:{cacheKey}");
try
{
using var ms = new MemoryStream(NonSpatialCache(streamType).Get(cacheKey).Bytes);
stream = MemoryStreamCompression.Decompress(ms);
stream.Position = 0;
}
catch (KeyNotFoundException)
{
return(FileSystemErrorStatus.GranuleDoesNotExist);
}
return(FileSystemErrorStatus.OK);
}
catch (Exception e)
{
_log.LogError(e, "Exception occurred:");
stream = null;
return(FileSystemErrorStatus.UnknownErrorReadingFromFS);
}
}
public IStorageProxyCache <INonSpatialAffinityKey, ISerialisedByteArrayWrapper> NonSpatialCache(FileSystemStreamType streamType) => null; // Not implemented public IStorageProxyCache <ISubGridSpatialAffinityKey, ISerialisedByteArrayWrapper> SpatialCache(FileSystemStreamType streamType) => null; // Not implemented
