public void Load(IStateSerializer stateSerializer)
{
foreach (var item in _storage.Get <ILoadState>())
{
item.Load(stateSerializer);
}
}
public CheckpointFrame(IStateSerializer <T> valueSerializer)
{
this.valueSerializer = valueSerializer;
this.writer = new InMemoryBinaryWriter(new MemoryStream());
this.ListElementsCount = 0;
this.listElementsBytes = 0;
}
public BlockAlignedWriter(
FileStream valueFileStream, FileStream keyFileStream, ValueCheckpointFile valueCheckpointFile, KeyCheckpointFile keyCheckpointFile,
InMemoryBinaryWriter valueBuffer, InMemoryBinaryWriter keyBuffer, IStateSerializer <TValue> valueSerializer, IStateSerializer <TKey> keySerializer,
long timeStamp, string traceType)
{
this.valueCheckpointFile = valueCheckpointFile;
this.keyCheckpointFile = keyCheckpointFile;
this.timeStamp = timeStamp;
this.valueFileStream = valueFileStream;
this.keyFileStream = keyFileStream;
this.valueBuffer = valueBuffer;
this.keyBuffer = keyBuffer;
this.valueSerializer = valueSerializer;
this.keySerializer = keySerializer;
this.traceType = traceType;
this.keyBlockAlignedWriter = new KeyBlockAlignedWriter <TKey, TValue>(
keyFileStream,
keyCheckpointFile,
keyBuffer,
keySerializer,
timeStamp,
traceType);
this.valueBlockAlignedWriter = new ValueBlockAlignedWriter <TKey, TValue>(
valueFileStream,
valueCheckpointFile,
valueBuffer,
valueSerializer,
traceType);
}
public virtual async Task <TValue> GetValueAsync(FileMetadata fileMetadata, IStateSerializer <TValue> valueSerializer, bool isValueAReferenceType, ReadMode readMode, LoadValueCounter valueCounter, CancellationToken cancellationToken, string traceType)
{
TValue value = this.Value;
if (this.ShouldValueBeLoadedFromDisk(isValueAReferenceType, value, traceType) == false || readMode == ReadMode.Off)
{
return(value);
}
Diagnostics.Assert(readMode != ReadMode.CacheResult || isValueAReferenceType == false || this.InUse == true, traceType, "If CacheResult, InUse must have been set.");
value = await MetadataManager.ReadValueAsync <TValue>(fileMetadata, this, valueSerializer, cancellationToken, traceType).ConfigureAwait(false);
// Value must be set before updating the flags.
// Example problem: After recovery two reads on the same key,
// T1 sees InUse (IsLoaded) == false, reads the value, updates InUse, *Context Switch*
// T2 sees InUse (IsLoaded) == true, reads this.Value which has not been set by T1.
if (this.ShouldCacheValue(readMode, isValueAReferenceType))
{
this.Value = value;
valueCounter.IncrementCounter();
}
// Flags must always be updated. For value type, inUse (IsLoadedFromDiskAfterRecovery) must be set.
this.UpdateFlagsFollowingLoadValue(isValueAReferenceType, value, readMode, traceType);
return(value);
}
public bool TryAddStateSerializer <T>(Uri name, IStateSerializer <T> stateSerializer)
{
// TODO: Add Support for prefix matching.
var namedType = new NamedType(name, typeof(T));
return(this.serializerDictionary.TryAdd(namedType, stateSerializer));
}
public void Save(IStateSerializer stateSerializer)
{
foreach (var item in _storage.Get <ISaveLoadState>())
{
item.Save(stateSerializer);
}
}
public KeyCheckpointFileAsyncEnumerator <TKey, TValue> GetAsyncEnumerator <TKey, TValue>(
IStateSerializer <TKey> keySerializer,
string traceType,
Kernel32Types.PRIORITY_HINT priorityHint = Kernel32Types.PRIORITY_HINT.IoPriorityHintNormal)
{
return(this.KeyCheckpointFile.GetAsyncEnumerator <TKey, TValue>(keySerializer, traceType, priorityHint));
}
public void Load(IStateSerializer stateSerializer)
{
foreach (var item in _items)
{
item.Load(_stateSerializer);
}
}
public UserInstanceStateProvider(
IStateSerializer stateSerializer,
IUserInstanceStateStore store)
{
_stateSerializer = stateSerializer ?? throw new ArgumentNullException(nameof(stateSerializer));
_store = store ?? throw new ArgumentNullException(nameof(store));
}
public CheckpointManager(Checkpoint <T> currentCheckpoint, string directory, string FileName, IStateSerializer <T> valueSerializer, string traceType)
{
this.traceType = traceType;
this.CurrentCheckpoint = currentCheckpoint;
this.directory = directory;
this.FileName = FileName;
this.valueSerializer = valueSerializer;
}
private static async Task <ConditionalValue <CheckpointManager <T> > > TryReadCheckpointFile(
string directory,
string fileName,
IStateSerializer <T> valueSerializer,
string traceType)
{
var filePath = Path.Combine(directory, fileName);
if (!FabricFile.Exists(filePath))
{
return(new ConditionalValue <CheckpointManager <T> >(false, default(CheckpointManager <T>)));
}
using (var stream = FabricFile.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.SequentialScan))
{
var intSegment = new ArraySegment <byte>(new byte[sizeof(int)]);
var versionRead = await SerializationHelper.ReadIntAsync(intSegment, stream).ConfigureAwait(false);
if (versionRead != FileVersion)
{
throw new InvalidDataException(string.Format("versionRead '{0}' != FileVersion '{1}'", versionRead, FileVersion));
}
var nameLength = await SerializationHelper.ReadIntAsync(intSegment, stream).ConfigureAwait(false);
if (nameLength < 0)
{
throw new InvalidDataException(string.Format("nameLength '{0}' < 0", nameLength));
}
if (nameLength == 0)
{
return(new ConditionalValue <CheckpointManager <T> >(true, new CheckpointManager <T>(null, directory, fileName, valueSerializer, traceType)));
}
var nameSegment = new ArraySegment <byte>(new byte[nameLength]);
await SerializationHelper.ReadBytesAsync(nameSegment, nameLength, stream).ConfigureAwait(false);
string name;
using (var reader = new InMemoryBinaryReader(new MemoryStream(nameSegment.Array)))
{
name = reader.ReadString();
}
var path = Path.Combine(directory, name);
if (!FabricFile.Exists(path))
{
throw new InvalidDataException(string.Format("Current checkpoint file does not exist: {0}", path));
}
return(new ConditionalValue <CheckpointManager <T> >(
true,
new CheckpointManager <T>(new Checkpoint <T>(directory, name, valueSerializer, traceType), directory, fileName, valueSerializer, traceType)));
}
}
public Checkpoint(string directory, string fileName, List <IListElement <T> > listElements, IStateSerializer <T> valueSerializer, string traceType)
{
this.traceType = traceType;
this.ListElements = listElements;
this.valueSerializer = valueSerializer;
this.FileName = fileName;
this.FilePath = Path.Combine(directory, fileName);
this.refCount = 1;
}
public static Task <TValue> ReadValueAsync <TValue>(
FileMetadata fileMetadata, TVersionedItem <TValue> item, IStateSerializer <TValue> valueSerializer, CancellationToken cancellationToken, string traceType)
{
// Consistency checks.
Diagnostics.Assert(fileMetadata.CheckpointFile != null, traceType, "Checkpoint file with id '{0}' does not exist in memory.", item.FileId);
// Read from disk.
return(fileMetadata.CheckpointFile.ReadValueAsync <TValue>(item, valueSerializer));
}
public void Load(IStateSerializer stateSerializer)
{
try {
Count = stateSerializer.Deserialize <int>(KEY);
}
catch (NotSavedException e) {
Count = Arcanoid.Instance.DefaultParameters.LivesCount;
}
}
Region(string regionName, IViewModelLocator viewModelLocator, IViewLocator viewLocator, IStateSerializer stateSerializer)
{
LogicalSerializationMode = LogicalSerializationMode.Enabled;
this.RegionName = regionName;
this.viewModelLocator = viewModelLocator;
this.viewLocator = viewLocator;
this.stateSerializer = stateSerializer;
this.serviceManager = new WeakReferenceManager <IUIRegion>();
this.items = new List <RegionItem>();
}
public ValueBlockAlignedWriter(
FileStream valueFileStream, ValueCheckpointFile valueCheckpointFile, InMemoryBinaryWriter valueBuffer, IStateSerializer <TValue> valueSerializer,
string traceType)
{
this.valueCheckpointFile = valueCheckpointFile;
this.valueFileStream = valueFileStream;
this.valueBuffer = valueBuffer;
this.valueSerializer = valueSerializer;
this.traceType = traceType;
this.blockAlignmentSize = BlockAlignedWriter <TKey, TValue> .DefaultBlockAlignmentSize;
}
/// <summary>
/// Initialize a new instance of the RecoveryStoreComponent class.
/// </summary>
/// <param name="metadataTable"></param>
/// <param name="workDirectory"></param>
/// <param name="traceType"></param>
/// <param name="keyConverter"></param>
/// <param name="isValueReferenceType"></param>
public RecoveryStoreComponent(MetadataTable metadataTable, string workDirectory, string traceType, IStateSerializer <TKey> keyConverter, bool isValueReferenceType)
{
this.metadataTable = metadataTable;
this.workDirectory = workDirectory;
this.traceType = traceType;
this.keyConverter = keyConverter;
this.fileId = InvalidFileId;
this.logicalCheckpointFileTimeStamp = InvalidTimeStamp;
this.component = new PartitionedSortedList <TKey, TVersionedItem <TValue> >(KeyComparer);
this.isValueReferenceType = isValueReferenceType;
}
/// <summary>
/// Serializes this transaction
/// </summary>
/// <param name="serializer">Serializer to write to</param>
public void Serialize(IStateSerializer serializer)
{
serializer.Serialize(nameof(this.Id), this.Id);
serializer.Serialize(nameof(this.Revision), this.Revision);
serializer.Serialize(nameof(this.Created), this.Created);
serializer.Serialize(nameof(this.Expires), this.Expires);
serializer.Serialize(nameof(this.Expired), this.Expired);
serializer.Serialize(nameof(this.Status), this.Status);
serializer.Serialize(nameof(this.Script), this.Script);
serializer.Serialize(nameof(this.Parent), this.Parent);
serializer.Serialize(nameof(this.Payload), this.Payload);
serializer.Serialize(nameof(this.Error), this.Error);
}
public static CheckpointManager <T> CreateCurrentCheckpointManager(
List <IListElement <T> > listElements,
string directory,
IStateSerializer <T> valueSerializer,
string traceType)
{
return(new CheckpointManager <T>(
new Checkpoint <T>(directory, Checkpoint <T> .GenerateFileName(), listElements, valueSerializer, traceType),
directory,
CurrentCheckpointManagerFileName,
valueSerializer,
traceType));
}
public KeyCheckpointFileAsyncEnumerator <TKey, TValue> GetAsyncEnumerator <TKey, TValue>(
IStateSerializer <TKey> keySerializer,
string traceType,
Kernel32Types.PRIORITY_HINT priorityHint = Kernel32Types.PRIORITY_HINT.IoPriorityHintNormal)
{
return(new KeyCheckpointFileAsyncEnumerator <TKey, TValue>(
this,
keySerializer,
this.Properties.KeysHandle.Offset,
this.Properties.KeysHandle.EndOffset,
traceType,
priorityHint));
}
public KeyBlockAlignedWriter(
FileStream keyFileStream, KeyCheckpointFile keyCheckpointFile, InMemoryBinaryWriter keyBuffer, IStateSerializer <TKey> keySerializer,
long timeStamp, string traceType)
{
this.keyCheckpointFile = keyCheckpointFile;
this.timeStamp = timeStamp;
this.keyFileStream = keyFileStream;
this.keyBuffer = keyBuffer;
this.keySerializer = keySerializer;
this.traceType = traceType;
this.blockAlignmentSize = BlockAlignedWriter <TKey, TValue> .DefaultBlockAlignmentSize;
// Most key records are lesser than 4k, when they get beyond 4k, do not shrink the buffer.
this.tempKeyBuffer = new InMemoryBinaryWriter(new MemoryStream(capacity: 4 * 1024));
}
public MainWindow(MainViewModel mainViewModel, IStateSerializer stateSerializer,ILogger logger)
{
if (mainViewModel == null)
throw new ArgumentNullException("mainViewModel");
if (stateSerializer == null)
throw new ArgumentNullException("stateSerializer");
if (logger == null)
throw new ArgumentNullException("logger");
InitializeComponent();
this.DataContext = mainViewModel;
this.stateSerializer = stateSerializer;
this.logger = logger;
this.Closing += MainWindow_Closing;
this.Dispatcher.UnhandledException += Dispatcher_UnhandledException;
TaskScheduler.UnobservedTaskException += TaskScheduler_UnobservedTaskException;
}
public KeyCheckpointFileAsyncEnumerator(
KeyCheckpointFile keyCheckpointFile,
IStateSerializer <TKey> keySerializer,
long startOffset,
long endOffset,
string traceType,
Kernel32Types.PRIORITY_HINT priorityHint = Kernel32Types.PRIORITY_HINT.IoPriorityHintNormal)
{
this.traceType = traceType;
this.keyCheckpointFile = keyCheckpointFile;
this.stateZero = true;
this.index = -1;
this.keySerializer = keySerializer;
this.startOffset = startOffset;
this.endOffset = endOffset;
this.priorityHint = priorityHint;
}
RegionItem(IViewModelLocator viewModelLocator, IViewLocator viewLocator, IStateSerializer stateSerializer,
string key, Func <object> factory, string viewModelName, string viewName, Type viewType, object parameter)
{
this.viewModelLocator = viewModelLocator;
this.viewLocator = viewLocator;
this.stateSerializer = stateSerializer;
Key = key;
Factory = factory;
if (Factory == null)
{
ViewModelName = viewModelName;
}
ViewType = viewType;
if (ViewType == null)
{
ViewName = viewName;
}
Parameter = parameter;
}
public AsyncEnumerable(
bool isValueAReferenceType,
ReadMode readMode,
IEnumerable <TKey> keyEnumerables,
IReadableStoreComponent <TKey, TVersionedItem <TValue> > differentState,
IReadableStoreComponent <TKey, TVersionedItem <TValue> > consolidatedState,
MetadataTable currentMetadataTable,
LoadValueCounter loadValueCounter,
IStateSerializer <TValue> valueSerializer,
string traceType)
{
this.traceType = traceType;
this.isValueAReferenceType = isValueAReferenceType;
this.readMode = readMode;
this.keyEnumerable = keyEnumerables;
this.differentState = differentState;
this.consolidatedState = consolidatedState;
this.currentMetadataTable = currentMetadataTable;
this.loadValueCounter = loadValueCounter;
this.valueSerializer = valueSerializer;
this.isInvalidated = false;
}
public MainWindow(MainViewModel mainViewModel, IStateSerializer stateSerializer, ILogger logger)
{
if (mainViewModel == null)
{
throw new ArgumentNullException("mainViewModel");
}
if (stateSerializer == null)
{
throw new ArgumentNullException("stateSerializer");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
InitializeComponent();
this.DataContext = mainViewModel;
this.stateSerializer = stateSerializer;
this.logger = logger;
this.Closing += MainWindow_Closing;
this.Dispatcher.UnhandledException += Dispatcher_UnhandledException;
TaskScheduler.UnobservedTaskException += TaskScheduler_UnobservedTaskException;
}
private void WriteValue <TValue>(
InMemoryBinaryWriter memoryBuffer, long basePosition, TVersionedItem <TValue> item, IStateSerializer <TValue> valueSerializer)
{
// Deleted items don't have values. Only serialize valid items.
if (item.Kind != RecordKind.DeletedVersion)
{
// WriteItemAsync valueSerializer followed by checksum.
// Serialize the value.
var valueStartPosition = memoryBuffer.BaseStream.Position;
valueSerializer.Write(item.Value, memoryBuffer);
var valueEndPosition = memoryBuffer.BaseStream.Position;
Diagnostics.Assert(
valueEndPosition >= valueStartPosition,
DifferentialStoreConstants.TraceType,
"User's value IStateSerializer moved the stream position backwards unexpectedly!");
// Write the checksum of just that value's bytes.
var valueSize = checked ((int)(valueEndPosition - valueStartPosition));
var checksum = CRC64.ToCRC64(memoryBuffer.BaseStream.GetBuffer(), checked ((int)valueStartPosition), valueSize);
// Update the in-memory offset and size for this item.
item.Offset = basePosition + valueStartPosition;
item.ValueSize = valueSize;
item.ValueChecksum = checksum;
// Update checkpoint file in-memory metadata.
this.Properties.ValueCount++;
}
// Update the in-memory metadata about which file this key-value exists in on disk.
item.FileId = this.FileId;
}
/// <summary>
/// Add a value to the given file stream, using the memory buffer to stage writes before issuing bulk disk IOs.
/// </summary>
/// <typeparam name="TValue"></typeparam>
/// <param name="fileStream"></param>
/// <param name="memoryBuffer"></param>
/// <param name="item"></param>
/// <param name="valueSerializer"></param>
/// <param name="traceType"></param>
/// <returns></returns>
public void WriteItem <TValue>(
Stream fileStream, InMemoryBinaryWriter memoryBuffer, TVersionedItem <TValue> item, IStateSerializer <TValue> valueSerializer, string traceType)
{
// Write the value into the memory buffer.
this.WriteValue(memoryBuffer, fileStream.Position, item, valueSerializer);
}
/// <summary>
/// Read the given value from disk.
/// </summary>
/// <typeparam name="TValue"></typeparam>
/// <param name="item"></param>
/// <param name="valueSerializer"></param>
/// <returns></returns>
public async Task <TValue> ReadValueAsync <TValue>(TVersionedItem <TValue> item, IStateSerializer <TValue> valueSerializer)
{
// Validate the item has a value that can be read from disk.
if (item == null)
{
throw new ArgumentNullException(SR.Error_Item);
}
if (item.Kind == RecordKind.DeletedVersion)
{
throw new ArgumentException(SR.Error_ValueCheckpoint_DeletedItemValue, SR.Error_Item);
}
// Validate that the item's disk properties are valid.
if (item.Offset < this.Properties.ValuesHandle.Offset)
{
throw new ArgumentOutOfRangeException(SR.Error_Item, SR.Error_ValueCheckpoint_TVersionedItem_Offset_Negative);
}
if (item.ValueSize < 0)
{
throw new ArgumentOutOfRangeException(string.Format(CultureInfo.CurrentCulture, SR.Error_Item, SR.Error_ValueCheckpoint_TVersionedItem_ValueSize_Negative, item.ValueSize));
}
if (item.Offset + item.ValueSize > this.Properties.ValuesHandle.EndOffset)
{
throw new ArgumentOutOfRangeException(SR.Error_Item, SR.Error_ValueCheckpoint_StreamRangeExceeded);
}
Stream fileStream = null;
try
{
// Acquire a re-usable file stream for exclusive use during this read.
fileStream = this.ReaderPool.AcquireStream();
var snapFileStream = fileStream as FileStream;
Diagnostics.Assert(snapFileStream != null, this.traceType, "fileStream must be a FileStream");
Microsoft.ServiceFabric.Replicator.Utility.SetIoPriorityHint(snapFileStream.SafeFileHandle, this.priorityHint);
// TODO: use memory stream pool here.
using (var stream = new MemoryStream(capacity: item.ValueSize))
using (var reader = new BinaryReader(stream))
{
// Read the value bytes and the checksum into memory.
fileStream.Position = item.Offset;
stream.SetLength(item.ValueSize);
await fileStream.ReadAsync(stream.GetBuffer(), 0, item.ValueSize).ConfigureAwait(false);
// Read the checksum from memory.
stream.Position = item.ValueSize;
var checksum = item.ValueChecksum;
// Re-compute the checksum.
var expectedChecksum = CRC64.ToCRC64(stream.GetBuffer(), 0, item.ValueSize);
if (checksum != expectedChecksum)
{
throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_FailedReadValue_ChecksumMismatch_TwoArgs, checksum, expectedChecksum));
}
// Deserialize the value into memory.
stream.Position = 0;
return(valueSerializer.Read(reader));
}
}
finally
{
// Return the file stream to the pool for re-use.
this.ReaderPool.ReleaseStream(fileStream);
}
}
public bool TryAddStateSerializer <T>(IStateSerializer <T> stateSerializer)
{
throw new NotImplementedException();
}
public void RegisterStateSerializer <T>(IStateSerializer <T> serializer) => _registrations.Add(sm => sm.TryAddStateSerializer(serializer));
public DownloaderApplication(IFileOperations fileOperations, IStateSerializer serializer, IFileProcessor fileProcessor)
{
_fileOperations = fileOperations;
_serializer = serializer;
_fileProcessor = fileProcessor;
}
