public async IAsyncEnumerable <PartitionSnapshot> LoadSnapshotsAsync() { const string ignoreContainerName = "nosqlsnapshots"; await foreach (var container in _storageAccount.GetListOfContainersAsync()) { if (container.Name == ignoreContainerName) { continue; } await foreach (var blockBlob in container.GetListOfBlobsAsync()) { var memoryStream = new MemoryStream(); await blockBlob.DownloadToStreamAsync(memoryStream); var snapshot = new PartitionSnapshot { TableName = container.Name, PartitionKey = blockBlob.Name.Base64ToString(), Snapshot = memoryStream.ToArray() }; yield return(snapshot); Console.WriteLine("Loaded snapshot: " + snapshot); } } }
public async Task LoadSnapshotsAsync(Action <IEnumerable <string> > tablesCallback, Action <PartitionSnapshot> callbackPartitionSnapshot) { const string ignoreContainerName = "nosqlsnapshots"; var containers = await _storageAccount.GetListOfContainersAsync(); tablesCallback(containers.Select(itm => itm.Name).Where(name => name != ignoreContainerName)); foreach (var container in containers.Where(c => c.Name != ignoreContainerName)) { await foreach (var blockBlob in container.GetListOfBlobsAsync()) { var memoryStream = new MemoryStream(); await blockBlob.DownloadToStreamAsync(memoryStream); var snapshot = new PartitionSnapshot { TableName = container.Name, PartitionKey = blockBlob.Name.Base64ToString(), Snapshot = memoryStream.ToArray() }; callbackPartitionSnapshot(snapshot); Console.WriteLine("Loaded snapshot: " + snapshot); } } }
public void InitPartitionFromSnapshot(PartitionSnapshot partitionSnapshot) { _readerWriterLockSlim.EnterWriteLock(); try { var partition = DbPartition.Create(partitionSnapshot.PartitionKey); if (_partitions.ContainsKey(partitionSnapshot.PartitionKey)) { return; } _partitions.Add(partition.PartitionKey, partition); var partitionAsMyMemory = new MyMemoryAsByteArray(partitionSnapshot.Snapshot); foreach (var dbRowMemory in partitionAsMyMemory.SplitJsonArrayToObjects()) { var entity = dbRowMemory.ParseDynamicEntity(); var dbRow = DbRow.RestoreSnapshot(entity, dbRowMemory); partition.InsertOrReplace(dbRow); } } finally { _readerWriterLockSlim.ExitWriteLock(); } }
public The_snapshot_of_a_partition() { var timeBetweenLastEnqueueAndLastCheckpoint = TimeSpan.FromMinutes(1); _timeBetweenLastEnqueueAndPreviousEnqueue = TimeSpan.FromMinutes(10); _timeBetweenCheckpoints = TimeSpan.FromMinutes(1); _howFarBehindForCheckpoint = 1500; _howFarBehindForPreviousEnqueue = 6000; _lastEnqueuedOffset = "12345"; _lastEnqueuedTimeUtc = new DateTime(1975, 4, 1); _lastCheckpointTimeUtc = _lastEnqueuedTimeUtc.Subtract(timeBetweenLastEnqueueAndLastCheckpoint); _incomingByteRate = 99; _outgoingByteRate = 111; var mostRecentCheckpoint = new PartitionCheckpoint { SequenceNumber = _sequenceNumberOfMostRecentEvent - _howFarBehindForCheckpoint, LastCheckpointTimeUtc = _lastCheckpointTimeUtc }; var monitor = new EventHubMonitor( new[] { PartitionId }, partitionId => Task.FromResult(mostRecentCheckpoint), partitionId => Task.FromResult(CreatePartitionDescription(partitionId)), TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(1)); _snapshot = monitor .CaptureSnapshotAsync(PartitionId) .Result; }
/// <summary> /// Formats the event as a string. /// </summary> /// <param name="snapshot">The partition snapshot to format.</param> /// <param name="formatter">The formatter to use.</param> /// <returns>A formatted snapshot.</returns> public static string WriteEvent(this IEventTextFormatter formatter, PartitionSnapshot snapshot) { Guard.ArgumentNotNull(formatter, "formatter"); using (var writer = new StringWriter(CultureInfo.CurrentCulture)) { formatter.WriteEvent(snapshot, writer); return writer.ToString(); } }
public void WriteEvent(PartitionSnapshot snapshot, TextWriter writer) { Guard.ArgumentNotNull(snapshot, "PartitionSnapshot"); Guard.ArgumentNotNull(writer, "writer"); writer.WriteLine("{0},{1},{2}", snapshot.PartitionId, snapshot.CapturedAtTimeUtc, snapshot.UnprocessedEvents); }
/// <summary> /// Formats the event as a string. /// </summary> /// <param name="snapshot">The partition snapshot to format.</param> /// <param name="formatter">The formatter to use.</param> /// <returns>A formatted snapshot.</returns> public static string WriteEvent(this IEventTextFormatter formatter, PartitionSnapshot snapshot) { Guard.ArgumentNotNull(formatter, "formatter"); using (var writer = new StringWriter(CultureInfo.CurrentCulture)) { formatter.WriteEvent(snapshot, writer); return(writer.ToString()); } }
public void WriteEvent(PartitionSnapshot snapshot, TextWriter writer) { Guard.ArgumentNotNull(snapshot, "PartitionSnapshot"); Guard.ArgumentNotNull(writer, "writer"); writer.WriteLine("{0},{1},{2}", snapshot.PartitionId, snapshot.CapturedAtTimeUtc, snapshot.UnprocessedEvents); }
/// <summary> /// Formats an <see cref="PartitionSnapshot" /> as a string using an <see cref="IEventTextFormatter" />. /// </summary> /// <param name="snapshot">The snapshot to format.</param> /// <param name="formatter">The formatter to use.</param> /// <returns>A formatted snapshot, or <see langword="null" /> if an exception is thrown by the <paramref name="formatter" />.</returns> public static string TryFormatAsString(this PartitionSnapshot snapshot, IEventTextFormatter formatter) { try { return(formatter.WriteEvent(snapshot)); } catch (Exception e) { //SemanticLoggingEventSource.Log.FormatEntryAsStringFailed(e.ToString()); } return(null); }
public async Task SavePartitionSnapshotAsync(PartitionSnapshot partitionSnapshot) { var container = await _storageAccount.GetBlockBlobReferenceAsync(partitionSnapshot.TableName); if (container == null) { Console.WriteLine($"{DateTime.UtcNow:s} Skipped synchronizing snapshot: {partitionSnapshot}"); return; } await container.SavePartitionAsync(partitionSnapshot.PartitionKey, partitionSnapshot.Snapshot); Console.WriteLine($"{DateTime.UtcNow:s} Saved snapshot: {partitionSnapshot}"); }
public static ValueTask <IActionResult> SynchronizePartitionAsync(this IActionResult result, DbTable dbTable, DbPartition partitionToSave, DataSynchronizationPeriod period) { if (ServiceLocator.SnapshotStorage == null) { return(new ValueTask <IActionResult>(result)); } if (period == DataSynchronizationPeriod.Immediately) { var partitionSnapshot = PartitionSnapshot.Create(dbTable, partitionToSave); return(result.ResponseWithActionAsync(() => ServiceLocator .SnapshotStorage .SavePartitionSnapshotAsync(partitionSnapshot))); } ServiceLocator.SnapshotSaverScheduler.SynchronizePartition(dbTable, partitionToSave, period); return(new ValueTask <IActionResult>(result)); }
public async Task TheLoop() { await LoadSnapshotsAsync(); while (!_appIsShuttingDown || ServiceLocator.SnapshotSaverScheduler.TasksToSyncCount() > 0) { try { var elementToSave = ServiceLocator.SnapshotSaverScheduler.GetTaskToSync(_appIsShuttingDown); while (elementToSave != null) { switch (elementToSave) { case SyncTable syncTable: await ServiceLocator.SnapshotStorage.SaveTableSnapshotAsync(syncTable.DbTable); break; case SyncPartition syncPartition: var partitionSnapshot = PartitionSnapshot.Create(syncPartition.DbTable, syncPartition.DbPartition); await ServiceLocator.SnapshotStorage.SavePartitionSnapshotAsync(partitionSnapshot); break; case SyncDeletePartition syncDeletePartition: await ServiceLocator.SnapshotStorage.DeleteTablePartitionAsync(syncDeletePartition.TableName, syncDeletePartition.PartitionKey); break; } elementToSave = ServiceLocator.SnapshotSaverScheduler.GetTaskToSync(_appIsShuttingDown); } } catch (Exception e) { Console.WriteLine("There is something wrong during saving the snapshot. " + e.Message); } }
public async IAsyncEnumerable <PartitionSnapshot> GetPartitionsAsync() { await foreach (var blockBlob in _container.GetListOfBlobsAsync()) { if (blockBlob.Name == SystemFileNames.TableMetadataFileName) { continue; } var memoryStream = new MemoryStream(); await blockBlob.DownloadToStreamAsync(memoryStream); var snapshot = new PartitionSnapshot { PartitionKey = blockBlob.Name.Base64ToString(), Snapshot = memoryStream.ToArray() }; yield return(snapshot); Console.WriteLine("Loaded snapshot: " + snapshot); } }
public ValueTask SavePartitionSnapshotAsync(DbTable dbTable, PartitionSnapshot partitionSnapshot) { return(new ValueTask()); }