Ejemplo n.º 1
0
        public async IAsyncEnumerable <PartitionSnapshot> LoadSnapshotsAsync()
        {
            const string ignoreContainerName = "nosqlsnapshots";


            await foreach (var container in _storageAccount.GetListOfContainersAsync())
            {
                if (container.Name == ignoreContainerName)
                {
                    continue;
                }

                await foreach (var blockBlob in container.GetListOfBlobsAsync())
                {
                    var memoryStream = new MemoryStream();

                    await blockBlob.DownloadToStreamAsync(memoryStream);

                    var snapshot = new PartitionSnapshot
                    {
                        TableName    = container.Name,
                        PartitionKey = blockBlob.Name.Base64ToString(),
                        Snapshot     = memoryStream.ToArray()
                    };

                    yield return(snapshot);

                    Console.WriteLine("Loaded snapshot: " + snapshot);
                }
            }
        }
        public async Task LoadSnapshotsAsync(Action <IEnumerable <string> > tablesCallback, Action <PartitionSnapshot> callbackPartitionSnapshot)
        {
            const string ignoreContainerName = "nosqlsnapshots";

            var containers = await _storageAccount.GetListOfContainersAsync();

            tablesCallback(containers.Select(itm => itm.Name).Where(name => name != ignoreContainerName));

            foreach (var container in containers.Where(c => c.Name != ignoreContainerName))
            {
                await foreach (var blockBlob in container.GetListOfBlobsAsync())
                {
                    var memoryStream = new MemoryStream();

                    await blockBlob.DownloadToStreamAsync(memoryStream);

                    var snapshot = new PartitionSnapshot
                    {
                        TableName    = container.Name,
                        PartitionKey = blockBlob.Name.Base64ToString(),
                        Snapshot     = memoryStream.ToArray()
                    };

                    callbackPartitionSnapshot(snapshot);

                    Console.WriteLine("Loaded snapshot: " + snapshot);
                }
            }
        }
Ejemplo n.º 3
0
        public void InitPartitionFromSnapshot(PartitionSnapshot partitionSnapshot)
        {
            _readerWriterLockSlim.EnterWriteLock();
            try
            {
                var partition = DbPartition.Create(partitionSnapshot.PartitionKey);

                if (_partitions.ContainsKey(partitionSnapshot.PartitionKey))
                {
                    return;
                }

                _partitions.Add(partition.PartitionKey, partition);


                var partitionAsMyMemory = new MyMemoryAsByteArray(partitionSnapshot.Snapshot);


                foreach (var dbRowMemory in partitionAsMyMemory.SplitJsonArrayToObjects())
                {
                    var entity = dbRowMemory.ParseDynamicEntity();
                    var dbRow  = DbRow.RestoreSnapshot(entity, dbRowMemory);
                    partition.InsertOrReplace(dbRow);
                }
            }
            finally
            {
                _readerWriterLockSlim.ExitWriteLock();
            }
        }
Ejemplo n.º 4
0
        public The_snapshot_of_a_partition()
        {
            var timeBetweenLastEnqueueAndLastCheckpoint = TimeSpan.FromMinutes(1);

            _timeBetweenLastEnqueueAndPreviousEnqueue = TimeSpan.FromMinutes(10);
            _timeBetweenCheckpoints = TimeSpan.FromMinutes(1);

            _howFarBehindForCheckpoint      = 1500;
            _howFarBehindForPreviousEnqueue = 6000;

            _lastEnqueuedOffset    = "12345";
            _lastEnqueuedTimeUtc   = new DateTime(1975, 4, 1);
            _lastCheckpointTimeUtc = _lastEnqueuedTimeUtc.Subtract(timeBetweenLastEnqueueAndLastCheckpoint);
            _incomingByteRate      = 99;
            _outgoingByteRate      = 111;

            var mostRecentCheckpoint = new PartitionCheckpoint
            {
                SequenceNumber        = _sequenceNumberOfMostRecentEvent - _howFarBehindForCheckpoint,
                LastCheckpointTimeUtc = _lastCheckpointTimeUtc
            };

            var monitor = new EventHubMonitor(
                new[] { PartitionId },
                partitionId => Task.FromResult(mostRecentCheckpoint),
                partitionId => Task.FromResult(CreatePartitionDescription(partitionId)),
                TimeSpan.FromSeconds(1),
                TimeSpan.FromSeconds(1));

            _snapshot = monitor
                        .CaptureSnapshotAsync(PartitionId)
                        .Result;
        }
        /// <summary>
        ///     Formats the event as a string.
        /// </summary>
        /// <param name="snapshot">The partition snapshot to format.</param>
        /// <param name="formatter">The formatter to use.</param>
        /// <returns>A formatted snapshot.</returns>
        public static string WriteEvent(this IEventTextFormatter formatter, PartitionSnapshot snapshot)
        {
            Guard.ArgumentNotNull(formatter, "formatter");

            using (var writer = new StringWriter(CultureInfo.CurrentCulture))
            {
                formatter.WriteEvent(snapshot, writer);
                return writer.ToString();
            }
        }
Ejemplo n.º 6
0
        public void WriteEvent(PartitionSnapshot snapshot, TextWriter writer)
        {
            Guard.ArgumentNotNull(snapshot, "PartitionSnapshot");
            Guard.ArgumentNotNull(writer, "writer");

            writer.WriteLine("{0},{1},{2}",
                             snapshot.PartitionId,
                             snapshot.CapturedAtTimeUtc,
                             snapshot.UnprocessedEvents);
        }
Ejemplo n.º 7
0
        /// <summary>
        ///     Formats the event as a string.
        /// </summary>
        /// <param name="snapshot">The partition snapshot to format.</param>
        /// <param name="formatter">The formatter to use.</param>
        /// <returns>A formatted snapshot.</returns>
        public static string WriteEvent(this IEventTextFormatter formatter, PartitionSnapshot snapshot)
        {
            Guard.ArgumentNotNull(formatter, "formatter");

            using (var writer = new StringWriter(CultureInfo.CurrentCulture))
            {
                formatter.WriteEvent(snapshot, writer);
                return(writer.ToString());
            }
        }
        public void WriteEvent(PartitionSnapshot snapshot, TextWriter writer)
        {
            Guard.ArgumentNotNull(snapshot, "PartitionSnapshot");
            Guard.ArgumentNotNull(writer, "writer");

            writer.WriteLine("{0},{1},{2}",
                snapshot.PartitionId,
                snapshot.CapturedAtTimeUtc,
                snapshot.UnprocessedEvents);
        }
        /// <summary>
        ///     Formats an <see cref="PartitionSnapshot" /> as a string using an <see cref="IEventTextFormatter" />.
        /// </summary>
        /// <param name="snapshot">The snapshot to format.</param>
        /// <param name="formatter">The formatter to use.</param>
        /// <returns>A formatted snapshot, or <see langword="null" /> if an exception is thrown by the <paramref name="formatter" />.</returns>
        public static string TryFormatAsString(this PartitionSnapshot snapshot, IEventTextFormatter formatter)
        {
            try
            {
                return(formatter.WriteEvent(snapshot));
            }
            catch (Exception e)
            {
                //SemanticLoggingEventSource.Log.FormatEntryAsStringFailed(e.ToString());
            }

            return(null);
        }
        public async Task SavePartitionSnapshotAsync(PartitionSnapshot partitionSnapshot)
        {
            var container = await _storageAccount.GetBlockBlobReferenceAsync(partitionSnapshot.TableName);

            if (container == null)
            {
                Console.WriteLine($"{DateTime.UtcNow:s} Skipped synchronizing snapshot: {partitionSnapshot}");
                return;
            }

            await container.SavePartitionAsync(partitionSnapshot.PartitionKey, partitionSnapshot.Snapshot);

            Console.WriteLine($"{DateTime.UtcNow:s} Saved snapshot: {partitionSnapshot}");
        }
        public static ValueTask <IActionResult> SynchronizePartitionAsync(this IActionResult result, DbTable dbTable, DbPartition partitionToSave,
                                                                          DataSynchronizationPeriod period)
        {
            if (ServiceLocator.SnapshotStorage == null)
            {
                return(new ValueTask <IActionResult>(result));
            }

            if (period == DataSynchronizationPeriod.Immediately)
            {
                var partitionSnapshot = PartitionSnapshot.Create(dbTable, partitionToSave);

                return(result.ResponseWithActionAsync(() => ServiceLocator
                                                      .SnapshotStorage
                                                      .SavePartitionSnapshotAsync(partitionSnapshot)));
            }

            ServiceLocator.SnapshotSaverScheduler.SynchronizePartition(dbTable, partitionToSave, period);
            return(new ValueTask <IActionResult>(result));
        }
Ejemplo n.º 12
0
        public async Task TheLoop()
        {
            await LoadSnapshotsAsync();

            while (!_appIsShuttingDown || ServiceLocator.SnapshotSaverScheduler.TasksToSyncCount() > 0)
            {
                try
                {
                    var elementToSave = ServiceLocator.SnapshotSaverScheduler.GetTaskToSync(_appIsShuttingDown);

                    while (elementToSave != null)
                    {
                        switch (elementToSave)
                        {
                        case SyncTable syncTable:
                            await ServiceLocator.SnapshotStorage.SaveTableSnapshotAsync(syncTable.DbTable);

                            break;

                        case SyncPartition syncPartition:

                            var partitionSnapshot = PartitionSnapshot.Create(syncPartition.DbTable, syncPartition.DbPartition);
                            await ServiceLocator.SnapshotStorage.SavePartitionSnapshotAsync(partitionSnapshot);

                            break;

                        case SyncDeletePartition syncDeletePartition:
                            await ServiceLocator.SnapshotStorage.DeleteTablePartitionAsync(syncDeletePartition.TableName,
                                                                                           syncDeletePartition.PartitionKey);

                            break;
                        }

                        elementToSave = ServiceLocator.SnapshotSaverScheduler.GetTaskToSync(_appIsShuttingDown);
                    }
                }
                catch (Exception e)
                {
                    Console.WriteLine("There is something wrong during saving the snapshot. " + e.Message);
                }
            }
        public async IAsyncEnumerable <PartitionSnapshot> GetPartitionsAsync()
        {
            await foreach (var blockBlob in _container.GetListOfBlobsAsync())
            {
                if (blockBlob.Name == SystemFileNames.TableMetadataFileName)
                {
                    continue;
                }

                var memoryStream = new MemoryStream();

                await blockBlob.DownloadToStreamAsync(memoryStream);

                var snapshot = new PartitionSnapshot
                {
                    PartitionKey = blockBlob.Name.Base64ToString(),
                    Snapshot     = memoryStream.ToArray()
                };

                yield return(snapshot);

                Console.WriteLine("Loaded snapshot: " + snapshot);
            }
        }
Ejemplo n.º 14
0
 public ValueTask SavePartitionSnapshotAsync(DbTable dbTable, PartitionSnapshot partitionSnapshot)
 {
     return(new ValueTask());
 }