public virtual Task <IReadOnlyList <VersionedInstanceIdentifier> > GetInstanceIdentifiersByWatermarkRangeAsync(
     WatermarkRange watermarkRange,
     IndexStatus indexStatus,
     CancellationToken cancellationToken = default)
 {
     throw new BadRequestException(DicomSqlServerResource.SchemaVersionNeedsToBeUpgraded);
 }
        public async Task Dispose()
        {
            _status = IndexStatus.Disposed;
            //get all silos
            Dictionary <SiloAddress, SiloStatus> hosts = await SiloUtils.GetHosts(true);

            var numHosts = hosts.Keys.Count;

            Task[] disposeToSilos = new Task[numHosts];

            int i = 0;
            IList <IOrleansQueryResultStream <V> > result = new List <IOrleansQueryResultStream <V> >();
            GrainId grainID = GetGrainID(IndexUtils.GetIndexNameFromIndexGrain(this));

            foreach (SiloAddress siloAddress in hosts.Keys)
            {
                //dispose the index on each silo
                disposeToSilos[i] = InsideRuntimeClient.Current.InternalGrainFactory.GetSystemTarget <ActiveHashIndexPartitionedPerSiloBucket>(
                    grainID,
                    siloAddress
                    ).Dispose();
                ++i;
            }
            await Task.WhenAll(disposeToSilos);
        }
示例#3
0
        public static long LogIndexesChanges(List <EngineCustomStatus> A, List <EngineCustomStatus> B)
        {
            long totalChanges = 0;

            if (A.Except(B).Count() != 0)
            {
                Sys.LogError($"Cannot compare different lists of Engine Status");
                return(-1);
            }

            foreach (EngineCustomStatus A_ECS in A)
            {
                LogTable logTabeIndexesChanges = new LogTable($"Engine [{A_ECS.Name}]");
                logTabeIndexesChanges.SetInnerColumnSpaces(1, 1);

                foreach (IndexStatus A_ECS_IDX in A_ECS.indexesStatus)
                {
                    IndexStatus B_ECS_IDX = B.Single(B_ECS => Str.EQ(B_ECS.Name, A_ECS.Name)).indexesStatus.Single(x => Str.EQ(x.Name, A_ECS_IDX.Name));

                    long delta = B_ECS_IDX.DocumentCount - A_ECS_IDX.DocumentCount;
                    logTabeIndexesChanges.AddItem(A_ECS_IDX.Name, "From", A_ECS_IDX.DocumentCount.ToString("+0.#;-0.#;0"));
                    logTabeIndexesChanges.AddItem(A_ECS_IDX.Name, "To", B_ECS_IDX.DocumentCount.ToString("+0.#;-0.#;0"));
                    logTabeIndexesChanges.AddItem(A_ECS_IDX.Name, "Change", delta.ToString("+0.#;-0.#;0"));
                    totalChanges += Math.Abs(delta);
                }

                logTabeIndexesChanges.SysLog();
            }

            return(totalChanges);
        }
        public override async Task <IReadOnlyList <WatermarkRange> > GetInstanceBatchesAsync(
            int batchSize,
            int batchCount,
            IndexStatus indexStatus,
            long?maxWatermark = null,
            CancellationToken cancellationToken = default)
        {
            EnsureArg.IsGt(batchSize, 0, nameof(batchSize));
            EnsureArg.IsGt(batchCount, 0, nameof(batchCount));

            using SqlConnectionWrapper sqlConnectionWrapper = await SqlConnectionWrapperFactory.ObtainSqlConnectionWrapperAsync(cancellationToken);

            using SqlCommandWrapper sqlCommandWrapper = sqlConnectionWrapper.CreateSqlCommand();

            VLatest.GetInstanceBatches.PopulateCommand(sqlCommandWrapper, batchSize, batchCount, (byte)indexStatus, maxWatermark);

            try
            {
                var batches = new List <WatermarkRange>();
                using SqlDataReader reader = await sqlCommandWrapper.ExecuteReaderAsync(cancellationToken);

                while (await reader.ReadAsync(cancellationToken))
                {
                    batches.Add(new WatermarkRange(reader.GetInt64(0), reader.GetInt64(1)));
                }

                return(batches);
            }
            catch (SqlException ex)
            {
                throw new DataStoreException(ex);
            }
        }
示例#5
0
        public async Task <bool> ShouldBeRetriedAsync(ProjectionVersion version)
        {
            bool isVersionTrackerMissing = await IsVersionTrackerMissingAsync().ConfigureAwait(false);

            if (isVersionTrackerMissing)
            {
                InitializeNewProjectionVersion();

                if (version.ProjectionName.Equals(ProjectionVersionsHandler.ContractId, StringComparison.OrdinalIgnoreCase) == false)
                {
                    return(true);
                }
            }

            IndexStatus indexStatus = await GetIndexStatusAsync <EventToAggregateRootId>().ConfigureAwait(false);

            Type projectionType = version.ProjectionName.GetTypeByContract();

            if (isVersionTrackerMissing && IsNotSystemProjection(projectionType))
            {
                return(true);
            }
            if (indexStatus.IsNotPresent() && IsNotSystemProjection(projectionType))
            {
                return(true);
            }

            return(false);
        }
示例#6
0
        public IndexStatus Get(string indexId)
        {
            BoundStatement bs  = session.Prepare(Read).Bind(indexId);
            var            row = session.Execute(bs).GetRows().SingleOrDefault();

            return(IndexStatus.Parse(row?.GetValue <string>("status")));
        }
示例#7
0
 public GlobalSecondaryIndexDescription(string indexArn, string indexName, long indexSizeBytes, long itemCount,
                                        IReadOnlyList <KeySchemaElement> keySchema, Projection projection, bool backfilling, IndexStatus indexStatus,
                                        ProvisionedThroughputDescription provisionedThroughput) : base(indexArn, indexName, indexSizeBytes, itemCount, keySchema, projection)
 {
     Backfilling           = backfilling;
     IndexStatus           = indexStatus;
     ProvisionedThroughput = provisionedThroughput;
 }
 /// <summary>
 /// Index operation finished
 /// </summary>
 /// <param name="sender">Object</param>
 /// <param name="e">OperationFinishedArg</param>
 public static void index_OperationFinished(object sender, OperationFinishedEventArgs e)
 {
     DateTime      time          = e.Time;          // Time when documents added
     Guid          indexId       = e.IndexId;       // Index Id
     string        indexFolder   = e.IndexFolder;   // Index Folder
     IndexStatus   status        = e.Status;        // Index Status
     OperationType operationType = e.OperationType; // Operation Type.
 }
 public virtual Task <IReadOnlyList <WatermarkRange> > GetInstanceBatchesAsync(
     int batchSize,
     int batchCount,
     IndexStatus indexStatus,
     long?maxWatermark = null,
     CancellationToken cancellationToken = default)
 {
     throw new BadRequestException(DicomSqlServerResource.SchemaVersionNeedsToBeUpgraded);
 }
示例#10
0
 public void Save(string indexId, IndexStatus status)
 {
     try
     {
         PreparedStatement statement = session.Prepare(Write);
         session.Execute(statement.Bind(indexId, status.ToString()));
     }
     catch (WriteTimeoutException ex)
     {
         log.WarnException("[EventStore] Write timeout while persisting in CassandraIndexStatusStore", ex);
     }
 }
        public async Task Dispose()
        {
            _status = IndexStatus.Disposed;
            var indexName = IndexUtils.GetIndexNameFromIndexGrain(this);

            GrainReference makeGrainReference(SiloAddress siloAddress) => GetGrainReference(this.SiloIndexManager, indexName, siloAddress);

            // Get and Dispose() all buckets in silos
            Dictionary <SiloAddress, SiloStatus> hosts = await this.SiloIndexManager.GetSiloHosts(true);

            await Task.WhenAll(hosts.Keys.Select(sa => this.SiloIndexManager.GetGrainService <IActiveHashIndexPartitionedPerSiloBucket>(makeGrainReference(sa)).Dispose()));
        }
示例#12
0
        public static string GetDescription(this IndexStatus value)
        {
            Type   type = value.GetType();
            string name = Enum.GetName(type, value);

            if (string.IsNullOrEmpty(name))
            {
                return(string.Empty);
            }
            FieldInfo            field     = type.GetField(name);
            DescriptionAttribute attribute = Attribute.GetCustomAttribute(field, typeof(DescriptionAttribute)) as DescriptionAttribute;

            return(attribute == null ? string.Empty : attribute.Description);
        }
示例#13
0
        public void GetStatus()
        {
            var indicesApi = new IndicesApi("http://localhost:9800");

            var response = indicesApi.GetIndexStatus("country");

            if (response.Error?.Message != null)
            {
                Console.WriteLine(response.Error.Message);
            }
            else
            {
                IndexStatus status = response.Data.IndexStatus;
            }
        }
 public IndexInformation(string health, IndexStatus status, string indexName, string uuid,
                         int primaryShards, int replicatedShards, int documentsCount, int documentsDeleted,
                         string storeSize, string storeSizePrimary)
 {
     Health           = health;
     Status           = status;
     IndexName        = indexName;
     Uuid             = uuid;
     PrimaryShards    = primaryShards;
     ReplicatedShards = replicatedShards;
     DocumentsCount   = documentsCount;
     DocumentsDeleted = documentsDeleted;
     StoreSize        = storeSize;
     StoreSizePrimary = storeSizePrimary;
 }
示例#15
0
        public async Task UpdateInstanceIndexStatusAsync(
            VersionedInstanceIdentifier versionedInstanceIdentifier,
            IndexStatus status,
            CancellationToken cancellationToken)
        {
            EnsureArg.IsNotNull(versionedInstanceIdentifier, nameof(versionedInstanceIdentifier));
            EnsureArg.IsTrue(Enum.IsDefined(typeof(IndexStatus), status));
            EnsureArg.IsTrue((int)status < byte.MaxValue);

            await _sqlServerIndexSchema.EnsureInitialized();

            using (SqlConnectionWrapper sqlConnectionWrapper = _sqlConnectionFactoryWrapper.ObtainSqlConnectionWrapper())
                using (SqlCommandWrapper sqlCommandWrapper = sqlConnectionWrapper.CreateSqlCommand())
                {
                    VLatest.UpdateInstanceStatus.PopulateCommand(
                        sqlCommandWrapper,
                        versionedInstanceIdentifier.StudyInstanceUid,
                        versionedInstanceIdentifier.SeriesInstanceUid,
                        versionedInstanceIdentifier.SopInstanceUid,
                        versionedInstanceIdentifier.Version,
                        (byte)status);

                    try
                    {
                        await sqlCommandWrapper.ExecuteScalarAsync(cancellationToken);
                    }
                    catch (SqlException ex)
                    {
                        switch (ex.Number)
                        {
                        case SqlErrorCodes.NotFound:
                            throw new InstanceNotFoundException();

                        default:
                            throw new DataStoreException(ex);
                        }
                    }
                }
        }
        public override async Task <IReadOnlyList <VersionedInstanceIdentifier> > GetInstanceIdentifiersByWatermarkRangeAsync(
            WatermarkRange watermarkRange,
            IndexStatus indexStatus,
            CancellationToken cancellationToken = default)
        {
            var results = new List <VersionedInstanceIdentifier>();

            using (SqlConnectionWrapper sqlConnectionWrapper = await SqlConnectionWrapperFactory.ObtainSqlConnectionWrapperAsync(cancellationToken))
                using (SqlCommandWrapper sqlCommandWrapper = sqlConnectionWrapper.CreateSqlCommand())
                {
                    VLatest.GetInstancesByWatermarkRange.PopulateCommand(
                        sqlCommandWrapper,
                        watermarkRange.Start,
                        watermarkRange.End,
                        (byte)indexStatus);

                    using (var reader = await sqlCommandWrapper.ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken))
                    {
                        while (await reader.ReadAsync(cancellationToken))
                        {
                            (string rStudyInstanceUid, string rSeriesInstanceUid, string rSopInstanceUid, long watermark) = reader.ReadRow(
                                VLatest.Instance.StudyInstanceUid,
                                VLatest.Instance.SeriesInstanceUid,
                                VLatest.Instance.SopInstanceUid,
                                VLatest.Instance.Watermark);

                            results.Add(new VersionedInstanceIdentifier(
                                            rStudyInstanceUid,
                                            rSeriesInstanceUid,
                                            rSopInstanceUid,
                                            watermark));
                        }
                    }
                }

            return(results);
        }
示例#17
0
        internal bool WaitForStatus(int timeout = 10, string status = "yellow")
        {
            var uri = $"{_settings.Host}/_cluster/health/{_name}?wait_for_status={status}&timeout={timeout}s";

            Logger.Debug($"Waiting {timeout}s for status '{status}' on index '{_name}'");

            try
            {
                string response = HttpClientHelper.GetString(new Uri(uri));

                IndexStatus indexStatus = JsonConvert.DeserializeObject <IndexStatus>(response);

                bool success = !indexStatus.TimedOut && status.Contains(indexStatus.Status);

                Logger.Debug($"Success: {success}. Timeout: {indexStatus.TimedOut}. Status: {indexStatus.Status}");

                return(success);
            }
            catch (Exception ex)
            {
                Logger.Error("Could not get status", ex);
                return(false);
            }
        }
示例#18
0
        private async Task <TableDescription> TableIndexWaitOnStatusAsync(string tableName, string indexName, IndexStatus whileStatus, IndexStatus desiredStatus = null, int delay = 2000)
        {
            TableDescription ret;
            GlobalSecondaryIndexDescription index = null;

            do
            {
                if (index != null)
                {
                    await Task.Delay(delay);
                }

                ret = await GetTableDescription(tableName);

                index = ret.GlobalSecondaryIndexes.FirstOrDefault(index => index.IndexName == indexName);
            } while (index.IndexStatus == whileStatus);

            if (desiredStatus != null && index.IndexStatus != desiredStatus)
            {
                throw new InvalidOperationException($"Index {indexName} in table {tableName} has failed to reach the desired status of {desiredStatus}");
            }

            return(ret);
        }
示例#19
0
        /// <summary>
        /// CollectingBlobに配置されたファイル群をPrimaryBlobに適切に配置する。
        /// </summary>
        /// <remarks>sd 02-7.BLOBインデックス作成</remarks>
        public void Index()
        {
            _logger.Enter();

            try
            {
                IEnumerable <CollectedFile> collectedFiles;
                try
                {
                    // ファイル一覧を取得する
                    var directory = new ArchiveDirectory()
                    {
                        ContainerName = _settings.CollectingBlobContainerNameCollect
                    };

                    // Sq2.1: ファイルを取得する
                    collectedFiles =
                        _collectingRepository.GetArchiveFiles(directory)
                        .Select(x => new CollectedFile(x, _settings, _timeProvider));
                }
                catch (Exception ex)
                {
                    _logger.Error(ex, nameof(Resources.CO_BLI_BLI_003), new object[] { _settings.CollectingBlobContainerNameCollect });
                    return;
                }

                foreach (CollectedFile collectFile in collectedFiles)
                {
                    IndexStatus status = IndexStatus.Copy;

                    try
                    {
                        // メタデータが想定と異なる場合
                        // file作成時刻が既定年月日以上(一定時間以上処理されないケース)
                        if (!collectFile.IsValid())
                        {
                            // Sq2.2: 異常ファイルの集積コンテナにファイルを保存する
                            _collectingRepository.Copy(collectFile.SourceArchiveFile, collectFile.MakeUnknownFile());

                            // Sq2.3: ファイルを削除する(CollectingBlobのファイルを削除する。)
                            status = IndexStatus.Delete;
                            _collectingRepository.Delete(collectFile.SourceArchiveFile);
                            continue;
                        }

                        // メタデータからfileのコピー先を生成する。
                        var fileOnPrimaryBlob = collectFile.MakeClassifiedFile();

                        // Sq2.4: ファイルを保存する
                        status = IndexStatus.CopyToPrimary;
                        _collectingRepository.CopyToPrimary(collectFile.SourceArchiveFile, fileOnPrimaryBlob);

                        // Sq2.5: ファイル情報を保存する
                        if (!TryAddDtDeviceFile(collectFile, fileOnPrimaryBlob))
                        {
                            // Sq2.6: ファイルを削除する(PrimaryBlobのファイルを削除)
                            TryDeleteFromCoreMain(fileOnPrimaryBlob);
                            continue;
                        }

                        // Sq2.7: ファイルを削除する(ファイルの移動を完了したとしてCollectingBlobのファイルを削除する。)
                        status = IndexStatus.Delete;
                        _collectingRepository.Delete(collectFile.SourceArchiveFile);

                        _logger.Info(nameof(Resources.CO_BLI_BLI_009), new object[] { collectFile.SourceArchiveFile.ContainerName, collectFile.SourceArchiveFile.FilePath });
                    }
                    catch (Exception ex)
                    {
                        switch (status)
                        {
                        case IndexStatus.Delete:
                            _logger.Error(ex, nameof(Resources.CO_BLI_BLI_005), new object[] { collectFile.SourceArchiveFile.ContainerName, collectFile.SourceArchiveFile.FilePath });
                            break;

                        case IndexStatus.CopyToPrimary:
                            _logger.Error(ex, nameof(Resources.CO_BLI_BLI_006), new object[] { collectFile.SourceArchiveFile.ContainerName, collectFile.SourceArchiveFile.FilePath });
                            break;

                        case IndexStatus.Copy:
                        default:
                            _logger.Error(ex, nameof(Resources.CO_BLI_BLI_004), new object[] { collectFile.SourceArchiveFile.ContainerName, collectFile.SourceArchiveFile.FilePath });
                            break;
                        }
                    }
                }
            }
            finally
            {
                _logger.Leave();
            }
        }
示例#20
0
 public ItemStatus(IndexStatus index, WorkingStatus working)
 {
     IndexStatus = index; WorkingStatus = working;
 }
示例#21
0
 public static bool ValidToStart(IndexStatus status) => status == IndexStatus.Disposed || status == IndexStatus.Idle;
 public override Task OnActivateAsync()
 {
     _status = IndexStatus.Available;
     return(base.OnActivateAsync());
 }
示例#23
0
        public ReplayResult Rebuild(ProjectionVersion version, DateTime rebuildUntil)
        {
            if (ReferenceEquals(null, version))
            {
                throw new ArgumentNullException(nameof(version));
            }

            Type projectionType = version.ProjectionName.GetTypeByContract();

            try
            {
                IndexStatus indexStatus = GetIndexStatus <EventToAggregateRootId>();
                if (indexStatus.IsNotPresent() && IsNotSystemProjection(projectionType))
                {
                    return(ReplayResult.RetryLater($"The index is not present"));
                }

                if (IsVersionTrackerMissing() && IsNotSystemProjection(projectionType))
                {
                    return(ReplayResult.RetryLater($"Projection `{version}` still don't have present index."));                                                                    //WHEN TO RETRY AGAIN
                }
                if (HasReplayTimeout(rebuildUntil))
                {
                    return(ReplayResult.Timeout($"Rebuild of projection `{version}` has expired. Version:{version} Deadline:{rebuildUntil}."));
                }

                var allVersions = GetAllVersions(version);
                if (allVersions.IsOutdatad(version))
                {
                    return(new ReplayResult($"Version `{version}` is outdated. There is a newer one which is already live."));
                }
                if (allVersions.IsCanceled(version))
                {
                    return(new ReplayResult($"Version `{version}` was canceled."));
                }

                DateTime startRebuildTimestamp = DateTime.UtcNow;
                int      progressCounter       = 0;
                logger.Info(() => $"Start rebuilding projection `{version.ProjectionName}` for version {version}. Deadline is {rebuildUntil}");
                Dictionary <int, string> processedAggregates = new Dictionary <int, string>();

                projectionStoreInitializer.Initialize(version);

                var projectionHandledEventTypes = GetInvolvedEvents(projectionType);
                foreach (var eventType in projectionHandledEventTypes)
                {
                    logger.Info(() => $"Rebuilding projection `{version.ProjectionName}` for version {version} using eventType `{eventType}`. Deadline is {rebuildUntil}");

                    IEnumerable <IndexRecord> indexRecords = index.EnumerateRecords(eventType);
                    foreach (IndexRecord indexRecord in indexRecords)
                    {
                        // TODO: (5) Decorator pattern which will give us the tracking
                        progressCounter++;
                        if (progressCounter % 1000 == 0)
                        {
                            logger.Info(() => $"Rebuilding projection {version.ProjectionName} => PROGRESS:{progressCounter} Version:{version} EventType:{eventType} Deadline:{rebuildUntil} Total minutes working:{(DateTime.UtcNow - startRebuildTimestamp).TotalMinutes}. logId:{Guid.NewGuid().ToString()} ProcessedAggregatesSize:{processedAggregates.Count}");
                        }

                        int aggreagteRootIdHash = indexRecord.AggregateRootId.GetHashCode();
                        if (processedAggregates.ContainsKey(aggreagteRootIdHash))
                        {
                            continue;
                        }
                        processedAggregates.Add(aggreagteRootIdHash, null);

                        string           mess   = Encoding.UTF8.GetString(indexRecord.AggregateRootId);
                        IAggregateRootId arId   = GetAggregateRootId(mess);
                        EventStream      stream = eventStore.Load(arId);

                        foreach (AggregateCommit arCommit in stream.Commits)
                        {
                            for (int i = 0; i < arCommit.Events.Count; i++)
                            {
                                IEvent theEvent = arCommit.Events[i].Unwrap();

                                if (projectionHandledEventTypes.Contains(theEvent.GetType().GetContractId())) // filter out the events which are not part of the projection
                                {
                                    var origin = new EventOrigin(mess, arCommit.Revision, i, arCommit.Timestamp);
                                    projectionWriter.Save(projectionType, theEvent, origin, version);
                                }
                            }
                        }
                    }
                }

                logger.Info(() => $"Finish rebuilding projection `{projectionType.Name}` for version {version}. Deadline was {rebuildUntil}");
                return(new ReplayResult());
            }
            catch (Exception ex)
            {
                string message = $"Unable to replay projection. Version:{version} ProjectionType:{projectionType.FullName}";
                logger.ErrorException(ex, () => message);
                return(new ReplayResult(message + Environment.NewLine + ex.Message + Environment.NewLine + ex.StackTrace));
            }
        }
示例#24
0
文件: Document.cs 项目: unicloud/FRP
 /// <summary>
 ///     设置索引状态
 /// </summary>
 /// <param name="status">索引状态</param>
 public void SetIndexStatus(IndexStatus status)
 {
     switch (status)
     {
         case IndexStatus.未建:
             Status = IndexStatus.未建;
             break;
         case IndexStatus.已建:
             Status = IndexStatus.已建;
             break;
         case IndexStatus.需要更新:
             Status = IndexStatus.需要更新;
             break;
         default:
             throw new ArgumentOutOfRangeException("status");
     }
 }
示例#25
0
        string PrepPrintIndexCell(IndexStatus stats, ColumnDefinition col)
        {
            ConsoleColor color;
            string symbol;

            switch (stats) {
                case IndexStatus.Added:
                    symbol = "+"; color = ConsoleColor.Green; break;

                case IndexStatus.Modified:
                    symbol = "~"; color = ConsoleColor.Cyan; break;

                case IndexStatus.Removed:
                    symbol = "-"; color = ConsoleColor.Yellow; break;

                case IndexStatus.Unmerged:
                    symbol = "!"; color = ConsoleColor.White; break;

                default:
                    symbol = string.Empty;
                    color = ConsoleColor.Gray;
                    break;
            }

            col.Foreground = color;
            return symbol;
        }
示例#26
0
 public IndexStatusInfo(IndexStatus indexStatus, IndexConfig indexConfig)
 {
     indexConfig.RequireNotNull(nameof(indexConfig));
     IndexStatus = indexStatus;
     IndexConfig = indexConfig;
 }
        /// <inheritdoc />
        public async Task UpdateInstanceIndexStatusAsync(VersionedInstanceIdentifier versionedInstanceIdentifier, IndexStatus status, CancellationToken cancellationToken)
        {
            LogUpdateInstanceIndexStatusDelegate(_logger, versionedInstanceIdentifier, status, null);

            try
            {
                await IndexDataStore.UpdateInstanceIndexStatusAsync(versionedInstanceIdentifier, status, cancellationToken);

                LogOperationSucceededDelegate(_logger, null);
            }
            catch (Exception ex)
            {
                LogOperationFailedDelegate(_logger, ex);

                throw;
            }
        }