public virtual Task <IReadOnlyList <VersionedInstanceIdentifier> > GetInstanceIdentifiersByWatermarkRangeAsync(
WatermarkRange watermarkRange,
IndexStatus indexStatus,
CancellationToken cancellationToken = default)
{
throw new BadRequestException(DicomSqlServerResource.SchemaVersionNeedsToBeUpgraded);
}
public async Task Dispose()
{
_status = IndexStatus.Disposed;
//get all silos
Dictionary <SiloAddress, SiloStatus> hosts = await SiloUtils.GetHosts(true);
var numHosts = hosts.Keys.Count;
Task[] disposeToSilos = new Task[numHosts];
int i = 0;
IList <IOrleansQueryResultStream <V> > result = new List <IOrleansQueryResultStream <V> >();
GrainId grainID = GetGrainID(IndexUtils.GetIndexNameFromIndexGrain(this));
foreach (SiloAddress siloAddress in hosts.Keys)
{
//dispose the index on each silo
disposeToSilos[i] = InsideRuntimeClient.Current.InternalGrainFactory.GetSystemTarget <ActiveHashIndexPartitionedPerSiloBucket>(
grainID,
siloAddress
).Dispose();
++i;
}
await Task.WhenAll(disposeToSilos);
}
public static long LogIndexesChanges(List <EngineCustomStatus> A, List <EngineCustomStatus> B)
{
long totalChanges = 0;
if (A.Except(B).Count() != 0)
{
Sys.LogError($"Cannot compare different lists of Engine Status");
return(-1);
}
foreach (EngineCustomStatus A_ECS in A)
{
LogTable logTabeIndexesChanges = new LogTable($"Engine [{A_ECS.Name}]");
logTabeIndexesChanges.SetInnerColumnSpaces(1, 1);
foreach (IndexStatus A_ECS_IDX in A_ECS.indexesStatus)
{
IndexStatus B_ECS_IDX = B.Single(B_ECS => Str.EQ(B_ECS.Name, A_ECS.Name)).indexesStatus.Single(x => Str.EQ(x.Name, A_ECS_IDX.Name));
long delta = B_ECS_IDX.DocumentCount - A_ECS_IDX.DocumentCount;
logTabeIndexesChanges.AddItem(A_ECS_IDX.Name, "From", A_ECS_IDX.DocumentCount.ToString("+0.#;-0.#;0"));
logTabeIndexesChanges.AddItem(A_ECS_IDX.Name, "To", B_ECS_IDX.DocumentCount.ToString("+0.#;-0.#;0"));
logTabeIndexesChanges.AddItem(A_ECS_IDX.Name, "Change", delta.ToString("+0.#;-0.#;0"));
totalChanges += Math.Abs(delta);
}
logTabeIndexesChanges.SysLog();
}
return(totalChanges);
}
public override async Task <IReadOnlyList <WatermarkRange> > GetInstanceBatchesAsync(
int batchSize,
int batchCount,
IndexStatus indexStatus,
long?maxWatermark = null,
CancellationToken cancellationToken = default)
{
EnsureArg.IsGt(batchSize, 0, nameof(batchSize));
EnsureArg.IsGt(batchCount, 0, nameof(batchCount));
using SqlConnectionWrapper sqlConnectionWrapper = await SqlConnectionWrapperFactory.ObtainSqlConnectionWrapperAsync(cancellationToken);
using SqlCommandWrapper sqlCommandWrapper = sqlConnectionWrapper.CreateSqlCommand();
VLatest.GetInstanceBatches.PopulateCommand(sqlCommandWrapper, batchSize, batchCount, (byte)indexStatus, maxWatermark);
try
{
var batches = new List <WatermarkRange>();
using SqlDataReader reader = await sqlCommandWrapper.ExecuteReaderAsync(cancellationToken);
while (await reader.ReadAsync(cancellationToken))
{
batches.Add(new WatermarkRange(reader.GetInt64(0), reader.GetInt64(1)));
}
return(batches);
}
catch (SqlException ex)
{
throw new DataStoreException(ex);
}
}
public async Task <bool> ShouldBeRetriedAsync(ProjectionVersion version)
{
bool isVersionTrackerMissing = await IsVersionTrackerMissingAsync().ConfigureAwait(false);
if (isVersionTrackerMissing)
{
InitializeNewProjectionVersion();
if (version.ProjectionName.Equals(ProjectionVersionsHandler.ContractId, StringComparison.OrdinalIgnoreCase) == false)
{
return(true);
}
}
IndexStatus indexStatus = await GetIndexStatusAsync <EventToAggregateRootId>().ConfigureAwait(false);
Type projectionType = version.ProjectionName.GetTypeByContract();
if (isVersionTrackerMissing && IsNotSystemProjection(projectionType))
{
return(true);
}
if (indexStatus.IsNotPresent() && IsNotSystemProjection(projectionType))
{
return(true);
}
return(false);
}
public IndexStatus Get(string indexId)
{
BoundStatement bs = session.Prepare(Read).Bind(indexId);
var row = session.Execute(bs).GetRows().SingleOrDefault();
return(IndexStatus.Parse(row?.GetValue <string>("status")));
}
public GlobalSecondaryIndexDescription(string indexArn, string indexName, long indexSizeBytes, long itemCount,
IReadOnlyList <KeySchemaElement> keySchema, Projection projection, bool backfilling, IndexStatus indexStatus,
ProvisionedThroughputDescription provisionedThroughput) : base(indexArn, indexName, indexSizeBytes, itemCount, keySchema, projection)
{
Backfilling = backfilling;
IndexStatus = indexStatus;
ProvisionedThroughput = provisionedThroughput;
}
/// <summary>
/// Index operation finished
/// </summary>
/// <param name="sender">Object</param>
/// <param name="e">OperationFinishedArg</param>
public static void index_OperationFinished(object sender, OperationFinishedEventArgs e)
{
DateTime time = e.Time; // Time when documents added
Guid indexId = e.IndexId; // Index Id
string indexFolder = e.IndexFolder; // Index Folder
IndexStatus status = e.Status; // Index Status
OperationType operationType = e.OperationType; // Operation Type.
}
public virtual Task <IReadOnlyList <WatermarkRange> > GetInstanceBatchesAsync(
int batchSize,
int batchCount,
IndexStatus indexStatus,
long?maxWatermark = null,
CancellationToken cancellationToken = default)
{
throw new BadRequestException(DicomSqlServerResource.SchemaVersionNeedsToBeUpgraded);
}
public void Save(string indexId, IndexStatus status)
{
try
{
PreparedStatement statement = session.Prepare(Write);
session.Execute(statement.Bind(indexId, status.ToString()));
}
catch (WriteTimeoutException ex)
{
log.WarnException("[EventStore] Write timeout while persisting in CassandraIndexStatusStore", ex);
}
}
public async Task Dispose()
{
_status = IndexStatus.Disposed;
var indexName = IndexUtils.GetIndexNameFromIndexGrain(this);
GrainReference makeGrainReference(SiloAddress siloAddress) => GetGrainReference(this.SiloIndexManager, indexName, siloAddress);
// Get and Dispose() all buckets in silos
Dictionary <SiloAddress, SiloStatus> hosts = await this.SiloIndexManager.GetSiloHosts(true);
await Task.WhenAll(hosts.Keys.Select(sa => this.SiloIndexManager.GetGrainService <IActiveHashIndexPartitionedPerSiloBucket>(makeGrainReference(sa)).Dispose()));
}
public static string GetDescription(this IndexStatus value)
{
Type type = value.GetType();
string name = Enum.GetName(type, value);
if (string.IsNullOrEmpty(name))
{
return(string.Empty);
}
FieldInfo field = type.GetField(name);
DescriptionAttribute attribute = Attribute.GetCustomAttribute(field, typeof(DescriptionAttribute)) as DescriptionAttribute;
return(attribute == null ? string.Empty : attribute.Description);
}
public void GetStatus()
{
var indicesApi = new IndicesApi("http://localhost:9800");
var response = indicesApi.GetIndexStatus("country");
if (response.Error?.Message != null)
{
Console.WriteLine(response.Error.Message);
}
else
{
IndexStatus status = response.Data.IndexStatus;
}
}
public IndexInformation(string health, IndexStatus status, string indexName, string uuid,
int primaryShards, int replicatedShards, int documentsCount, int documentsDeleted,
string storeSize, string storeSizePrimary)
{
Health = health;
Status = status;
IndexName = indexName;
Uuid = uuid;
PrimaryShards = primaryShards;
ReplicatedShards = replicatedShards;
DocumentsCount = documentsCount;
DocumentsDeleted = documentsDeleted;
StoreSize = storeSize;
StoreSizePrimary = storeSizePrimary;
}
public async Task UpdateInstanceIndexStatusAsync(
VersionedInstanceIdentifier versionedInstanceIdentifier,
IndexStatus status,
CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(versionedInstanceIdentifier, nameof(versionedInstanceIdentifier));
EnsureArg.IsTrue(Enum.IsDefined(typeof(IndexStatus), status));
EnsureArg.IsTrue((int)status < byte.MaxValue);
await _sqlServerIndexSchema.EnsureInitialized();
using (SqlConnectionWrapper sqlConnectionWrapper = _sqlConnectionFactoryWrapper.ObtainSqlConnectionWrapper())
using (SqlCommandWrapper sqlCommandWrapper = sqlConnectionWrapper.CreateSqlCommand())
{
VLatest.UpdateInstanceStatus.PopulateCommand(
sqlCommandWrapper,
versionedInstanceIdentifier.StudyInstanceUid,
versionedInstanceIdentifier.SeriesInstanceUid,
versionedInstanceIdentifier.SopInstanceUid,
versionedInstanceIdentifier.Version,
(byte)status);
try
{
await sqlCommandWrapper.ExecuteScalarAsync(cancellationToken);
}
catch (SqlException ex)
{
switch (ex.Number)
{
case SqlErrorCodes.NotFound:
throw new InstanceNotFoundException();
default:
throw new DataStoreException(ex);
}
}
}
}
public override async Task <IReadOnlyList <VersionedInstanceIdentifier> > GetInstanceIdentifiersByWatermarkRangeAsync(
WatermarkRange watermarkRange,
IndexStatus indexStatus,
CancellationToken cancellationToken = default)
{
var results = new List <VersionedInstanceIdentifier>();
using (SqlConnectionWrapper sqlConnectionWrapper = await SqlConnectionWrapperFactory.ObtainSqlConnectionWrapperAsync(cancellationToken))
using (SqlCommandWrapper sqlCommandWrapper = sqlConnectionWrapper.CreateSqlCommand())
{
VLatest.GetInstancesByWatermarkRange.PopulateCommand(
sqlCommandWrapper,
watermarkRange.Start,
watermarkRange.End,
(byte)indexStatus);
using (var reader = await sqlCommandWrapper.ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken))
{
while (await reader.ReadAsync(cancellationToken))
{
(string rStudyInstanceUid, string rSeriesInstanceUid, string rSopInstanceUid, long watermark) = reader.ReadRow(
VLatest.Instance.StudyInstanceUid,
VLatest.Instance.SeriesInstanceUid,
VLatest.Instance.SopInstanceUid,
VLatest.Instance.Watermark);
results.Add(new VersionedInstanceIdentifier(
rStudyInstanceUid,
rSeriesInstanceUid,
rSopInstanceUid,
watermark));
}
}
}
return(results);
}
internal bool WaitForStatus(int timeout = 10, string status = "yellow")
{
var uri = $"{_settings.Host}/_cluster/health/{_name}?wait_for_status={status}&timeout={timeout}s";
Logger.Debug($"Waiting {timeout}s for status '{status}' on index '{_name}'");
try
{
string response = HttpClientHelper.GetString(new Uri(uri));
IndexStatus indexStatus = JsonConvert.DeserializeObject <IndexStatus>(response);
bool success = !indexStatus.TimedOut && status.Contains(indexStatus.Status);
Logger.Debug($"Success: {success}. Timeout: {indexStatus.TimedOut}. Status: {indexStatus.Status}");
return(success);
}
catch (Exception ex)
{
Logger.Error("Could not get status", ex);
return(false);
}
}
private async Task <TableDescription> TableIndexWaitOnStatusAsync(string tableName, string indexName, IndexStatus whileStatus, IndexStatus desiredStatus = null, int delay = 2000)
{
TableDescription ret;
GlobalSecondaryIndexDescription index = null;
do
{
if (index != null)
{
await Task.Delay(delay);
}
ret = await GetTableDescription(tableName);
index = ret.GlobalSecondaryIndexes.FirstOrDefault(index => index.IndexName == indexName);
} while (index.IndexStatus == whileStatus);
if (desiredStatus != null && index.IndexStatus != desiredStatus)
{
throw new InvalidOperationException($"Index {indexName} in table {tableName} has failed to reach the desired status of {desiredStatus}");
}
return(ret);
}
/// <summary>
/// CollectingBlobに配置されたファイル群をPrimaryBlobに適切に配置する。
/// </summary>
/// <remarks>sd 02-7.BLOBインデックス作成</remarks>
public void Index()
{
_logger.Enter();
try
{
IEnumerable <CollectedFile> collectedFiles;
try
{
// ファイル一覧を取得する
var directory = new ArchiveDirectory()
{
ContainerName = _settings.CollectingBlobContainerNameCollect
};
// Sq2.1: ファイルを取得する
collectedFiles =
_collectingRepository.GetArchiveFiles(directory)
.Select(x => new CollectedFile(x, _settings, _timeProvider));
}
catch (Exception ex)
{
_logger.Error(ex, nameof(Resources.CO_BLI_BLI_003), new object[] { _settings.CollectingBlobContainerNameCollect });
return;
}
foreach (CollectedFile collectFile in collectedFiles)
{
IndexStatus status = IndexStatus.Copy;
try
{
// メタデータが想定と異なる場合
// file作成時刻が既定年月日以上(一定時間以上処理されないケース)
if (!collectFile.IsValid())
{
// Sq2.2: 異常ファイルの集積コンテナにファイルを保存する
_collectingRepository.Copy(collectFile.SourceArchiveFile, collectFile.MakeUnknownFile());
// Sq2.3: ファイルを削除する（CollectingBlobのファイルを削除する。）
status = IndexStatus.Delete;
_collectingRepository.Delete(collectFile.SourceArchiveFile);
continue;
}
// メタデータからfileのコピー先を生成する。
var fileOnPrimaryBlob = collectFile.MakeClassifiedFile();
// Sq2.4: ファイルを保存する
status = IndexStatus.CopyToPrimary;
_collectingRepository.CopyToPrimary(collectFile.SourceArchiveFile, fileOnPrimaryBlob);
// Sq2.5: ファイル情報を保存する
if (!TryAddDtDeviceFile(collectFile, fileOnPrimaryBlob))
{
// Sq2.6: ファイルを削除する（PrimaryBlobのファイルを削除）
TryDeleteFromCoreMain(fileOnPrimaryBlob);
continue;
}
// Sq2.7: ファイルを削除する（ファイルの移動を完了したとしてCollectingBlobのファイルを削除する。）
status = IndexStatus.Delete;
_collectingRepository.Delete(collectFile.SourceArchiveFile);
_logger.Info(nameof(Resources.CO_BLI_BLI_009), new object[] { collectFile.SourceArchiveFile.ContainerName, collectFile.SourceArchiveFile.FilePath });
}
catch (Exception ex)
{
switch (status)
{
case IndexStatus.Delete:
_logger.Error(ex, nameof(Resources.CO_BLI_BLI_005), new object[] { collectFile.SourceArchiveFile.ContainerName, collectFile.SourceArchiveFile.FilePath });
break;
case IndexStatus.CopyToPrimary:
_logger.Error(ex, nameof(Resources.CO_BLI_BLI_006), new object[] { collectFile.SourceArchiveFile.ContainerName, collectFile.SourceArchiveFile.FilePath });
break;
case IndexStatus.Copy:
default:
_logger.Error(ex, nameof(Resources.CO_BLI_BLI_004), new object[] { collectFile.SourceArchiveFile.ContainerName, collectFile.SourceArchiveFile.FilePath });
break;
}
}
}
}
finally
{
_logger.Leave();
}
}
public ItemStatus(IndexStatus index, WorkingStatus working)
{
IndexStatus = index; WorkingStatus = working;
}
public static bool ValidToStart(IndexStatus status) => status == IndexStatus.Disposed || status == IndexStatus.Idle;
public override Task OnActivateAsync()
{
_status = IndexStatus.Available;
return(base.OnActivateAsync());
}
public ReplayResult Rebuild(ProjectionVersion version, DateTime rebuildUntil)
{
if (ReferenceEquals(null, version))
{
throw new ArgumentNullException(nameof(version));
}
Type projectionType = version.ProjectionName.GetTypeByContract();
try
{
IndexStatus indexStatus = GetIndexStatus <EventToAggregateRootId>();
if (indexStatus.IsNotPresent() && IsNotSystemProjection(projectionType))
{
return(ReplayResult.RetryLater($"The index is not present"));
}
if (IsVersionTrackerMissing() && IsNotSystemProjection(projectionType))
{
return(ReplayResult.RetryLater($"Projection `{version}` still don't have present index.")); //WHEN TO RETRY AGAIN
}
if (HasReplayTimeout(rebuildUntil))
{
return(ReplayResult.Timeout($"Rebuild of projection `{version}` has expired. Version:{version} Deadline:{rebuildUntil}."));
}
var allVersions = GetAllVersions(version);
if (allVersions.IsOutdatad(version))
{
return(new ReplayResult($"Version `{version}` is outdated. There is a newer one which is already live."));
}
if (allVersions.IsCanceled(version))
{
return(new ReplayResult($"Version `{version}` was canceled."));
}
DateTime startRebuildTimestamp = DateTime.UtcNow;
int progressCounter = 0;
logger.Info(() => $"Start rebuilding projection `{version.ProjectionName}` for version {version}. Deadline is {rebuildUntil}");
Dictionary <int, string> processedAggregates = new Dictionary <int, string>();
projectionStoreInitializer.Initialize(version);
var projectionHandledEventTypes = GetInvolvedEvents(projectionType);
foreach (var eventType in projectionHandledEventTypes)
{
logger.Info(() => $"Rebuilding projection `{version.ProjectionName}` for version {version} using eventType `{eventType}`. Deadline is {rebuildUntil}");
IEnumerable <IndexRecord> indexRecords = index.EnumerateRecords(eventType);
foreach (IndexRecord indexRecord in indexRecords)
{
// TODO: (5) Decorator pattern which will give us the tracking
progressCounter++;
if (progressCounter % 1000 == 0)
{
logger.Info(() => $"Rebuilding projection {version.ProjectionName} => PROGRESS:{progressCounter} Version:{version} EventType:{eventType} Deadline:{rebuildUntil} Total minutes working:{(DateTime.UtcNow - startRebuildTimestamp).TotalMinutes}. logId:{Guid.NewGuid().ToString()} ProcessedAggregatesSize:{processedAggregates.Count}");
}
int aggreagteRootIdHash = indexRecord.AggregateRootId.GetHashCode();
if (processedAggregates.ContainsKey(aggreagteRootIdHash))
{
continue;
}
processedAggregates.Add(aggreagteRootIdHash, null);
string mess = Encoding.UTF8.GetString(indexRecord.AggregateRootId);
IAggregateRootId arId = GetAggregateRootId(mess);
EventStream stream = eventStore.Load(arId);
foreach (AggregateCommit arCommit in stream.Commits)
{
for (int i = 0; i < arCommit.Events.Count; i++)
{
IEvent theEvent = arCommit.Events[i].Unwrap();
if (projectionHandledEventTypes.Contains(theEvent.GetType().GetContractId())) // filter out the events which are not part of the projection
{
var origin = new EventOrigin(mess, arCommit.Revision, i, arCommit.Timestamp);
projectionWriter.Save(projectionType, theEvent, origin, version);
}
}
}
}
}
logger.Info(() => $"Finish rebuilding projection `{projectionType.Name}` for version {version}. Deadline was {rebuildUntil}");
return(new ReplayResult());
}
catch (Exception ex)
{
string message = $"Unable to replay projection. Version:{version} ProjectionType:{projectionType.FullName}";
logger.ErrorException(ex, () => message);
return(new ReplayResult(message + Environment.NewLine + ex.Message + Environment.NewLine + ex.StackTrace));
}
}
/// <summary>
/// 设置索引状态
/// </summary>
/// <param name="status">索引状态</param>
public void SetIndexStatus(IndexStatus status)
{
switch (status)
{
case IndexStatus.未建:
Status = IndexStatus.未建;
break;
case IndexStatus.已建:
Status = IndexStatus.已建;
break;
case IndexStatus.需要更新:
Status = IndexStatus.需要更新;
break;
default:
throw new ArgumentOutOfRangeException("status");
}
}
string PrepPrintIndexCell(IndexStatus stats, ColumnDefinition col)
{
ConsoleColor color;
string symbol;
switch (stats) {
case IndexStatus.Added:
symbol = "+"; color = ConsoleColor.Green; break;
case IndexStatus.Modified:
symbol = "~"; color = ConsoleColor.Cyan; break;
case IndexStatus.Removed:
symbol = "-"; color = ConsoleColor.Yellow; break;
case IndexStatus.Unmerged:
symbol = "!"; color = ConsoleColor.White; break;
default:
symbol = string.Empty;
color = ConsoleColor.Gray;
break;
}
col.Foreground = color;
return symbol;
}
public IndexStatusInfo(IndexStatus indexStatus, IndexConfig indexConfig)
{
indexConfig.RequireNotNull(nameof(indexConfig));
IndexStatus = indexStatus;
IndexConfig = indexConfig;
}
/// <inheritdoc />
public async Task UpdateInstanceIndexStatusAsync(VersionedInstanceIdentifier versionedInstanceIdentifier, IndexStatus status, CancellationToken cancellationToken)
{
LogUpdateInstanceIndexStatusDelegate(_logger, versionedInstanceIdentifier, status, null);
try
{
await IndexDataStore.UpdateInstanceIndexStatusAsync(versionedInstanceIdentifier, status, cancellationToken);
LogOperationSucceededDelegate(_logger, null);
}
catch (Exception ex)
{
LogOperationFailedDelegate(_logger, ex);
throw;
}
}