/// <summary>
/// Returns the size of a row in the specified table.
/// </summary>
/// <exception cref="ArgumentNullException"><paramref name="reader"/> is null.</exception>
/// <exception cref="ArgumentOutOfRangeException"><paramref name="tableIndex"/> is not a valid table index.</exception>
public static int GetTableRowSize(this MetadataReader reader, TableIndex tableIndex)
{
if (reader == null)
{
throw new ArgumentNullException("reader");
}
switch (tableIndex)
{
case TableIndex.Module: return reader.ModuleTable.RowSize;
case TableIndex.TypeRef: return reader.TypeRefTable.RowSize;
case TableIndex.TypeDef: return reader.TypeDefTable.RowSize;
case TableIndex.FieldPtr: return reader.FieldPtrTable.RowSize;
case TableIndex.Field: return reader.FieldTable.RowSize;
case TableIndex.MethodPtr: return reader.MethodPtrTable.RowSize;
case TableIndex.MethodDef: return reader.MethodDefTable.RowSize;
case TableIndex.ParamPtr: return reader.ParamPtrTable.RowSize;
case TableIndex.Param: return reader.ParamTable.RowSize;
case TableIndex.InterfaceImpl: return reader.InterfaceImplTable.RowSize;
case TableIndex.MemberRef: return reader.MemberRefTable.RowSize;
case TableIndex.Constant: return reader.ConstantTable.RowSize;
case TableIndex.CustomAttribute: return reader.CustomAttributeTable.RowSize;
case TableIndex.FieldMarshal: return reader.FieldMarshalTable.RowSize;
case TableIndex.DeclSecurity: return reader.DeclSecurityTable.RowSize;
case TableIndex.ClassLayout: return reader.ClassLayoutTable.RowSize;
case TableIndex.FieldLayout: return reader.FieldLayoutTable.RowSize;
case TableIndex.StandAloneSig: return reader.StandAloneSigTable.RowSize;
case TableIndex.EventMap: return reader.EventMapTable.RowSize;
case TableIndex.EventPtr: return reader.EventPtrTable.RowSize;
case TableIndex.Event: return reader.EventTable.RowSize;
case TableIndex.PropertyMap: return reader.PropertyMapTable.RowSize;
case TableIndex.PropertyPtr: return reader.PropertyPtrTable.RowSize;
case TableIndex.Property: return reader.PropertyTable.RowSize;
case TableIndex.MethodSemantics: return reader.MethodSemanticsTable.RowSize;
case TableIndex.MethodImpl: return reader.MethodImplTable.RowSize;
case TableIndex.ModuleRef: return reader.ModuleRefTable.RowSize;
case TableIndex.TypeSpec: return reader.TypeSpecTable.RowSize;
case TableIndex.ImplMap: return reader.ImplMapTable.RowSize;
case TableIndex.FieldRva: return reader.FieldRvaTable.RowSize;
case TableIndex.EncLog: return reader.EncLogTable.RowSize;
case TableIndex.EncMap: return reader.EncMapTable.RowSize;
case TableIndex.Assembly: return reader.AssemblyTable.RowSize;
case TableIndex.AssemblyProcessor: return reader.AssemblyProcessorTable.RowSize;
case TableIndex.AssemblyOS: return reader.AssemblyOSTable.RowSize;
case TableIndex.AssemblyRef: return reader.AssemblyRefTable.RowSize;
case TableIndex.AssemblyRefProcessor: return reader.AssemblyRefProcessorTable.RowSize;
case TableIndex.AssemblyRefOS: return reader.AssemblyRefOSTable.RowSize;
case TableIndex.File: return reader.FileTable.RowSize;
case TableIndex.ExportedType: return reader.ExportedTypeTable.RowSize;
case TableIndex.ManifestResource: return reader.ManifestResourceTable.RowSize;
case TableIndex.NestedClass: return reader.NestedClassTable.RowSize;
case TableIndex.GenericParam: return reader.GenericParamTable.RowSize;
case TableIndex.MethodSpec: return reader.MethodSpecTable.RowSize;
case TableIndex.GenericParamConstraint: return reader.GenericParamConstraintTable.RowSize;
default:
throw new ArgumentOutOfRangeException("tableIndex");
}
}
/// <summary>
/// Returns the number of rows in the specified table.
/// </summary>
/// <exception cref="ArgumentNullException"><paramref name="reader"/> is null.</exception>
/// <exception cref="ArgumentOutOfRangeException"><paramref name="tableIndex"/> is not a valid table index.</exception>
public static int GetTableRowCount(this MetadataReader reader, TableIndex tableIndex)
{
if (reader == null)
{
throw new ArgumentNullException(nameof(reader));
}
if ((int)tableIndex >= TableIndexExtensions.Count)
{
Throw.TableIndexOutOfRange();
}
return reader.TableRowCounts[(int)tableIndex];
}
/// <summary>
/// Returns the number of rows in the specified table.
/// </summary>
/// <exception cref="ArgumentNullException"><paramref name="reader"/> is null.</exception>
/// <exception cref="ArgumentOutOfRangeException"><paramref name="tableIndex"/> is not a valid table index.</exception>
public static int GetTableRowCount(this MetadataReader reader, TableIndex tableIndex)
{
if (reader == null)
{
throw new ArgumentNullException("reader");
}
if ((int)tableIndex >= TableIndexExtensions.Count)
{
throw new ArgumentOutOfRangeException("tableIndex");
}
return (int)reader.TableRowCounts[(int)tableIndex];
}
/// <summary>
/// Returns the number of rows in the specified table.
/// </summary>
/// <exception cref="ArgumentNullException"><paramref name="reader"/> is null.</exception>
/// <exception cref="ArgumentOutOfRangeException"><paramref name="tableIndex"/> is not a valid table index.</exception>
public static int GetTableRowCount(this MetadataReader reader, TableIndex tableIndex)
{
if (reader == null)
{
Throw.ArgumentNull(nameof(reader));
}
if ((int)tableIndex >= MetadataTokens.TableCount)
{
Throw.TableIndexOutOfRange();
}
return reader.TableRowCounts[(int)tableIndex];
}
public static void lua_getfield(this LuaState luaState, TableIndex index, string k) => Lua.lua_getfield(luaState, index, k);
public bool IsPresent(TableIndex table) => (PresentTablesMask & (1UL << (int)table)) != 0;
private static int[] CombineRowCounts(int[] local, int[] external, TableIndex firstLocalTableIndex)
{
Debug.Assert(local.Length == external.Length);
var rowCounts = new int[local.Length];
for (int i = 0; i < (int)firstLocalTableIndex; i++)
{
rowCounts[i] = external[i];
}
for (int i = (int)firstLocalTableIndex; i < rowCounts.Length; i++)
{
rowCounts[i] = local[i];
}
return rowCounts;
}
/// <summary>
/// Gets the <see cref="TableIndex"/> of the table corresponding to the specified <see cref="HandleKind"/>.
/// </summary>
/// <param name="type">Handle type.</param>
/// <param name="index">Table index.</param>
/// <returns>True if the handle type corresponds to an Ecma335 or Portable PDB table, false otherwise.</returns>
public static bool TryGetTableIndex(HandleKind type, out TableIndex index)
{
// We don't have a HandleKind for PropertyMap, EventMap, MethodSemantics, *Ptr, AssemblyOS, etc. tables,
// but one can get ahold of one by creating a handle from a token and getting its Kind.
if ((int)type < TableCount && ((1UL << (int)type) & (ulong)TableMask.AllTables) != 0)
{
index = (TableIndex)type;
return true;
}
index = 0;
return false;
}
public static void lua_getfield(TableIndex index, string k) => lua_getfield(_state, index, k);
private int GetTableSize(TableIndex index, int rowSize)
{
return((PresentTablesMask & (1UL << (int)index)) != 0 ? RowCounts[(int)index] * rowSize : 0);
}
public abstract Script AddIndex(TableIndex index);
internal LockEntry FindFirstTabLockEntry(TableIndex index)
{
return(TabLocks.TryGetValue(index, out var list) ? list.First.Value : null);
}
protected static List <TMember> GetMembers <TMember>(ModuleDefinition module, TableIndex tableIndex)
{
int count = module.DotNetDirectory.Metadata
.GetStream <TablesStream>()
.GetTable(tableIndex)
.Count;
var result = new List <TMember>();
for (uint rid = 1; rid <= count; rid++)
{
result.Add((TMember)module.LookupMember(new MetadataToken(tableIndex, rid)));
}
return(result);
}
/// <summary>
/// Creates a handle from a token value.
/// </summary>
/// <exception cref="ArgumentException">
/// <paramref name="tableIndex"/> is not a valid table index.</exception>
public static Handle Handle(TableIndex tableIndex, int rowNumber)
{
int token = ((int)tableIndex << TokenTypeIds.RowIdBitCount) | rowNumber;
if (!TokenTypeIds.IsEcmaToken(unchecked((uint)token)))
{
ThrowInvalidTableIndex();
}
return new Handle((uint)token);
}
internal static void InvalidOperation_TableNotSorted(TableIndex tableIndex)
{
throw new InvalidOperationException(SR.Format(SR.MetadataTableNotSorted, tableIndex));
}
internal static Handle Handle(int rowNumber, TableIndex table)
{
return(MetadataTokens.Handle(table, rowNumber));
}
public override Script DropIndex(TableIndex index)
{
return(new DropDbObjectScript <TableIndex>($"ALTER TABLE {this.GetQuotedString(index.TableName)} DROP INDEX {this.GetQuotedString(index.Name)}"));
}
public abstract Script DropIndex(TableIndex index);
private bool ReferenceFits(int bitCount, TableIndex[] tables)
{
int maxIndex = (1 << bitCount) - 1;
foreach (TableIndex table in tables)
{
// table can be either local or external, but not both:
Debug.Assert(RowCounts[(int)table] == 0 || ExternalRowCounts[(int)table] == 0);
if (RowCounts[(int)table] + ExternalRowCounts[(int)table] > maxIndex)
{
return false;
}
}
return true;
}
internal static void TableNotSorted(TableIndex tableIndex)
{
throw new BadImageFormatException(SR.Format(SR.MetadataTableNotSorted, tableIndex));
}
internal static Handle Handle(int rowNumber, TableIndex table)
{
return MetadataTokens.Handle(table, rowNumber);
}
internal static void InvalidOperation_TableNotSorted(TableIndex tableIndex)
{
throw new InvalidOperationException(SR.Format(SR.MetadataTableNotSorted, tableIndex));
}
private static MemoryBlock GetTableMetadataBlock(this MetadataReader reader, TableIndex tableIndex)
{
Debug.Assert(reader != null);
switch (tableIndex)
{
case TableIndex.Module: return reader.ModuleTable.Block;
case TableIndex.TypeRef: return reader.TypeRefTable.Block;
case TableIndex.TypeDef: return reader.TypeDefTable.Block;
case TableIndex.FieldPtr: return reader.FieldPtrTable.Block;
case TableIndex.Field: return reader.FieldTable.Block;
case TableIndex.MethodPtr: return reader.MethodPtrTable.Block;
case TableIndex.MethodDef: return reader.MethodDefTable.Block;
case TableIndex.ParamPtr: return reader.ParamPtrTable.Block;
case TableIndex.Param: return reader.ParamTable.Block;
case TableIndex.InterfaceImpl: return reader.InterfaceImplTable.Block;
case TableIndex.MemberRef: return reader.MemberRefTable.Block;
case TableIndex.Constant: return reader.ConstantTable.Block;
case TableIndex.CustomAttribute: return reader.CustomAttributeTable.Block;
case TableIndex.FieldMarshal: return reader.FieldMarshalTable.Block;
case TableIndex.DeclSecurity: return reader.DeclSecurityTable.Block;
case TableIndex.ClassLayout: return reader.ClassLayoutTable.Block;
case TableIndex.FieldLayout: return reader.FieldLayoutTable.Block;
case TableIndex.StandAloneSig: return reader.StandAloneSigTable.Block;
case TableIndex.EventMap: return reader.EventMapTable.Block;
case TableIndex.EventPtr: return reader.EventPtrTable.Block;
case TableIndex.Event: return reader.EventTable.Block;
case TableIndex.PropertyMap: return reader.PropertyMapTable.Block;
case TableIndex.PropertyPtr: return reader.PropertyPtrTable.Block;
case TableIndex.Property: return reader.PropertyTable.Block;
case TableIndex.MethodSemantics: return reader.MethodSemanticsTable.Block;
case TableIndex.MethodImpl: return reader.MethodImplTable.Block;
case TableIndex.ModuleRef: return reader.ModuleRefTable.Block;
case TableIndex.TypeSpec: return reader.TypeSpecTable.Block;
case TableIndex.ImplMap: return reader.ImplMapTable.Block;
case TableIndex.FieldRva: return reader.FieldRvaTable.Block;
case TableIndex.EncLog: return reader.EncLogTable.Block;
case TableIndex.EncMap: return reader.EncMapTable.Block;
case TableIndex.Assembly: return reader.AssemblyTable.Block;
case TableIndex.AssemblyProcessor: return reader.AssemblyProcessorTable.Block;
case TableIndex.AssemblyOS: return reader.AssemblyOSTable.Block;
case TableIndex.AssemblyRef: return reader.AssemblyRefTable.Block;
case TableIndex.AssemblyRefProcessor: return reader.AssemblyRefProcessorTable.Block;
case TableIndex.AssemblyRefOS: return reader.AssemblyRefOSTable.Block;
case TableIndex.File: return reader.FileTable.Block;
case TableIndex.ExportedType: return reader.ExportedTypeTable.Block;
case TableIndex.ManifestResource: return reader.ManifestResourceTable.Block;
case TableIndex.NestedClass: return reader.NestedClassTable.Block;
case TableIndex.GenericParam: return reader.GenericParamTable.Block;
case TableIndex.MethodSpec: return reader.MethodSpecTable.Block;
case TableIndex.GenericParamConstraint: return reader.GenericParamConstraintTable.Block;
// debug tables
case TableIndex.Document: return reader.DocumentTable.Block;
case TableIndex.MethodDebugInformation: return reader.MethodDebugInformationTable.Block;
case TableIndex.LocalScope: return reader.LocalScopeTable.Block;
case TableIndex.LocalVariable: return reader.LocalVariableTable.Block;
case TableIndex.LocalConstant: return reader.LocalConstantTable.Block;
case TableIndex.ImportScope: return reader.ImportScopeTable.Block;
case TableIndex.StateMachineMethod: return reader.StateMachineMethodTable.Block;
case TableIndex.CustomDebugInformation: return reader.CustomDebugInformationTable.Block;
default:
throw new ArgumentOutOfRangeException(nameof(tableIndex));
}
}
public override async Task TestFixtureSetUp()
{
await base.TestFixtureSetUp();
var writerCheckpoint = new InMemoryCheckpoint(0);
var chaserCheckpoint = new InMemoryCheckpoint(0);
var bus = new InMemoryBus("bus");
new IODispatcher(bus, new PublishEnvelope(bus));
_db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, writerCheckpoint, chaserCheckpoint));
_db.Open();
// create db
Writer = new TFChunkWriter(_db);
Writer.Open();
SetupDB();
Writer.Close();
Writer = null;
writerCheckpoint.Flush();
chaserCheckpoint.Write(writerCheckpoint.Read());
chaserCheckpoint.Flush();
var readers = new ObjectPool <ITransactionFileReader>("Readers", 2, 5,
() => new TFChunkReader(_db, _db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
() => new HashListMemTable(IndexBitnessVersion, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
IndexBitnessVersion,
int.MaxValue,
Constants.PTableMaxReaderCountDefault,
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
_tableIndex,
EventStore.Core.Settings.ESConsts.StreamInfoCacheCapacity,
additionalCommitChecks: PerformAdditionalCommitChecks,
metastreamMaxCount: MetastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault,
replicationCheckpoint: _db.Config.ReplicationCheckpoint);
((ReadIndex)ReadIndex).IndexCommitter.Init(chaserCheckpoint.Read());
_tableIndex.Close(false);
Writer = new TFChunkWriter(_db);
Writer.Open();
Given();
Writer.Close();
Writer = null;
writerCheckpoint.Flush();
chaserCheckpoint.Write(writerCheckpoint.Read());
chaserCheckpoint.Flush();
_tableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
() => new HashListMemTable(IndexBitnessVersion, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
IndexBitnessVersion,
int.MaxValue,
Constants.PTableMaxReaderCountDefault,
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
_tableIndex,
EventStore.Core.Settings.ESConsts.StreamInfoCacheCapacity,
additionalCommitChecks: PerformAdditionalCommitChecks,
metastreamMaxCount: MetastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault,
replicationCheckpoint: _db.Config.ReplicationCheckpoint);
((ReadIndex)ReadIndex).IndexCommitter.Init(chaserCheckpoint.Read());
}
/// <summary>
/// Creates an <see cref="Metadata.EntityHandle"/> from a token value.
/// </summary>
/// <exception cref="ArgumentException">
/// <paramref name="tableIndex"/> is not a valid table index.</exception>
public static EntityHandle Handle(TableIndex tableIndex, int rowNumber)
{
int token = ((int)tableIndex << TokenTypeIds.RowIdBitCount) | rowNumber;
if (!TokenTypeIds.IsEntityOrUserStringToken(unchecked((uint)token)))
{
Throw.TableIndexOutOfRange();
}
return new EntityHandle((uint)token);
}
public virtual void OnSelectStone(Board board, TableIndex index)
{
}
private int GetTableSize(TableIndex index, int rowSize)
{
return RowCounts[(int)index] * rowSize;
}
public SingleVNode(TFChunkDb db, SingleVNodeSettings vNodeSettings, SingleVNodeAppSettings appSettings)
{
Ensure.NotNull(db, "db");
Ensure.NotNull(vNodeSettings, "vNodeSettings");
db.OpenVerifyAndClean();
_tcpEndPoint = vNodeSettings.ExternalTcpEndPoint;
_httpEndPoint = vNodeSettings.ExternalHttpEndPoint;
_outputBus = new InMemoryBus("OutputBus");
_controller = new SingleVNodeController(Bus, _httpEndPoint);
_mainQueue = new QueuedHandler(_controller, "MainQueue");
_controller.SetMainQueue(MainQueue);
//MONITORING
var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
var monitoringQueue = new QueuedHandler(monitoringInnerBus, "MonitoringQueue", watchSlowMsg: true, slowMsgThresholdMs: 100);
var monitoring = new MonitoringService(monitoringQueue, monitoringRequestBus, db.Config.WriterCheckpoint, db.Config.Path, appSettings.StatsPeriod);
Bus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.SystemInit, Message>());
Bus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>());
monitoringInnerBus.Subscribe <SystemMessage.SystemInit>(monitoring);
monitoringInnerBus.Subscribe <SystemMessage.BecomeShuttingDown>(monitoring);
monitoringInnerBus.Subscribe <MonitoringMessage.GetFreshStats>(monitoring);
//STORAGE SUBSYSTEM
var indexPath = Path.Combine(db.Config.Path, "index");
var tableIndex = new TableIndex(indexPath,
() => new HashListMemTable(),
maxSizeForMemory: 1000000,
maxTablesPerLevel: 2);
var readIndex = new ReadIndex(_mainQueue, TFConsts.ReadIndexReaderCount, () => new TFChunkSequentialReader(db, db.Config.WriterCheckpoint, 0), () => new TFChunkReader(db, db.Config.WriterCheckpoint), tableIndex, new XXHashUnsafe());
var writer = new TFChunkWriter(db);
var storageWriter = new StorageWriter(_mainQueue, _outputBus, writer, readIndex);
var storageReader = new StorageReader(_mainQueue, _outputBus, readIndex, TFConsts.StorageReaderHandlerCount, db.Config.WriterCheckpoint);
monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageReader);
var chaser = new TFChunkChaser(db,
db.Config.WriterCheckpoint,
db.Config.GetNamedCheckpoint(Checkpoint.Chaser));
var storageChaser = new StorageChaser(_mainQueue, chaser);
_outputBus.Subscribe <SystemMessage.SystemInit>(storageChaser);
_outputBus.Subscribe <SystemMessage.SystemStart>(storageChaser);
_outputBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageChaser);
var storageScavenger = new StorageScavenger(db, readIndex);
_outputBus.Subscribe <SystemMessage.ScavengeDatabase>(storageScavenger);
//TCP
var tcpService = new TcpService(MainQueue, _tcpEndPoint);
Bus.Subscribe <SystemMessage.SystemInit>(tcpService);
Bus.Subscribe <SystemMessage.SystemStart>(tcpService);
Bus.Subscribe <SystemMessage.BecomeShuttingDown>(tcpService);
//HTTP
HttpService = new HttpService(ServiceAccessibility.Private, MainQueue, vNodeSettings.HttpPrefixes);
Bus.Subscribe <SystemMessage.SystemInit>(HttpService);
Bus.Subscribe <SystemMessage.BecomeShuttingDown>(HttpService);
Bus.Subscribe <HttpMessage.SendOverHttp>(HttpService);
Bus.Subscribe <HttpMessage.UpdatePendingRequests>(HttpService);
HttpService.SetupController(new AdminController(MainQueue));
HttpService.SetupController(new PingController());
HttpService.SetupController(new StatController(monitoringQueue));
HttpService.SetupController(new ReadEventDataController(MainQueue));
HttpService.SetupController(new AtomController(MainQueue));
HttpService.SetupController(new WebSiteController(MainQueue));
//REQUEST MANAGEMENT
var requestManagement = new RequestManagementService(MainQueue, 1, 1);
Bus.Subscribe <ReplicationMessage.CreateStreamRequestCreated>(requestManagement);
Bus.Subscribe <ReplicationMessage.WriteRequestCreated>(requestManagement);
Bus.Subscribe <ReplicationMessage.TransactionStartRequestCreated>(requestManagement);
Bus.Subscribe <ReplicationMessage.TransactionWriteRequestCreated>(requestManagement);
Bus.Subscribe <ReplicationMessage.TransactionCommitRequestCreated>(requestManagement);
Bus.Subscribe <ReplicationMessage.DeleteStreamRequestCreated>(requestManagement);
Bus.Subscribe <ReplicationMessage.RequestCompleted>(requestManagement);
Bus.Subscribe <ReplicationMessage.AlreadyCommitted>(requestManagement);
Bus.Subscribe <ReplicationMessage.CommitAck>(requestManagement);
Bus.Subscribe <ReplicationMessage.PrepareAck>(requestManagement);
Bus.Subscribe <ReplicationMessage.WrongExpectedVersion>(requestManagement);
Bus.Subscribe <ReplicationMessage.InvalidTransaction>(requestManagement);
Bus.Subscribe <ReplicationMessage.StreamDeleted>(requestManagement);
Bus.Subscribe <ReplicationMessage.PreparePhaseTimeout>(requestManagement);
Bus.Subscribe <ReplicationMessage.CommitPhaseTimeout>(requestManagement);
var clientService = new ClientService();
Bus.Subscribe <TcpMessage.ConnectionClosed>(clientService);
Bus.Subscribe <ClientMessage.SubscribeToStream>(clientService);
Bus.Subscribe <ClientMessage.UnsubscribeFromStream>(clientService);
Bus.Subscribe <ClientMessage.SubscribeToAllStreams>(clientService);
Bus.Subscribe <ClientMessage.UnsubscribeFromAllStreams>(clientService);
Bus.Subscribe <ReplicationMessage.EventCommited>(clientService);
//TIMER
//var timer = new TimerService(new TimerBasedScheduler(new RealTimer(), new RealTimeProvider()));
TimerService = new TimerService(new ThreadBasedScheduler(new RealTimeProvider()));
Bus.Subscribe <TimerMessage.Schedule>(TimerService);
MainQueue.Start();
monitoringQueue.Start();
}
private bool ReferenceFits(int bitCount, TableIndex[] tables)
{
int maxIndex = (1 << bitCount) - 1;
foreach (TableIndex table in tables)
{
if (RowCounts[(int)table] > maxIndex)
{
return false;
}
}
return true;
}
public void ConstructTableIndexWithCorruptIndexEntries(byte version, bool skipIndexVerify,
bool createForceVerifyFile = false)
{
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
var fakeReader = new TFReaderLease(new FakeIndexReader());
_tableIndex = new TableIndex(PathName, lowHasher, highHasher,
() => new HashListMemTable(version, maxSize: NumIndexEntries),
() => fakeReader,
version,
int.MaxValue, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: NumIndexEntries,
skipIndexVerify: skipIndexVerify);
_tableIndex.Initialize(long.MaxValue);
//create index entries
for (int i = 1; i <= NumIndexEntries; i++)
{
_tableIndex.Add(i * 1337, StreamName, i, i * 1337);
}
_tableIndex.Close(false);
//load index map to obtain ptable filenames
_indexMap = IndexMapTestFactory.FromFile(Path.Combine(PathName, TableIndex.IndexMapFilename));
List <string> ptableFiles = new List <string>();
foreach (string ptableFilename in _indexMap.GetAllFilenames())
{
ptableFiles.Add(ptableFilename);
}
_indexMap.Dispose(TimeSpan.FromSeconds(5));
//corrupt ptable files
foreach (string ptableFilename in ptableFiles)
{
CorruptPTableFile(ptableFilename, version, "zeroOutMiddleEntry");
}
//create force verify file if requested
if (createForceVerifyFile)
{
using (FileStream fs = new FileStream(Path.Combine(PathName, TableIndex.ForceIndexVerifyFilename),
FileMode.OpenOrCreate)) {
}
;
}
//load table index again
_tableIndex = new TableIndex(PathName, lowHasher, highHasher,
() => new HashListMemTable(version, maxSize: NumIndexEntries),
() => fakeReader,
version,
int.MaxValue, Constants.PTableMaxReaderCountDefault,
maxSizeForMemory: NumIndexEntries,
skipIndexVerify: skipIndexVerify,
indexCacheDepth: 8);
_tableIndex.Initialize(long.MaxValue);
}
internal static EditAndContinueLogEntry Row(int rowNumber, TableIndex table, EditAndContinueOperation operation)
{
return(new EditAndContinueLogEntry(MetadataTokens.Handle(table, rowNumber), operation));
}
private static MemoryBlock GetTableMetadataBlock(this MetadataReader reader, TableIndex tableIndex)
{
DebugCorlib.Assert(reader != null);
switch (tableIndex)
{
case TableIndex.Module: return(reader.ModuleTable.Block);
case TableIndex.TypeRef: return(reader.TypeRefTable.Block);
case TableIndex.TypeDef: return(reader.TypeDefTable.Block);
case TableIndex.FieldPtr: return(reader.FieldPtrTable.Block);
case TableIndex.Field: return(reader.FieldTable.Block);
case TableIndex.MethodPtr: return(reader.MethodPtrTable.Block);
case TableIndex.MethodDef: return(reader.MethodDefTable.Block);
case TableIndex.ParamPtr: return(reader.ParamPtrTable.Block);
case TableIndex.Param: return(reader.ParamTable.Block);
case TableIndex.InterfaceImpl: return(reader.InterfaceImplTable.Block);
case TableIndex.MemberRef: return(reader.MemberRefTable.Block);
case TableIndex.Constant: return(reader.ConstantTable.Block);
case TableIndex.CustomAttribute: return(reader.CustomAttributeTable.Block);
case TableIndex.FieldMarshal: return(reader.FieldMarshalTable.Block);
case TableIndex.DeclSecurity: return(reader.DeclSecurityTable.Block);
case TableIndex.ClassLayout: return(reader.ClassLayoutTable.Block);
case TableIndex.FieldLayout: return(reader.FieldLayoutTable.Block);
case TableIndex.StandAloneSig: return(reader.StandAloneSigTable.Block);
case TableIndex.EventMap: return(reader.EventMapTable.Block);
case TableIndex.EventPtr: return(reader.EventPtrTable.Block);
case TableIndex.Event: return(reader.EventTable.Block);
case TableIndex.PropertyMap: return(reader.PropertyMapTable.Block);
case TableIndex.PropertyPtr: return(reader.PropertyPtrTable.Block);
case TableIndex.Property: return(reader.PropertyTable.Block);
case TableIndex.MethodSemantics: return(reader.MethodSemanticsTable.Block);
case TableIndex.MethodImpl: return(reader.MethodImplTable.Block);
case TableIndex.ModuleRef: return(reader.ModuleRefTable.Block);
case TableIndex.TypeSpec: return(reader.TypeSpecTable.Block);
case TableIndex.ImplMap: return(reader.ImplMapTable.Block);
case TableIndex.FieldRva: return(reader.FieldRvaTable.Block);
case TableIndex.EncLog: return(reader.EncLogTable.Block);
case TableIndex.EncMap: return(reader.EncMapTable.Block);
case TableIndex.Assembly: return(reader.AssemblyTable.Block);
case TableIndex.AssemblyProcessor: return(reader.AssemblyProcessorTable.Block);
case TableIndex.AssemblyOS: return(reader.AssemblyOSTable.Block);
case TableIndex.AssemblyRef: return(reader.AssemblyRefTable.Block);
case TableIndex.AssemblyRefProcessor: return(reader.AssemblyRefProcessorTable.Block);
case TableIndex.AssemblyRefOS: return(reader.AssemblyRefOSTable.Block);
case TableIndex.File: return(reader.FileTable.Block);
case TableIndex.ExportedType: return(reader.ExportedTypeTable.Block);
case TableIndex.ManifestResource: return(reader.ManifestResourceTable.Block);
case TableIndex.NestedClass: return(reader.NestedClassTable.Block);
case TableIndex.GenericParam: return(reader.GenericParamTable.Block);
case TableIndex.MethodSpec: return(reader.MethodSpecTable.Block);
case TableIndex.GenericParamConstraint: return(reader.GenericParamConstraintTable.Block);
default:
throw new ArgumentOutOfRangeException("tableIndex");
}
}
/// <summary>
/// Returns the size of a row in the specified table.
/// </summary>
/// <exception cref="ArgumentNullException"><paramref name="reader"/> is null.</exception>
/// <exception cref="ArgumentOutOfRangeException"><paramref name="tableIndex"/> is not a valid table index.</exception>
public static int GetTableRowSize(this MetadataReader reader, TableIndex tableIndex)
{
if (reader == null)
{
throw new ArgumentNullException(nameof(reader));
}
return(tableIndex switch
{
TableIndex.Module => reader.ModuleTable.RowSize,
TableIndex.TypeRef => reader.TypeRefTable.RowSize,
TableIndex.TypeDef => reader.TypeDefTable.RowSize,
TableIndex.FieldPtr => reader.FieldPtrTable.RowSize,
TableIndex.Field => reader.FieldTable.RowSize,
TableIndex.MethodPtr => reader.MethodPtrTable.RowSize,
TableIndex.MethodDef => reader.MethodDefTable.RowSize,
TableIndex.ParamPtr => reader.ParamPtrTable.RowSize,
TableIndex.Param => reader.ParamTable.RowSize,
TableIndex.InterfaceImpl => reader.InterfaceImplTable.RowSize,
TableIndex.MemberRef => reader.MemberRefTable.RowSize,
TableIndex.Constant => reader.ConstantTable.RowSize,
TableIndex.CustomAttribute => reader.CustomAttributeTable.RowSize,
TableIndex.FieldMarshal => reader.FieldMarshalTable.RowSize,
TableIndex.DeclSecurity => reader.DeclSecurityTable.RowSize,
TableIndex.ClassLayout => reader.ClassLayoutTable.RowSize,
TableIndex.FieldLayout => reader.FieldLayoutTable.RowSize,
TableIndex.StandAloneSig => reader.StandAloneSigTable.RowSize,
TableIndex.EventMap => reader.EventMapTable.RowSize,
TableIndex.EventPtr => reader.EventPtrTable.RowSize,
TableIndex.Event => reader.EventTable.RowSize,
TableIndex.PropertyMap => reader.PropertyMapTable.RowSize,
TableIndex.PropertyPtr => reader.PropertyPtrTable.RowSize,
TableIndex.Property => reader.PropertyTable.RowSize,
TableIndex.MethodSemantics => reader.MethodSemanticsTable.RowSize,
TableIndex.MethodImpl => reader.MethodImplTable.RowSize,
TableIndex.ModuleRef => reader.ModuleRefTable.RowSize,
TableIndex.TypeSpec => reader.TypeSpecTable.RowSize,
TableIndex.ImplMap => reader.ImplMapTable.RowSize,
TableIndex.FieldRva => reader.FieldRvaTable.RowSize,
TableIndex.EncLog => reader.EncLogTable.RowSize,
TableIndex.EncMap => reader.EncMapTable.RowSize,
TableIndex.Assembly => reader.AssemblyTable.RowSize,
TableIndex.AssemblyProcessor => reader.AssemblyProcessorTable.RowSize,
TableIndex.AssemblyOS => reader.AssemblyOSTable.RowSize,
TableIndex.AssemblyRef => reader.AssemblyRefTable.RowSize,
TableIndex.AssemblyRefProcessor => reader.AssemblyRefProcessorTable.RowSize,
TableIndex.AssemblyRefOS => reader.AssemblyRefOSTable.RowSize,
TableIndex.File => reader.FileTable.RowSize,
TableIndex.ExportedType => reader.ExportedTypeTable.RowSize,
TableIndex.ManifestResource => reader.ManifestResourceTable.RowSize,
TableIndex.NestedClass => reader.NestedClassTable.RowSize,
TableIndex.GenericParam => reader.GenericParamTable.RowSize,
TableIndex.MethodSpec => reader.MethodSpecTable.RowSize,
TableIndex.GenericParamConstraint => reader.GenericParamConstraintTable.RowSize,
// debug tables
TableIndex.Document => reader.DocumentTable.RowSize,
TableIndex.MethodDebugInformation => reader.MethodDebugInformationTable.RowSize,
TableIndex.LocalScope => reader.LocalScopeTable.RowSize,
TableIndex.LocalVariable => reader.LocalVariableTable.RowSize,
TableIndex.LocalConstant => reader.LocalConstantTable.RowSize,
TableIndex.ImportScope => reader.ImportScopeTable.RowSize,
TableIndex.StateMachineMethod => reader.StateMachineMethodTable.RowSize,
TableIndex.CustomDebugInformation => reader.CustomDebugInformationTable.RowSize,
_ => throw new ArgumentOutOfRangeException(nameof(tableIndex)),
});
private void WriteTableName(TableIndex index)
{
WriteRows(MakeTableName(index));
}
public bool IsPresent(TableIndex table) => (PresentTablesMask & (1UL << (int)table)) != 0;
private string MakeTableName(TableIndex index)
{
return($"{index} (index: 0x{(byte)index:X2}, size: {_reader.GetTableRowCount(index) * _reader.GetTableRowSize(index)}): ");
}
/// <summary>
/// Gets the <see cref="TableIndex"/> of the table corresponding to the specified <see cref="HandleKind"/>.
/// </summary>
/// <param name="type">Handle type.</param>
/// <param name="index">Table index.</param>
/// <returns>True if the handle type corresponds to an Ecma335 table, false otherwise.</returns>
public static bool TryGetTableIndex(HandleKind type, out TableIndex index)
{
if ((int)type < TableIndexExtensions.Count)
{
index = (TableIndex)type;
return true;
}
index = 0;
return false;
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_indexDir = PathName;
var scavengeBlocker = new ManualResetEventSlim(false);
var scavengeStarted = new ManualResetEventSlim(false);
var fakeReader = new TFReaderLease(new FakeIndexReader(l => {
scavengeStarted.Set();
if (!scavengeBlocker.Wait(5000))
{
throw new Exception("Failed to continue.");
}
return(false);
}));
_lowHasher = new XXHashUnsafe();
_highHasher = new Murmur3AUnsafe();
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV4,
5,
maxSizeForMemory: 2,
maxTablesPerLevel: 5);
_tableIndex.Initialize(long.MaxValue);
_tableIndex.Add(1, "testStream-1", 0, 0);
_tableIndex.Add(1, "testStream-1", 1, 100);
_tableIndex.WaitForBackgroundTasks();
_log = new FakeTFScavengerLog();
var task = Task.Run(() => _tableIndex.Scavenge(_log, CancellationToken.None));
Assert.That(scavengeStarted.Wait(5000));
// Add enough for 2 more tables
_tableIndex.Add(1, "testStream-1", 2, 200);
_tableIndex.Add(1, "testStream-1", 3, 300);
_tableIndex.Add(1, "testStream-1", 4, 400);
_tableIndex.Add(1, "testStream-1", 5, 500);
// Release the scavenge process
scavengeBlocker.Set();
task.Wait();
// Check it's loadable.
_tableIndex.Close(false);
_tableIndex = new TableIndex(_indexDir, _lowHasher, _highHasher,
() => new HashListMemTable(PTableVersions.IndexV4, maxSize: 5),
() => fakeReader,
PTableVersions.IndexV4,
5,
maxSizeForMemory: 2,
maxTablesPerLevel: 5);
_tableIndex.Initialize(long.MaxValue);
}
public static void lua_getfield(LuaState luaState, TableIndex index, string k) => lua_getfield(luaState, (int)index, k);
public Stone this[TableIndex index] {
get => stones.Get(index);
internal static EditAndContinueLogEntry Row(int rowNumber, TableIndex table, EditAndContinueOperation operation)
{
return new EditAndContinueLogEntry(MetadataTokens.Handle(table, rowNumber), operation);
}
private void BindTableView()
{
TableView.GridLines = true;
TableView.View = View.Details;
TableView.Clear();
TableView.Columns.Add("状态", 40);
for (int i = 0; i < Analysis.Va.Count; ++i)
{
TableView.Columns.Add(Analysis.Va[i].ToString(), 30);
}
for (int i = 0; i < Analysis.Vg.Count; ++i)
{
TableView.Columns.Add(Analysis.Vg[i].ToString(), 30);
}
int c = TableView.Columns.Count;
for (int i = 0; i < Analysis._ItemSetMap.Count; ++i)
{
List <String> a = new List <string>();
a.Add(i.ToString());
for (int k = 0; k < Analysis.Va.Count; ++k)
{
char ch = Analysis.Va[k];
TableIndex index = new TableIndex(i, ch);
if (Analysis.Table.dic.ContainsKey(index))
{
TableItem item = Analysis.Table.dic[index];
if (item.ActionType == 'A')
{
a.Add("acc");
}
else
{
a.Add(item.ActionType.ToString() + item.Detail.ToString());
}
}
else
{
a.Add(".");
}
}
for (int k = 0; k < Analysis.Vg.Count; ++k)
{
char ch = Analysis.Vg[k];
TableIndex index = new TableIndex(i, ch);
if (Analysis.Table.dic.ContainsKey(index))
{
TableItem item = Analysis.Table.dic[index];
if (item.ActionType == 'A')
{
a.Add("acc");
}
else
{
a.Add(item.ActionType.ToString() + item.Detail.ToString());
}
}
else
{
a.Add(".");
}
}
ListViewItem listViewItem = new ListViewItem(a.ToArray());
TableView.Items.Add(listViewItem);
}
}
private void btnGenerate_Click(object sender, EventArgs e)
{
string path = txtFilePath.Text.Trim();
List<TableEntity> lstTable = new List<TableEntity>();
using (FileStream fs = new FileStream(path, FileMode.Open, FileAccess.Read))
{
XSSFWorkbook xssfworkbook = new XSSFWorkbook(fs);
for (int i = 0; i < xssfworkbook.NumberOfSheets; i++)
{
ISheet sheet = xssfworkbook.GetSheetAt(i);
TableEntity tbl = new TableEntity();//实例化一个
System.Collections.IEnumerator rows = sheet.GetRowEnumerator();
int rowIndex = 1;
int columnIndex = 0;//列
int indexIndex = 0;//索引
int fkIndex = 0;//外键
List<string> lstKeyWords = new List<string>();
string strCell0 = string.Empty;
BigCycle:
while (rows.MoveNext())
{
IRow row = (XSSFRow)rows.Current;
strCell0 = row.GetCell(0).ToStringEx();
#region 将每行第一个单元格保存下
if (!strCell0.IsNullOrEmpty())
{
if (!lstKeyWords.Contains(strCell0))
{
lstKeyWords.Add(strCell0);
}
}
#endregion
#region 获取开始行
if (strCell0 == "表名（英文）：")
{
tbl.TableName = row.GetCell(2).ToStringEx();
tbl.TableDesc = row.GetCell(7).ToStringEx();
rowIndex++;
continue;
}
if (strCell0 == "字段名（英文）")
{
columnIndex = rowIndex + 1;
rowIndex++;
continue;
}
if (strCell0 == "索引名称")
{
indexIndex = rowIndex + 1;
rowIndex++;
continue;
}
if (strCell0 == "外键列")
{
fkIndex = rowIndex + 1;
rowIndex++;
continue;
}
#endregion
#region 获取列
if (columnIndex == rowIndex)
{
TableColumn col = new TableColumn();
for (int j = 0; j < row.LastCellNum; j++)
{
ICell cell = row.GetCell(j);
switch (j)
{
case 0:
if (cell.ToStringEx().IsNullOrEmpty() || cell.ToStringEx() == "索引名称")
{
rowIndex++;
goto BigCycle;
}
columnIndex++;
col.ColumnName = cell.ToStringEx();
break;
case 1:
if (cell.ToStringEx().IsNullOrEmpty())
{
this.ShowMessage(tbl.TableName, col.ColumnName, "中文名称不能为空!");
return;
}
col.ColumnCnName = cell.ToStringEx();
break;
case 2:
if (cell.ToStringEx().IsNullOrEmpty())
{
this.ShowMessage(tbl.TableName, col.ColumnName, "数据类型不能为空!");
return;
}
col.DataType = cell.ToStringEx();
break;
case 4:
if (cell.ToStringEx().IsNullOrEmpty())
{
this.ShowMessage(tbl.TableName, col.ColumnName, "主键不能为空!");
return;
}
col.IsPrimaryKey = cell.ToStringEx();
break;
case 5:
if (cell.ToStringEx().IsNullOrEmpty())
{
this.ShowMessage(tbl.TableName, col.ColumnName, "允许空不能为空!");
return;
}
col.IsAllowNull = cell.ToStringEx();
break;
case 6:
if (cell.ToStringEx().IsNullOrEmpty())
{
this.ShowMessage(tbl.TableName, col.ColumnName, "是否唯一不能为空!");
return;
}
col.IsUnique = cell.ToStringEx();
break;
case 7:
if (!cell.ToStringEx().IsNullOrEmpty())
{
col.DefaultValue = cell.ToStringEx();
}
break;
case 8:
col.ColumnDesc = cell.ToStringEx();
break;
default:
break;
}
}
tbl.Columns.Add(col);
}
#endregion
#region 获取索引
if (indexIndex == rowIndex)
{
TableIndex idx = new TableIndex();
for (int j = 0; j < row.LastCellNum; j++)
{
ICell cell = row.GetCell(j);
switch (j)
{
case 0:
if (cell.ToStringEx().IsNullOrEmpty() || cell.ToStringEx() == "外键列")
{
rowIndex++;
goto BigCycle;
}
indexIndex++;
idx.IndexName = cell.ToStringEx();
break;
case 1:
if (cell.ToStringEx().IsNullOrEmpty())
{
this.ShowMessage(tbl.TableName, idx.IndexName, "索引列不能为空!");
return;
}
idx.IndexColumn = cell.ToStringEx();
break;
case 2:
idx.IndexIncludeColumn = cell.ToStringEx();
break;
default:
break;
}
}
tbl.Indexs.Add(idx);
}
#endregion
#region 获取外键
if (fkIndex == rowIndex)
{
TableForeignKey fk = new TableForeignKey();
for (int j = 0; j < row.LastCellNum; j++)
{
ICell cell = row.GetCell(j);
switch (j)
{
case 0:
if (cell.ToStringEx().IsNullOrEmpty())
{
rowIndex++;
goto BigCycle;
}
columnIndex++;
fk.ForeignKey = cell.ToStringEx();
break;
case 1:
if (cell.ToStringEx().IsNullOrEmpty())
{
this.ShowMessage(tbl.TableName, fk.ForeignKey, "主键表不能为空!");
return;
}
fk.ForeignTable = cell.ToStringEx();
break;
case 2:
if (cell.ToStringEx().IsNullOrEmpty())
{
this.ShowMessage(tbl.TableName, fk.ForeignKey, "对应主键表的列不能为空!");
return;
}
fk.ForeignTablePK = cell.ToStringEx();
break;
default:
break;
}
}
tbl.ForeignKeys.Add(fk);
}
#endregion
rowIndex++;
}
if (!tbl.TableName.IsNullOrEmpty() && tbl.TableName.Contains("_"))
{
if (!lstKeyWords.Contains("表名（英文）：") || !lstKeyWords.Contains("字段名（英文）")
|| !lstKeyWords.Contains("索引名称") || !lstKeyWords.Contains("外键列"))
{
MessageBox.Show("Sheet" + tbl.TableName + "格式有问题，缺少列：表名（英文）：、字段名（英文）、索引名称、外键列");
return;
}
else
{
lstTable.Add(tbl);
}
}
}
}
StringBuilder sb = this.GenerateSQL(lstTable);
string savePath = ConfigurationManager.AppSettings["GenerateTable"];
if (!string.IsNullOrEmpty(savePath))
{
if (!Directory.Exists(savePath))
{
Directory.CreateDirectory(savePath);
}
GenerateFileHelper.GenerateFile(sb.ToStringEx(), savePath + DateTime.Now.ToString("yyyy-MM-dd") + "-GenerateTable.sql");
MessageBox.Show("生成成功！", "消息");
System.Diagnostics.Process.Start("Explorer.exe", savePath);
}
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_tableIndex = new TableIndex(PathName, () => new HashListMemTable(), maxSizeForMemory: 2000);
_tableIndex.Initialize();
}
/// <summary>
/// Returns the offset from the start of metadata to the specified table.
/// </summary>
/// <exception cref="ArgumentNullException"><paramref name="reader"/> is null.</exception>
/// <exception cref="ArgumentOutOfRangeException"><paramref name="tableIndex"/> is not a valid table index.</exception>
public static unsafe int GetTableMetadataOffset(this MetadataReader reader, TableIndex tableIndex)
{
if (reader == null)
{
Throw.ArgumentNull(nameof(reader));
}
return (int)(reader.GetTableMetadataBlock(tableIndex).Pointer - reader.Block.Pointer);
}
public ClusterVNode(TFChunkDb db,
ClusterVNodeSettings vNodeSettings,
IGossipSeedSource gossipSeedSource,
InfoController infoController,
params ISubsystem[] subsystems)
{
Ensure.NotNull(db, "db");
Ensure.NotNull(vNodeSettings, "vNodeSettings");
Ensure.NotNull(gossipSeedSource, "gossipSeedSource");
#if DEBUG
AddTask(_taskAddedTrigger.Task);
#endif
var isSingleNode = vNodeSettings.ClusterNodeCount == 1;
_nodeInfo = vNodeSettings.NodeInfo;
_mainBus = new InMemoryBus("MainBus");
var forwardingProxy = new MessageForwardingProxy();
if (vNodeSettings.EnableHistograms)
{
HistogramService.CreateHistograms();
//start watching jitter
HistogramService.StartJitterMonitor();
}
// MISC WORKERS
_workerBuses = Enumerable.Range(0, vNodeSettings.WorkerThreads).Select(queueNum =>
new InMemoryBus(string.Format("Worker #{0} Bus", queueNum + 1),
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(200))).ToArray();
_workersHandler = new MultiQueuedHandler(
vNodeSettings.WorkerThreads,
queueNum => new QueuedHandlerThreadPool(_workerBuses[queueNum],
string.Format("Worker #{0}", queueNum + 1),
groupName: "Workers",
watchSlowMsg: true,
slowMsgThreshold: TimeSpan.FromMilliseconds(200)));
_subsystems = subsystems;
_controller = new ClusterVNodeController((IPublisher)_mainBus, _nodeInfo, db, vNodeSettings, this,
forwardingProxy, _subsystems);
_mainQueue = QueuedHandler.CreateQueuedHandler(_controller, "MainQueue");
_controller.SetMainQueue(_mainQueue);
//SELF
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(this);
_mainBus.Subscribe <SystemMessage.BecomeShutdown>(this);
// MONITORING
var monitoringInnerBus = new InMemoryBus("MonitoringInnerBus", watchSlowMsg: false);
var monitoringRequestBus = new InMemoryBus("MonitoringRequestBus", watchSlowMsg: false);
var monitoringQueue = new QueuedHandlerThreadPool(monitoringInnerBus, "MonitoringQueue", true,
TimeSpan.FromMilliseconds(800));
var monitoring = new MonitoringService(monitoringQueue,
monitoringRequestBus,
_mainQueue,
db.Config.WriterCheckpoint,
db.Config.Path,
vNodeSettings.StatsPeriod,
_nodeInfo.ExternalHttp,
vNodeSettings.StatsStorage,
_nodeInfo.ExternalTcp,
_nodeInfo.ExternalSecureTcp);
_mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.SystemInit, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.StateChangeMessage, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom <SystemMessage.BecomeShutdown, Message>());
_mainBus.Subscribe(monitoringQueue.WidenFrom <ClientMessage.WriteEventsCompleted, Message>());
monitoringInnerBus.Subscribe <SystemMessage.SystemInit>(monitoring);
monitoringInnerBus.Subscribe <SystemMessage.StateChangeMessage>(monitoring);
monitoringInnerBus.Subscribe <SystemMessage.BecomeShuttingDown>(monitoring);
monitoringInnerBus.Subscribe <SystemMessage.BecomeShutdown>(monitoring);
monitoringInnerBus.Subscribe <ClientMessage.WriteEventsCompleted>(monitoring);
monitoringInnerBus.Subscribe <MonitoringMessage.GetFreshStats>(monitoring);
monitoringInnerBus.Subscribe <MonitoringMessage.GetFreshTcpConnectionStats>(monitoring);
var truncPos = db.Config.TruncateCheckpoint.Read();
var writerCheckpoint = db.Config.WriterCheckpoint.Read();
var chaserCheckpoint = db.Config.ChaserCheckpoint.Read();
var epochCheckpoint = db.Config.EpochCheckpoint.Read();
if (truncPos != -1)
{
Log.Info(
"Truncate checkpoint is present. Truncate: {truncatePosition} (0x{truncatePosition:X}), Writer: {writerCheckpoint} (0x{writerCheckpoint:X}), Chaser: {chaserCheckpoint} (0x{chaserCheckpoint:X}), Epoch: {epochCheckpoint} (0x{epochCheckpoint:X})",
truncPos, truncPos, writerCheckpoint, writerCheckpoint, chaserCheckpoint, chaserCheckpoint,
epochCheckpoint, epochCheckpoint);
var truncator = new TFChunkDbTruncator(db.Config);
truncator.TruncateDb(truncPos);
}
// STORAGE SUBSYSTEM
db.Open(vNodeSettings.VerifyDbHash, threads: vNodeSettings.InitializationThreads);
var indexPath = vNodeSettings.Index ?? Path.Combine(db.Config.Path, "index");
var maxReaderCount = ESConsts.PTableMaxReaderCount + vNodeSettings.ReaderThreadsCount;
var readerPool = new ObjectPool <ITransactionFileReader>(
"ReadIndex readers pool", ESConsts.PTableInitialReaderCount, maxReaderCount,
() => new TFChunkReader(db, db.Config.WriterCheckpoint,
optimizeReadSideCache: db.Config.OptimizeReadSideCache));
var tableIndex = new TableIndex(indexPath,
new XXHashUnsafe(),
new Murmur3AUnsafe(),
() => new HashListMemTable(vNodeSettings.IndexBitnessVersion,
maxSize: vNodeSettings.MaxMemtableEntryCount * 2),
() => new TFReaderLease(readerPool),
vNodeSettings.IndexBitnessVersion,
maxSizeForMemory: vNodeSettings.MaxMemtableEntryCount,
maxTablesPerLevel: 2,
inMem: db.Config.InMemDb,
skipIndexVerify: vNodeSettings.SkipIndexVerify,
indexCacheDepth: vNodeSettings.IndexCacheDepth,
initializationThreads: vNodeSettings.InitializationThreads,
additionalReclaim: false,
maxAutoMergeIndexLevel: vNodeSettings.MaxAutoMergeIndexLevel);
var readIndex = new ReadIndex(_mainQueue,
readerPool,
tableIndex,
ESConsts.StreamInfoCacheCapacity,
Application.IsDefined(Application.AdditionalCommitChecks),
Application.IsDefined(Application.InfiniteMetastreams) ? int.MaxValue : 1,
vNodeSettings.HashCollisionReadLimit,
vNodeSettings.SkipIndexScanOnReads,
db.Config.ReplicationCheckpoint);
var writer = new TFChunkWriter(db);
var epochManager = new EpochManager(_mainQueue,
ESConsts.CachedEpochCount,
db.Config.EpochCheckpoint,
writer,
initialReaderCount: 1,
maxReaderCount: 5,
readerFactory: () => new TFChunkReader(db, db.Config.WriterCheckpoint,
optimizeReadSideCache: db.Config.OptimizeReadSideCache));
epochManager.Init();
var storageWriter = new ClusterStorageWriterService(_mainQueue, _mainBus, vNodeSettings.MinFlushDelay,
db, writer, readIndex.IndexWriter, epochManager,
() => readIndex.LastCommitPosition); // subscribes internally
AddTasks(storageWriter.Tasks);
monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageWriter);
var storageReader = new StorageReaderService(_mainQueue, _mainBus, readIndex,
vNodeSettings.ReaderThreadsCount, db.Config.WriterCheckpoint);
_mainBus.Subscribe <SystemMessage.SystemInit>(storageReader);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageReader);
_mainBus.Subscribe <SystemMessage.BecomeShutdown>(storageReader);
monitoringRequestBus.Subscribe <MonitoringMessage.InternalStatsRequest>(storageReader);
var indexCommitterService = new IndexCommitterService(readIndex.IndexCommitter, _mainQueue,
db.Config.ReplicationCheckpoint, db.Config.WriterCheckpoint, vNodeSettings.CommitAckCount, tableIndex);
AddTask(indexCommitterService.Task);
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(indexCommitterService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(indexCommitterService);
_mainBus.Subscribe <StorageMessage.CommitAck>(indexCommitterService);
_mainBus.Subscribe <ClientMessage.MergeIndexes>(indexCommitterService);
var chaser = new TFChunkChaser(db, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint,
db.Config.OptimizeReadSideCache);
var storageChaser = new StorageChaser(_mainQueue, db.Config.WriterCheckpoint, chaser, indexCommitterService,
epochManager);
AddTask(storageChaser.Task);
#if DEBUG
QueueStatsCollector.InitializeCheckpoints(
_nodeInfo.DebugIndex, db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint);
#endif
_mainBus.Subscribe <SystemMessage.SystemInit>(storageChaser);
_mainBus.Subscribe <SystemMessage.SystemStart>(storageChaser);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(storageChaser);
// AUTHENTICATION INFRASTRUCTURE - delegate to plugins
_internalAuthenticationProvider =
vNodeSettings.AuthenticationProviderFactory.BuildAuthenticationProvider(_mainQueue, _mainBus,
_workersHandler, _workerBuses);
Ensure.NotNull(_internalAuthenticationProvider, "authenticationProvider");
{
// EXTERNAL TCP
if (!vNodeSettings.DisableInsecureTCP)
{
var extTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalTcp, _workersHandler,
TcpServiceType.External, TcpSecurityType.Normal, new ClientTcpDispatcher(),
vNodeSettings.ExtTcpHeartbeatInterval, vNodeSettings.ExtTcpHeartbeatTimeout,
_internalAuthenticationProvider, null, vNodeSettings.ConnectionPendingSendBytesThreshold,
vNodeSettings.ConnectionQueueSizeThreshold);
_mainBus.Subscribe <SystemMessage.SystemInit>(extTcpService);
_mainBus.Subscribe <SystemMessage.SystemStart>(extTcpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(extTcpService);
}
// EXTERNAL SECURE TCP
if (_nodeInfo.ExternalSecureTcp != null)
{
var extSecTcpService = new TcpService(_mainQueue, _nodeInfo.ExternalSecureTcp, _workersHandler,
TcpServiceType.External, TcpSecurityType.Secure, new ClientTcpDispatcher(),
vNodeSettings.ExtTcpHeartbeatInterval, vNodeSettings.ExtTcpHeartbeatTimeout,
_internalAuthenticationProvider, vNodeSettings.Certificate,
vNodeSettings.ConnectionPendingSendBytesThreshold, vNodeSettings.ConnectionQueueSizeThreshold);
_mainBus.Subscribe <SystemMessage.SystemInit>(extSecTcpService);
_mainBus.Subscribe <SystemMessage.SystemStart>(extSecTcpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(extSecTcpService);
}
if (!isSingleNode)
{
// INTERNAL TCP
if (!vNodeSettings.DisableInsecureTCP)
{
var intTcpService = new TcpService(_mainQueue, _nodeInfo.InternalTcp, _workersHandler,
TcpServiceType.Internal, TcpSecurityType.Normal,
new InternalTcpDispatcher(),
vNodeSettings.IntTcpHeartbeatInterval, vNodeSettings.IntTcpHeartbeatTimeout,
_internalAuthenticationProvider, null, ESConsts.UnrestrictedPendingSendBytes,
ESConsts.MaxConnectionQueueSize);
_mainBus.Subscribe <SystemMessage.SystemInit>(intTcpService);
_mainBus.Subscribe <SystemMessage.SystemStart>(intTcpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(intTcpService);
}
// INTERNAL SECURE TCP
if (_nodeInfo.InternalSecureTcp != null)
{
var intSecTcpService = new TcpService(_mainQueue, _nodeInfo.InternalSecureTcp, _workersHandler,
TcpServiceType.Internal, TcpSecurityType.Secure,
new InternalTcpDispatcher(),
vNodeSettings.IntTcpHeartbeatInterval, vNodeSettings.IntTcpHeartbeatTimeout,
_internalAuthenticationProvider, vNodeSettings.Certificate,
ESConsts.UnrestrictedPendingSendBytes,
ESConsts.MaxConnectionQueueSize);
_mainBus.Subscribe <SystemMessage.SystemInit>(intSecTcpService);
_mainBus.Subscribe <SystemMessage.SystemStart>(intSecTcpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(intSecTcpService);
}
}
}
SubscribeWorkers(bus => {
var tcpSendService = new TcpSendService();
// ReSharper disable RedundantTypeArgumentsOfMethod
bus.Subscribe <TcpMessage.TcpSend>(tcpSendService);
// ReSharper restore RedundantTypeArgumentsOfMethod
});
var httpAuthenticationProviders = new List <HttpAuthenticationProvider> {
new BasicHttpAuthenticationProvider(_internalAuthenticationProvider),
};
if (vNodeSettings.EnableTrustedAuth)
{
httpAuthenticationProviders.Add(new TrustedHttpAuthenticationProvider());
}
httpAuthenticationProviders.Add(new AnonymousHttpAuthenticationProvider());
var httpPipe = new HttpMessagePipe();
var httpSendService = new HttpSendService(httpPipe, forwardRequests: true);
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(httpSendService);
_mainBus.Subscribe(new WideningHandler <HttpMessage.SendOverHttp, Message>(_workersHandler));
SubscribeWorkers(bus => {
bus.Subscribe <HttpMessage.HttpSend>(httpSendService);
bus.Subscribe <HttpMessage.HttpSendPart>(httpSendService);
bus.Subscribe <HttpMessage.HttpBeginSend>(httpSendService);
bus.Subscribe <HttpMessage.HttpEndSend>(httpSendService);
bus.Subscribe <HttpMessage.SendOverHttp>(httpSendService);
});
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(infoController);
var adminController = new AdminController(_mainQueue, _workersHandler);
var pingController = new PingController();
var histogramController = new HistogramController();
var statController = new StatController(monitoringQueue, _workersHandler);
var atomController = new AtomController(httpSendService, _mainQueue, _workersHandler,
vNodeSettings.DisableHTTPCaching);
var gossipController = new GossipController(_mainQueue, _workersHandler, vNodeSettings.GossipTimeout);
var persistentSubscriptionController =
new PersistentSubscriptionController(httpSendService, _mainQueue, _workersHandler);
var electController = new ElectController(_mainQueue);
// HTTP SENDERS
gossipController.SubscribeSenders(httpPipe);
electController.SubscribeSenders(httpPipe);
// EXTERNAL HTTP
_externalHttpService = new HttpService(ServiceAccessibility.Public, _mainQueue, new TrieUriRouter(),
_workersHandler, vNodeSettings.LogHttpRequests, vNodeSettings.GossipAdvertiseInfo.AdvertiseExternalIPAs,
vNodeSettings.GossipAdvertiseInfo.AdvertiseExternalHttpPortAs, vNodeSettings.DisableFirstLevelHttpAuthorization, vNodeSettings.ExtHttpPrefixes);
_externalHttpService.SetupController(persistentSubscriptionController);
if (vNodeSettings.AdminOnPublic)
{
_externalHttpService.SetupController(adminController);
}
_externalHttpService.SetupController(pingController);
_externalHttpService.SetupController(infoController);
if (vNodeSettings.StatsOnPublic)
{
_externalHttpService.SetupController(statController);
}
_externalHttpService.SetupController(atomController);
if (vNodeSettings.GossipOnPublic)
{
_externalHttpService.SetupController(gossipController);
}
_externalHttpService.SetupController(histogramController);
_mainBus.Subscribe <SystemMessage.SystemInit>(_externalHttpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_externalHttpService);
_mainBus.Subscribe <HttpMessage.PurgeTimedOutRequests>(_externalHttpService);
// INTERNAL HTTP
if (!isSingleNode)
{
_internalHttpService = new HttpService(ServiceAccessibility.Private, _mainQueue, new TrieUriRouter(),
_workersHandler, vNodeSettings.LogHttpRequests,
vNodeSettings.GossipAdvertiseInfo.AdvertiseInternalIPAs,
vNodeSettings.GossipAdvertiseInfo.AdvertiseInternalHttpPortAs, vNodeSettings.DisableFirstLevelHttpAuthorization, vNodeSettings.IntHttpPrefixes);
_internalHttpService.SetupController(adminController);
_internalHttpService.SetupController(pingController);
_internalHttpService.SetupController(infoController);
_internalHttpService.SetupController(statController);
_internalHttpService.SetupController(atomController);
_internalHttpService.SetupController(gossipController);
_internalHttpService.SetupController(electController);
_internalHttpService.SetupController(histogramController);
_internalHttpService.SetupController(persistentSubscriptionController);
}
// Authentication plugin HTTP
vNodeSettings.AuthenticationProviderFactory.RegisterHttpControllers(_externalHttpService,
_internalHttpService, httpSendService, _mainQueue, _workersHandler);
if (_internalHttpService != null)
{
_mainBus.Subscribe <SystemMessage.SystemInit>(_internalHttpService);
_mainBus.Subscribe <SystemMessage.BecomeShuttingDown>(_internalHttpService);
_mainBus.Subscribe <HttpMessage.PurgeTimedOutRequests>(_internalHttpService);
}
SubscribeWorkers(bus => {
HttpService.CreateAndSubscribePipeline(bus, httpAuthenticationProviders.ToArray());
});
// REQUEST FORWARDING
var forwardingService = new RequestForwardingService(_mainQueue, forwardingProxy, TimeSpan.FromSeconds(1));
_mainBus.Subscribe <SystemMessage.SystemStart>(forwardingService);
_mainBus.Subscribe <SystemMessage.RequestForwardingTimerTick>(forwardingService);
_mainBus.Subscribe <ClientMessage.NotHandled>(forwardingService);
_mainBus.Subscribe <ClientMessage.WriteEventsCompleted>(forwardingService);
_mainBus.Subscribe <ClientMessage.TransactionStartCompleted>(forwardingService);
_mainBus.Subscribe <ClientMessage.TransactionWriteCompleted>(forwardingService);
_mainBus.Subscribe <ClientMessage.TransactionCommitCompleted>(forwardingService);
_mainBus.Subscribe <ClientMessage.DeleteStreamCompleted>(forwardingService);
// REQUEST MANAGEMENT
var requestManagement = new RequestManagementService(_mainQueue,
vNodeSettings.PrepareAckCount,
vNodeSettings.PrepareTimeout,
vNodeSettings.CommitTimeout,
vNodeSettings.BetterOrdering);
_mainBus.Subscribe <SystemMessage.SystemInit>(requestManagement);
_mainBus.Subscribe <ClientMessage.WriteEvents>(requestManagement);
_mainBus.Subscribe <ClientMessage.TransactionStart>(requestManagement);
_mainBus.Subscribe <ClientMessage.TransactionWrite>(requestManagement);
_mainBus.Subscribe <ClientMessage.TransactionCommit>(requestManagement);
_mainBus.Subscribe <ClientMessage.DeleteStream>(requestManagement);
_mainBus.Subscribe <StorageMessage.RequestCompleted>(requestManagement);
_mainBus.Subscribe <StorageMessage.CheckStreamAccessCompleted>(requestManagement);
_mainBus.Subscribe <StorageMessage.AlreadyCommitted>(requestManagement);
_mainBus.Subscribe <StorageMessage.CommitReplicated>(requestManagement);
_mainBus.Subscribe <StorageMessage.PrepareAck>(requestManagement);
_mainBus.Subscribe <StorageMessage.WrongExpectedVersion>(requestManagement);
_mainBus.Subscribe <StorageMessage.InvalidTransaction>(requestManagement);
_mainBus.Subscribe <StorageMessage.StreamDeleted>(requestManagement);
_mainBus.Subscribe <StorageMessage.RequestManagerTimerTick>(requestManagement);
// SUBSCRIPTIONS
var subscrBus = new InMemoryBus("SubscriptionsBus", true, TimeSpan.FromMilliseconds(50));
var subscrQueue = new QueuedHandlerThreadPool(subscrBus, "Subscriptions", false);
_mainBus.Subscribe(subscrQueue.WidenFrom <SystemMessage.SystemStart, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <SystemMessage.BecomeShuttingDown, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <TcpMessage.ConnectionClosed, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.SubscribeToStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <ClientMessage.UnsubscribeFromStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <SubscriptionMessage.PollStream, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <SubscriptionMessage.CheckPollTimeout, Message>());
_mainBus.Subscribe(subscrQueue.WidenFrom <StorageMessage.EventCommitted, Message>());
var subscription = new SubscriptionsService(_mainQueue, subscrQueue, readIndex);
subscrBus.Subscribe <SystemMessage.SystemStart>(subscription);
subscrBus.Subscribe <SystemMessage.BecomeShuttingDown>(subscription);
subscrBus.Subscribe <TcpMessage.ConnectionClosed>(subscription);
subscrBus.Subscribe <ClientMessage.SubscribeToStream>(subscription);
subscrBus.Subscribe <ClientMessage.UnsubscribeFromStream>(subscription);
subscrBus.Subscribe <SubscriptionMessage.PollStream>(subscription);
subscrBus.Subscribe <SubscriptionMessage.CheckPollTimeout>(subscription);
subscrBus.Subscribe <StorageMessage.EventCommitted>(subscription);
// PERSISTENT SUBSCRIPTIONS
// IO DISPATCHER
var ioDispatcher = new IODispatcher(_mainQueue, new PublishEnvelope(_mainQueue));
_mainBus.Subscribe <ClientMessage.ReadStreamEventsBackwardCompleted>(ioDispatcher.BackwardReader);
_mainBus.Subscribe <ClientMessage.WriteEventsCompleted>(ioDispatcher.Writer);
_mainBus.Subscribe <ClientMessage.ReadStreamEventsForwardCompleted>(ioDispatcher.ForwardReader);
_mainBus.Subscribe <ClientMessage.DeleteStreamCompleted>(ioDispatcher.StreamDeleter);
_mainBus.Subscribe(ioDispatcher);
var perSubscrBus = new InMemoryBus("PersistentSubscriptionsBus", true, TimeSpan.FromMilliseconds(50));
var perSubscrQueue = new QueuedHandlerThreadPool(perSubscrBus, "PersistentSubscriptions", false);
_mainBus.Subscribe(perSubscrQueue.WidenFrom <SystemMessage.StateChangeMessage, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <TcpMessage.ConnectionClosed, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.CreatePersistentSubscription, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.UpdatePersistentSubscription, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.DeletePersistentSubscription, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ConnectToPersistentSubscription, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.UnsubscribeFromStream, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.PersistentSubscriptionAckEvents, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.PersistentSubscriptionNackEvents, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ReplayAllParkedMessages, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ReplayParkedMessage, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <ClientMessage.ReadNextNPersistentMessages, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <StorageMessage.EventCommitted, Message>());
_mainBus.Subscribe(perSubscrQueue
.WidenFrom <MonitoringMessage.GetAllPersistentSubscriptionStats, Message>());
_mainBus.Subscribe(
perSubscrQueue.WidenFrom <MonitoringMessage.GetStreamPersistentSubscriptionStats, Message>());
_mainBus.Subscribe(perSubscrQueue.WidenFrom <MonitoringMessage.GetPersistentSubscriptionStats, Message>());
_mainBus.Subscribe(perSubscrQueue
.WidenFrom <SubscriptionMessage.PersistentSubscriptionTimerTick, Message>());
//TODO CC can have multiple threads working on subscription if partition
var consumerStrategyRegistry =
new PersistentSubscriptionConsumerStrategyRegistry(_mainQueue, _mainBus,
vNodeSettings.AdditionalConsumerStrategies);
var persistentSubscription = new PersistentSubscriptionService(perSubscrQueue, readIndex, ioDispatcher,
_mainQueue, consumerStrategyRegistry);
perSubscrBus.Subscribe <SystemMessage.BecomeShuttingDown>(persistentSubscription);
perSubscrBus.Subscribe <SystemMessage.BecomeMaster>(persistentSubscription);
perSubscrBus.Subscribe <SystemMessage.StateChangeMessage>(persistentSubscription);
perSubscrBus.Subscribe <TcpMessage.ConnectionClosed>(persistentSubscription);
perSubscrBus.Subscribe <ClientMessage.ConnectToPersistentSubscription>(persistentSubscription);
perSubscrBus.Subscribe <ClientMessage.UnsubscribeFromStream>(persistentSubscription);
perSubscrBus.Subscribe <ClientMessage.PersistentSubscriptionAckEvents>(persistentSubscription);
perSubscrBus.Subscribe <ClientMessage.PersistentSubscriptionNackEvents>(persistentSubscription);
perSubscrBus.Subscribe <StorageMessage.EventCommitted>(persistentSubscription);
perSubscrBus.Subscribe <ClientMessage.DeletePersistentSubscription>(persistentSubscription);
perSubscrBus.Subscribe <ClientMessage.CreatePersistentSubscription>(persistentSubscription);
perSubscrBus.Subscribe <ClientMessage.UpdatePersistentSubscription>(persistentSubscription);
perSubscrBus.Subscribe <ClientMessage.ReplayAllParkedMessages>(persistentSubscription);
perSubscrBus.Subscribe <ClientMessage.ReplayParkedMessage>(persistentSubscription);
perSubscrBus.Subscribe <ClientMessage.ReadNextNPersistentMessages>(persistentSubscription);
perSubscrBus.Subscribe <MonitoringMessage.GetAllPersistentSubscriptionStats>(persistentSubscription);
perSubscrBus.Subscribe <MonitoringMessage.GetStreamPersistentSubscriptionStats>(persistentSubscription);
perSubscrBus.Subscribe <MonitoringMessage.GetPersistentSubscriptionStats>(persistentSubscription);
perSubscrBus.Subscribe <SubscriptionMessage.PersistentSubscriptionTimerTick>(persistentSubscription);
// STORAGE SCAVENGER
var scavengerLogManager = new TFChunkScavengerLogManager(_nodeInfo.ExternalHttp.ToString(),
TimeSpan.FromDays(vNodeSettings.ScavengeHistoryMaxAge), ioDispatcher);
var storageScavenger = new StorageScavenger(db,
tableIndex,
readIndex,
scavengerLogManager,
vNodeSettings.AlwaysKeepScavenged,
!vNodeSettings.DisableScavengeMerging,
unsafeIgnoreHardDeletes: vNodeSettings.UnsafeIgnoreHardDeletes);
// ReSharper disable RedundantTypeArgumentsOfMethod
_mainBus.Subscribe <ClientMessage.ScavengeDatabase>(storageScavenger);
_mainBus.Subscribe <ClientMessage.StopDatabaseScavenge>(storageScavenger);
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(storageScavenger);
// ReSharper restore RedundantTypeArgumentsOfMethod
// TIMER
_timeProvider = new RealTimeProvider();
var threadBasedScheduler = new ThreadBasedScheduler(_timeProvider);
AddTask(threadBasedScheduler.Task);
_timerService = new TimerService(threadBasedScheduler);
_mainBus.Subscribe <SystemMessage.BecomeShutdown>(_timerService);
_mainBus.Subscribe <TimerMessage.Schedule>(_timerService);
var gossipInfo = new VNodeInfo(_nodeInfo.InstanceId, _nodeInfo.DebugIndex,
vNodeSettings.GossipAdvertiseInfo.InternalTcp,
vNodeSettings.GossipAdvertiseInfo.InternalSecureTcp,
vNodeSettings.GossipAdvertiseInfo.ExternalTcp,
vNodeSettings.GossipAdvertiseInfo.ExternalSecureTcp,
vNodeSettings.GossipAdvertiseInfo.InternalHttp,
vNodeSettings.GossipAdvertiseInfo.ExternalHttp);
if (!isSingleNode)
{
// MASTER REPLICATION
var masterReplicationService = new MasterReplicationService(_mainQueue, gossipInfo.InstanceId, db,
_workersHandler,
epochManager, vNodeSettings.ClusterNodeCount);
AddTask(masterReplicationService.Task);
_mainBus.Subscribe <SystemMessage.SystemStart>(masterReplicationService);
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(masterReplicationService);
_mainBus.Subscribe <ReplicationMessage.ReplicaSubscriptionRequest>(masterReplicationService);
_mainBus.Subscribe <ReplicationMessage.ReplicaLogPositionAck>(masterReplicationService);
monitoringInnerBus.Subscribe <ReplicationMessage.GetReplicationStats>(masterReplicationService);
// REPLICA REPLICATION
var replicaService = new ReplicaService(_mainQueue, db, epochManager, _workersHandler,
_internalAuthenticationProvider,
gossipInfo, vNodeSettings.UseSsl, vNodeSettings.SslTargetHost, vNodeSettings.SslValidateServer,
vNodeSettings.IntTcpHeartbeatTimeout, vNodeSettings.ExtTcpHeartbeatInterval);
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(replicaService);
_mainBus.Subscribe <ReplicationMessage.ReconnectToMaster>(replicaService);
_mainBus.Subscribe <ReplicationMessage.SubscribeToMaster>(replicaService);
_mainBus.Subscribe <ReplicationMessage.AckLogPosition>(replicaService);
_mainBus.Subscribe <StorageMessage.PrepareAck>(replicaService);
_mainBus.Subscribe <StorageMessage.CommitAck>(replicaService);
_mainBus.Subscribe <ClientMessage.TcpForwardMessage>(replicaService);
}
// ELECTIONS
var electionsService = new ElectionsService(_mainQueue, gossipInfo, vNodeSettings.ClusterNodeCount,
db.Config.WriterCheckpoint, db.Config.ChaserCheckpoint,
epochManager, () => readIndex.LastCommitPosition, vNodeSettings.NodePriority);
electionsService.SubscribeMessages(_mainBus);
if (!isSingleNode || vNodeSettings.GossipOnSingleNode)
{
// GOSSIP
var gossip = new NodeGossipService(_mainQueue, gossipSeedSource, gossipInfo, db.Config.WriterCheckpoint,
db.Config.ChaserCheckpoint, epochManager, () => readIndex.LastCommitPosition,
vNodeSettings.NodePriority, vNodeSettings.GossipInterval,
vNodeSettings.GossipAllowedTimeDifference);
_mainBus.Subscribe <SystemMessage.SystemInit>(gossip);
_mainBus.Subscribe <GossipMessage.RetrieveGossipSeedSources>(gossip);
_mainBus.Subscribe <GossipMessage.GotGossipSeedSources>(gossip);
_mainBus.Subscribe <GossipMessage.Gossip>(gossip);
_mainBus.Subscribe <GossipMessage.GossipReceived>(gossip);
_mainBus.Subscribe <SystemMessage.StateChangeMessage>(gossip);
_mainBus.Subscribe <GossipMessage.GossipSendFailed>(gossip);
_mainBus.Subscribe <SystemMessage.VNodeConnectionEstablished>(gossip);
_mainBus.Subscribe <SystemMessage.VNodeConnectionLost>(gossip);
}
AddTasks(_workersHandler.Start());
AddTask(_mainQueue.Start());
AddTask(monitoringQueue.Start());
AddTask(subscrQueue.Start());
AddTask(perSubscrQueue.Start());
if (subsystems != null)
{
foreach (var subsystem in subsystems)
{
var http = isSingleNode
? new[] { _externalHttpService }
: new[] { _internalHttpService, _externalHttpService };
subsystem.Register(new StandardComponents(db, _mainQueue, _mainBus, _timerService, _timeProvider,
httpSendService, http, _workersHandler));
}
}
}
/// <summary>
/// Returns the offset from the start of metadata to the specified table.
/// </summary>
/// <exception cref="ArgumentNullException"><paramref name="reader"/> is null.</exception>
/// <exception cref="ArgumentOutOfRangeException"><paramref name="tableIndex"/> is not a valid table index.</exception>
public static unsafe int GetTableMetadataOffset(this MetadataReader reader, TableIndex tableIndex)
{
if (reader == null)
{
throw new ArgumentNullException("reader");
}
return (int)(reader.GetTableMetadataBlock(tableIndex).Pointer - reader.Block.Pointer);
}
private int GetTableSize(TableIndex index, int rowSize)
{
return(RowCounts[(int)index] * rowSize);
}
/// <summary>
/// Creates an <see cref="Metadata.EntityHandle"/> from a token value.
/// </summary>
/// <exception cref="ArgumentException">
/// <paramref name="tableIndex"/> is not a valid table index.</exception>
public static EntityHandle EntityHandle(TableIndex tableIndex, int rowNumber)
{
return Handle(tableIndex, rowNumber);
}
/// <summary>
/// Creates an <see cref="Metadata.EntityHandle"/> from a token value.
/// </summary>
/// <exception cref="ArgumentException">
/// <paramref name="tableIndex"/> is not a valid table index.</exception>
public static EntityHandle EntityHandle(TableIndex tableIndex, int rowNumber)
{
return(Handle(tableIndex, rowNumber));
}
private int GetReferenceSize(int[] rowCounts, TableIndex index)
{
return (rowCounts[(int)index] < MetadataStreamConstants.LargeTableRowCount && !IsMinimalDelta) ? SmallIndexSize : LargeIndexSize;
}
public StoneViewModel this[TableIndex index] => stoneViewModels.Get(index);
public void SetCapacity(TableIndex table, int capacity)
{
switch (table)
{
case TableIndex.Module: _moduleTable.Capacity = capacity; break;
case TableIndex.TypeRef: _typeRefTable.Capacity = capacity; break;
case TableIndex.TypeDef: _typeDefTable.Capacity = capacity; break;
case TableIndex.Field: _fieldTable.Capacity = capacity; break;
case TableIndex.MethodDef: _methodDefTable.Capacity = capacity; break;
case TableIndex.Param: _paramTable.Capacity = capacity; break;
case TableIndex.InterfaceImpl: _interfaceImplTable.Capacity = capacity; break;
case TableIndex.MemberRef: _memberRefTable.Capacity = capacity; break;
case TableIndex.Constant: _constantTable.Capacity = capacity; break;
case TableIndex.CustomAttribute: _customAttributeTable.Capacity = capacity; break;
case TableIndex.FieldMarshal: _fieldMarshalTable.Capacity = capacity; break;
case TableIndex.DeclSecurity: _declSecurityTable.Capacity = capacity; break;
case TableIndex.ClassLayout: _classLayoutTable.Capacity = capacity; break;
case TableIndex.FieldLayout: _fieldLayoutTable.Capacity = capacity; break;
case TableIndex.StandAloneSig: _standAloneSigTable.Capacity = capacity; break;
case TableIndex.EventMap: _eventMapTable.Capacity = capacity; break;
case TableIndex.Event: _eventTable.Capacity = capacity; break;
case TableIndex.PropertyMap: _propertyMapTable.Capacity = capacity; break;
case TableIndex.Property: _propertyTable.Capacity = capacity; break;
case TableIndex.MethodSemantics: _methodSemanticsTable.Capacity = capacity; break;
case TableIndex.MethodImpl: _methodImplTable.Capacity = capacity; break;
case TableIndex.ModuleRef: _moduleRefTable.Capacity = capacity; break;
case TableIndex.TypeSpec: _typeSpecTable.Capacity = capacity; break;
case TableIndex.ImplMap: _implMapTable.Capacity = capacity; break;
case TableIndex.FieldRva: _fieldRvaTable.Capacity = capacity; break;
case TableIndex.EncLog: _encLogTable.Capacity = capacity; break;
case TableIndex.EncMap: _encMapTable.Capacity = capacity; break;
case TableIndex.Assembly: _assemblyTable.Capacity = capacity; break;
case TableIndex.AssemblyRef: _assemblyRefTable.Capacity = capacity; break;
case TableIndex.File: _fileTable.Capacity = capacity; break;
case TableIndex.ExportedType: _exportedTypeTable.Capacity = capacity; break;
case TableIndex.ManifestResource: _manifestResourceTable.Capacity = capacity; break;
case TableIndex.NestedClass: _nestedClassTable.Capacity = capacity; break;
case TableIndex.GenericParam: _genericParamTable.Capacity = capacity; break;
case TableIndex.MethodSpec: _methodSpecTable.Capacity = capacity; break;
case TableIndex.GenericParamConstraint: _genericParamConstraintTable.Capacity = capacity; break;
case TableIndex.Document: _documentTable.Capacity = capacity; break;
case TableIndex.MethodDebugInformation: _methodDebugInformationTable.Capacity = capacity; break;
case TableIndex.LocalScope: _localScopeTable.Capacity = capacity; break;
case TableIndex.LocalVariable: _localVariableTable.Capacity = capacity; break;
case TableIndex.LocalConstant: _localConstantTable.Capacity = capacity; break;
case TableIndex.ImportScope: _importScopeTable.Capacity = capacity; break;
case TableIndex.StateMachineMethod: _stateMachineMethodTable.Capacity = capacity; break;
case TableIndex.CustomDebugInformation: _customDebugInformationTable.Capacity = capacity; break;
case TableIndex.AssemblyOS:
case TableIndex.AssemblyProcessor:
case TableIndex.AssemblyRefOS:
case TableIndex.AssemblyRefProcessor:
case TableIndex.EventPtr:
case TableIndex.FieldPtr:
case TableIndex.MethodPtr:
case TableIndex.ParamPtr:
case TableIndex.PropertyPtr:
throw new NotSupportedException();
default:
throw new ArgumentOutOfRangeException(nameof(table));
}
}
/// <summary>
/// Returns the size of a row in the specified table.
/// </summary>
/// <exception cref="ArgumentNullException"><paramref name="reader"/> is null.</exception>
/// <exception cref="ArgumentOutOfRangeException"><paramref name="tableIndex"/> is not a valid table index.</exception>
public static int GetTableRowSize(this MetadataReader reader, TableIndex tableIndex)
{
if (reader == null)
{
throw new ArgumentNullException(nameof(reader));
}
switch (tableIndex)
{
case TableIndex.Module: return(reader.ModuleTable.RowSize);
case TableIndex.TypeRef: return(reader.TypeRefTable.RowSize);
case TableIndex.TypeDef: return(reader.TypeDefTable.RowSize);
case TableIndex.FieldPtr: return(reader.FieldPtrTable.RowSize);
case TableIndex.Field: return(reader.FieldTable.RowSize);
case TableIndex.MethodPtr: return(reader.MethodPtrTable.RowSize);
case TableIndex.MethodDef: return(reader.MethodDefTable.RowSize);
case TableIndex.ParamPtr: return(reader.ParamPtrTable.RowSize);
case TableIndex.Param: return(reader.ParamTable.RowSize);
case TableIndex.InterfaceImpl: return(reader.InterfaceImplTable.RowSize);
case TableIndex.MemberRef: return(reader.MemberRefTable.RowSize);
case TableIndex.Constant: return(reader.ConstantTable.RowSize);
case TableIndex.CustomAttribute: return(reader.CustomAttributeTable.RowSize);
case TableIndex.FieldMarshal: return(reader.FieldMarshalTable.RowSize);
case TableIndex.DeclSecurity: return(reader.DeclSecurityTable.RowSize);
case TableIndex.ClassLayout: return(reader.ClassLayoutTable.RowSize);
case TableIndex.FieldLayout: return(reader.FieldLayoutTable.RowSize);
case TableIndex.StandAloneSig: return(reader.StandAloneSigTable.RowSize);
case TableIndex.EventMap: return(reader.EventMapTable.RowSize);
case TableIndex.EventPtr: return(reader.EventPtrTable.RowSize);
case TableIndex.Event: return(reader.EventTable.RowSize);
case TableIndex.PropertyMap: return(reader.PropertyMapTable.RowSize);
case TableIndex.PropertyPtr: return(reader.PropertyPtrTable.RowSize);
case TableIndex.Property: return(reader.PropertyTable.RowSize);
case TableIndex.MethodSemantics: return(reader.MethodSemanticsTable.RowSize);
case TableIndex.MethodImpl: return(reader.MethodImplTable.RowSize);
case TableIndex.ModuleRef: return(reader.ModuleRefTable.RowSize);
case TableIndex.TypeSpec: return(reader.TypeSpecTable.RowSize);
case TableIndex.ImplMap: return(reader.ImplMapTable.RowSize);
case TableIndex.FieldRva: return(reader.FieldRvaTable.RowSize);
case TableIndex.EncLog: return(reader.EncLogTable.RowSize);
case TableIndex.EncMap: return(reader.EncMapTable.RowSize);
case TableIndex.Assembly: return(reader.AssemblyTable.RowSize);
case TableIndex.AssemblyProcessor: return(reader.AssemblyProcessorTable.RowSize);
case TableIndex.AssemblyOS: return(reader.AssemblyOSTable.RowSize);
case TableIndex.AssemblyRef: return(reader.AssemblyRefTable.RowSize);
case TableIndex.AssemblyRefProcessor: return(reader.AssemblyRefProcessorTable.RowSize);
case TableIndex.AssemblyRefOS: return(reader.AssemblyRefOSTable.RowSize);
case TableIndex.File: return(reader.FileTable.RowSize);
case TableIndex.ExportedType: return(reader.ExportedTypeTable.RowSize);
case TableIndex.ManifestResource: return(reader.ManifestResourceTable.RowSize);
case TableIndex.NestedClass: return(reader.NestedClassTable.RowSize);
case TableIndex.GenericParam: return(reader.GenericParamTable.RowSize);
case TableIndex.MethodSpec: return(reader.MethodSpecTable.RowSize);
case TableIndex.GenericParamConstraint: return(reader.GenericParamConstraintTable.RowSize);
// debug tables
case TableIndex.Document: return(reader.DocumentTable.RowSize);
case TableIndex.MethodDebugInformation: return(reader.MethodDebugInformationTable.RowSize);
case TableIndex.LocalScope: return(reader.LocalScopeTable.RowSize);
case TableIndex.LocalVariable: return(reader.LocalVariableTable.RowSize);
case TableIndex.LocalConstant: return(reader.LocalConstantTable.RowSize);
case TableIndex.ImportScope: return(reader.ImportScopeTable.RowSize);
case TableIndex.StateMachineMethod: return(reader.StateMachineMethodTable.RowSize);
case TableIndex.CustomDebugInformation: return(reader.CustomDebugInformationTable.RowSize);
default:
throw new ArgumentOutOfRangeException(nameof(tableIndex));
}
}
internal static void TableNotSorted(TableIndex tableIndex)
{
throw new BadImageFormatException(SR.Format(SR.MetadataTableNotSorted, (int)tableIndex));
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
TableIndex.EnterUnsafeState(PathName);
}
private int GetTableSize(TableIndex index, int rowSize)
{
return (PresentTablesMask & (1UL << (int)index)) != 0 ? RowCounts[(int)index] * rowSize : 0;
}
public void the_map_says_its_in_corrupted_state()
{
Assert.IsTrue(TableIndex.IsCorrupt(PathName));
}
internal static void ThrowTableNotSorted(TableIndex tableIndex)
{
throw new BadImageFormatException(string.Format(MetadataResources.MetadataTableNotSorted, (int)tableIndex));
}
private TestContext CreateTarget(MemoryBinaryStore store)
{
var configuration = new IndexConfiguration {
EntityType = typeof (Car),
Type = typeof (int),
UniqueName = "Nationality"
};
var storage = new IndexStorage<int>(store, configuration);
storage.Initialize();
var index = new TableIndex<int>(storage, new ComparableIndexAlgorithm<int>());
return new TestContext {Store = store, Index = index};
}
