#pragma warning disable 1591
public IdController(
#pragma warning restore 1591
IDataRouter dataRouter,
AuthorizationModule authorizationModule,
NewCollectionTasks newCollectionTasks)
{
this.dataRouter = dataRouter;
this.authorizationModule = authorizationModule;
this.newCollectionTasks = newCollectionTasks;
}
#pragma warning disable 1591
public DownloadController(
#pragma warning restore 1591
IDataRouter dataRouter,
AuthorizationModule authorizationModule,
IEventLogger apiEventLogger)
{
this.dataRouter = dataRouter;
this.authorizationModule = authorizationModule;
this.apiEventLogger = apiEventLogger;
}
public ValuesController(
IDataRouter router,
ClusterClient clusterClient,
NodeStateService node,
IStateMachine <TestState> stateMachine)
{
_clusterClient = clusterClient;
_router = (TestDataRouter)router;
_node = node;
_stateMachine = stateMachine;
}
public CreateUserCommandHandler(
IEntitiesRepository entitiesRepository,
ILogger <CreateUserCommandHandler> logger,
IServiceProvider prov,
IDataRouter router,
IClusterRequestHandler node
)
{
_entitiesRepository = entitiesRepository;
_node = (IClusterRequestHandler)prov.GetService(typeof(IClusterRequestHandler));
}
#pragma warning disable 1591
public ViewController(
#pragma warning restore 1591
ViewManager viewManager,
IDataRouter dataRouter,
AuthorizationModule authorizationModule,
IEventLogger apiEventLogger)
{
this.viewManager = viewManager;
this.dataRouter = dataRouter;
this.authorizationModule = authorizationModule;
this.apiEventLogger = apiEventLogger;
}
public static async Task <IActionResult> PerformSearch(IDataRouter dataRouter, string query, ResultFormat resultFormat)
{
try
{
var parsedQuery = DataApiSqlQueryParser.Parse(query);
var collection = parsedQuery.FromArguments.Trim();
var rdDataStorage = await dataRouter.GetSourceSystemAsync(collection);
var result = rdDataStorage.SearchAsync(parsedQuery);
switch (resultFormat)
{
case ResultFormat.Json:
return(BuildJsonResultAsync(result));
case ResultFormat.Csv:
return(BuildCsvResultAsync(result));
default:
throw new ArgumentOutOfRangeException(nameof(resultFormat), resultFormat, null);
}
}
catch (FormatException formatException)
{
return(new ContentResult
{
ContentType = "text/plain",
Content = formatException.Message,
StatusCode = (int)HttpStatusCode.BadRequest
});
}
catch (AggregateException aggregateException)
{
var innermostException = aggregateException.InnermostException();
if (innermostException is FormatException)
{
return(new ContentResult
{
ContentType = "text/plain",
Content = aggregateException.Message,
StatusCode = (int)HttpStatusCode.BadRequest
});
}
return(new ContentResult
{
Content = aggregateException.InnerException?.Message ?? aggregateException.Message,
ContentType = "text/plain",
StatusCode = (int)HttpStatusCode.InternalServerError
});
}
}
public Writer(
ILogger <Writer <State> > logger,
IShardRepository shardRepository,
IDataRouter dataRouter,
IStateMachine <State> stateMachine,
NodeStateService nodeStateService,
ClusterClient clusterClient)
{
_logger = logger;
_dataRouter = dataRouter;
_shardRepository = shardRepository;
_stateMachine = stateMachine;
_nodeStateService = nodeStateService;
_clusterClient = clusterClient;
}
//------------------------------------------------------------------
/// <summary>
/// Get writer preconfigured to track unsaved changes and allow subfolders creation
/// </summary>
/// <param name="targetFolder"></param>
/// <param name="desiredFileSize"></param>
/// <param name="router"></param>
/// <returns></returns>
public IRepositoryWriter GetWriter(IRepositoryFolder targetFolder, IDataRouter router)
{
IRepositoryWriter writer = targetFolder.GetWriter();
writer.TrackUnsavedItems = true;
if (router != null)
{
writer.DataRouter = router;
}
writer.AllowSubfoldersCreation = true;
return(writer);
}
public TextRuleEvaluator(IDataRouter dataRouter)
{
var referenceRuleEvaluator = new IsReferenceRuleEvaluator(dataRouter);
Commands = new List <TextRuleCommand>
{
new TextRuleCommand("(NOT )?EXISTS", CheckExistance, false),
new TextRuleCommand("IS( NOT)? LIST", (jToken, regexMatch) => IsRuleEvaluator.CheckType(jToken, regexMatch, JTokenType.Array), true),
new TextRuleCommand("IS( NOT)? ARRAY", (jToken, regexMatch) => IsRuleEvaluator.CheckType(jToken, regexMatch, JTokenType.Array), true),
new TextRuleCommand("IS( NOT)? STRING", (jToken, regexMatch) => IsRuleEvaluator.CheckType(jToken, regexMatch, JTokenType.String, JTokenType.Guid, JTokenType.Uri), false),
new TextRuleCommand("IS( NOT)? TEXT", (jToken, regexMatch) => IsRuleEvaluator.CheckType(jToken, regexMatch, JTokenType.String, JTokenType.Guid, JTokenType.Uri), false),
new TextRuleCommand("IS( NOT)? NUMBER", (jToken, regexMatch) => IsRuleEvaluator.CheckType(jToken, regexMatch, JTokenType.Integer, JTokenType.Float), false),
new TextRuleCommand("IS( NOT)? INT", (jToken, regexMatch) => IsRuleEvaluator.CheckType(jToken, regexMatch, JTokenType.Integer), false),
new TextRuleCommand("IS( NOT)? FLOAT", (jToken, regexMatch) => IsRuleEvaluator.CheckType(jToken, regexMatch, JTokenType.Float), false),
new TextRuleCommand("IS( NOT)? DATE(TIME)?", (jToken, regexMatch) => IsRuleEvaluator.CheckType(jToken, regexMatch, JTokenType.Date), false),
new TextRuleCommand("IS( NOT)? NULL", (jToken, regexMatch) => IsRuleEvaluator.CheckType(jToken, regexMatch, JTokenType.Null), false),
new TextRuleCommand("IS( NOT)? EMPTY", IsRuleEvaluator.CheckEmpty, true),
new TextRuleCommand("IS( NOT)?( EQUAL TO| EXACTLY)? ('.*'|\".*\"|[\\-0-9.]+)", IsRuleEvaluator.CheckEquality, false),
new TextRuleCommand("IS( NOT)? BETWEEN ([\\-0-9.]+) AND ([\\-0-9.]+)( OR( IS)? NaN)?", (jToken, regexMatch) => IsRuleEvaluator.CheckComparison(jToken, regexMatch, ComparisonType.Between), false),
new TextRuleCommand("IS( NOT)? (GREATER|MORE) THAN ([\\-0-9.]+)( OR( IS)? NaN)?", (jToken, regexMatch) => IsRuleEvaluator.CheckComparison(jToken, regexMatch, ComparisonType.GreaterThan), false),
new TextRuleCommand("IS( NOT)? (GREATER|MORE) THAN OR (EQUAL TO|EXACTLY) ([\\-0-9.]+)( OR( IS)? NaN)?", (jToken, regexMatch) => IsRuleEvaluator.CheckComparison(jToken, regexMatch, ComparisonType.GreaterThanOrEqual), false),
new TextRuleCommand("IS( NOT)? (LESS|SMALLER) THAN ([\\-0-9.]+)( OR( IS)? NaN)?", (jToken, regexMatch) => IsRuleEvaluator.CheckComparison(jToken, regexMatch, ComparisonType.LessThan), false),
new TextRuleCommand("IS( NOT)? (LESS|SMALLER) THAN OR (EQUAL TO|EXACTLY) ([\\-0-9.]+)( OR( IS)? NaN)?", (jToken, regexMatch) => IsRuleEvaluator.CheckComparison(jToken, regexMatch, ComparisonType.LessThanOrEqual), false),
new TextRuleCommand($"IS REFERENCE TO ({NamingConventions.CollectionNamePattern})", (jToken, regexMatch) => referenceRuleEvaluator.CheckValidReferenceAsync(jToken, regexMatch).Result, false),
new TextRuleCommand("HAS( NOT)? (EXACTLY )?([0-9]+) ITEMS", (jToken, regexMatch) => HasRuleEvaluator.CheckItemCount(jToken, regexMatch, ComparisonType.Equal), true),
new TextRuleCommand("HAS( NOT)? MORE THAN ([0-9]+) ITEMS", (jToken, regexMatch) => HasRuleEvaluator.CheckItemCount(jToken, regexMatch, ComparisonType.GreaterThan), true),
new TextRuleCommand("HAS( NOT)? MORE THAN OR (EQUAL TO|EXACTLY) ([0-9]+) ITEMS", (jToken, regexMatch) => HasRuleEvaluator.CheckItemCount(jToken, regexMatch, ComparisonType.GreaterThanOrEqual), true),
new TextRuleCommand("HAS( NOT)? LESS THAN ([0-9]+) ITEMS", (jToken, regexMatch) => HasRuleEvaluator.CheckItemCount(jToken, regexMatch, ComparisonType.LessThan), true),
new TextRuleCommand("HAS( NOT)? LESS THAN OR (EQUAL TO|EXACTLY) ([0-9]+) ITEMS", (jToken, regexMatch) => HasRuleEvaluator.CheckItemCount(jToken, regexMatch, ComparisonType.LessThanOrEqual), true),
new TextRuleCommand("HAS( NOT)? BETWEEN ([0-9]+) AND ([0-9]+) ITEMS", (jToken, regexMatch) => HasRuleEvaluator.CheckItemCount(jToken, regexMatch, ComparisonType.Between), true),
new TextRuleCommand("HAS( NOT)? LENGTH( EQUAL TO| EXACTLY) ([0-9]+)", (jToken, regexMatch) => HasLengthRuleEvaluator.CheckStringLength(jToken, regexMatch, ComparisonType.Equal), false),
new TextRuleCommand("HAS( NOT)? LENGTH (GREATER|MORE) THAN ([0-9]+)", (jToken, regexMatch) => HasLengthRuleEvaluator.CheckStringLength(jToken, regexMatch, ComparisonType.GreaterThan), false),
new TextRuleCommand("HAS( NOT)? LENGTH (GREATER|MORE) THAN OR (EQUAL TO|EXACTLY) ([0-9]+)", (jToken, regexMatch) => HasLengthRuleEvaluator.CheckStringLength(jToken, regexMatch, ComparisonType.GreaterThanOrEqual), false),
new TextRuleCommand("HAS( NOT)? LENGTH (LESS|SMALLER) THAN ([0-9]+)", (jToken, regexMatch) => HasLengthRuleEvaluator.CheckStringLength(jToken, regexMatch, ComparisonType.LessThan), false),
new TextRuleCommand("HAS( NOT)? LENGTH (LESS|SMALLER) THAN OR (EQUAL TO|EXACTLY) ([0-9]+)", (jToken, regexMatch) => HasLengthRuleEvaluator.CheckStringLength(jToken, regexMatch, ComparisonType.LessThanOrEqual), false),
new TextRuleCommand("HAS( NOT)? LENGTH BETWEEN ([0-9]+) AND ([0-9]+)", (jToken, regexMatch) => HasLengthRuleEvaluator.CheckStringLength(jToken, regexMatch, ComparisonType.Between), false),
new TextRuleCommand("(NOT )?MATCHES ([^\\s].*[^\\s])", (jToken, regexMatch) => MatchRuleEvaluator.CheckRegexMatch(jToken, regexMatch), false),
};
}
#pragma warning disable 1591
public DataIoController(
#pragma warning restore 1591
AuthorizationModule authorizationModule,
CollectionInformationManager collectionInformationManager,
ValidatorManager validatorManager,
SubscriptionManager subscriptionManager,
IEventLogger apiEventLogger,
IDataRouter dataRouter,
IIdPolicy idPolicy,
NewCollectionTasks newCollectionTasks)
{
this.authorizationModule = authorizationModule;
this.validatorManager = validatorManager;
this.collectionInformationManager = collectionInformationManager;
this.subscriptionManager = subscriptionManager;
this.apiEventLogger = apiEventLogger;
this.dataRouter = dataRouter;
this.idPolicy = idPolicy;
this.newCollectionTasks = newCollectionTasks;
}
public IsReferenceRuleEvaluator(IDataRouter dataRouter)
{
this.dataRouter = dataRouter;
}
public DataService(
ILoggerFactory loggerFactory,
IShardRepository shardRepository,
IDataRouter dataRouter,
IStateMachine <State> stateMachine,
NodeStateService nodeStateService,
ClusterClient clusterClient,
IOptions <ClusterOptions> clusterOptions,
IOperationCacheRepository transactionCacheRepository,
IOptions <NodeOptions> nodeOptions)
{
_nodeStateService = nodeStateService;
_nodeOptions = nodeOptions.Value;
_clusterOptions = clusterOptions.Value;
_stateMachine = stateMachine;
_writeCache = new WriteCache(transactionCacheRepository, loggerFactory.CreateLogger <WriteCache>(), _nodeOptions.PersistWriteQueue);
_logger = loggerFactory.CreateLogger <DataService <State> >();
_shardRepository = shardRepository;
_clusterClient = clusterClient;
Reader = new Reader <State>(
loggerFactory.CreateLogger <Reader <State> >(),
shardRepository,
dataRouter,
stateMachine,
nodeStateService,
clusterClient);;
Allocator = new Allocator <State>(
loggerFactory.CreateLogger <Allocator <State> >(),
shardRepository,
dataRouter,
stateMachine,
nodeStateService,
clusterClient);
Writer = new Writer <State>(loggerFactory.CreateLogger <Writer <State> >(),
shardRepository,
dataRouter,
stateMachine,
nodeStateService,
clusterClient
);
Syncer = new Syncer <State>(shardRepository, loggerFactory.CreateLogger <Syncer <State> >(), stateMachine, clusterClient, nodeStateService, Writer);
_writeTask = new Task(async() =>
{
//Before you write you should first dequeue all transactions
if (_writeCache.TransitQueue.Count() > 0)
{
_logger.LogInformation("Found transactions in transit, attempting to reapply them...");
foreach (var operationKV in _writeCache.TransitQueue.ToDictionary(entry => entry.Key, entry => entry.Value))
{
var operation = operationKV.Value;
try
{
var result = await Writer.WriteShardData(operation.Data, operation.Operation, operation.Id, operation.TransactionDate);
}
catch (Exception e)
{
_logger.LogError("Failed to apply operation " + operation.Id + " with exception " + e.Message + Environment.NewLine + e.StackTrace);
try
{
await _writeCache.CompleteOperation(operation.Id);
}
catch (Exception completionError)
{
_logger.LogError("Error removing operation from transit queue with error " + completionError.Message + Environment.NewLine
+ completionError.StackTrace + Environment.NewLine + JsonConvert.SerializeObject(operation, Formatting.Indented));
}
}
}
}
while (true)
{
try
{
var operation = await _writeCache.DequeueOperation();
if (operation != null)
{
try
{
var result = await Writer.WriteShardData(operation.Data, operation.Operation, operation.Id, operation.TransactionDate);
}
catch (Exception e)
{
_logger.LogError("Failed to write operation with exception " + e.Message + Environment.NewLine + JsonConvert.SerializeObject(operation, Formatting.Indented));
}
await _writeCache.CompleteOperation(operation.Id);
}
else
{
//Release write thread for a small amount of time
await Task.Delay(100);
}
}
catch (Exception e)
{
_logger.LogError("Encountered critical write error " + e.Message + Environment.NewLine + e.StackTrace);
}
}
});
_writeTask.Start();
TaskUtility.RestartTask(ref _indexCreationTask, async() => await CreateIndexLoop());
_allocationTask = new Task(async() => await AllocateShards());
_allocationTask.Start();
_replicaValidationChecksTask = Task.Run(async() => await CheckAllReplicas());
_objectLockWatcher = new Task(async() => await CheckLocks());
_objectLockWatcher.Start();
if (_nodeOptions.EnablePerformanceLogging)
{
var performancePrinting = new Task(() =>
{
while (true)
{
Console.Clear();
Console.WriteLine("Performance Report...");
foreach (var value in totals)
{
Console.WriteLine(value.Key + ":" + (value.Value / totalRequests));
}
Console.WriteLine("Queue:" + _writeCache.OperationsInQueue);
Task.Delay(1000);
}
});
performancePrinting.Start();
}
}
public ValidatorFactory(IDataRouter dataRouter)
{
textRuleEvaluator = new TextRuleEvaluator(dataRouter);
}
public SubscriptionManager(RdDataMongoClient rdDataMongoClient, IDataRouter dataRouter)
{
this.dataRouter = dataRouter;
subscriptionCollection = rdDataMongoClient.BackendDatabase.GetCollection <Subscription>(nameof(Subscription));
dataNotificationCollection = rdDataMongoClient.BackendDatabase.GetCollection <DataNotification>(nameof(DataNotification));
}
/// <summary>
/// Get writer preconfigured to track unsaved changes and allow subfolders creation and targeting
/// child folder named <paramref name="targetFolderName"/> of <see cref="FixtureRootRepoFolder"/>
/// </summary>
public IRepositoryWriter GetStandaloneWriter(string targetFolderName, int desiredFileSize, IDataRouter router)
{
IRepositoryManager manager = GetStandaloneRepository();
manager.Settings.StorageTransactionSettings = StorageTransactionSettings.RequireTransactions | StorageTransactionSettings.DisallowJoiningAmbientManaged;
IRepositoryFolder targetFolder = manager.RootFolder.GetDescendant(FixtureRootRepoFolder.LogicalPath, false).GetSubFolder(targetFolderName);
targetFolder.Properties.DesiredItemsPerFile = desiredFileSize;
return(GetWriter(targetFolder: targetFolder, router: router));
}
