/// <summary>
/// Configures the specified property info.
/// </summary>
/// <param name="propertyInfo">The property info.</param>
/// <param name="config">The config.</param>
public void Configure(PropertyInfo propertyInfo, QueryConfiguration config)
{
config.Query = Query;
config.IsLazy = IsLazy;
config.IsRelative = IsRelative;
config.InferType = InferType;
base.Configure(propertyInfo, config);
}
public SqlServerQueryExecutor(QueryConfiguration configuration, Akka.Serialization.Serialization serialization)
: base(configuration, serialization)
{
CreateSnapshotTableSql = $@"
IF NOT EXISTS (SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '{
configuration.SchemaName
}' AND TABLE_NAME = '{configuration.SnapshotTableName}')
BEGIN
CREATE TABLE {configuration.FullSnapshotTableName} (
{configuration.PersistenceIdColumnName} NVARCHAR(255) NOT NULL,
{configuration.SequenceNrColumnName} BIGINT NOT NULL,
{configuration.TimestampColumnName} DATETIME2 NOT NULL,
{configuration.ManifestColumnName} NVARCHAR(500) NOT NULL,
{configuration.PayloadColumnName} VARBINARY(MAX) NOT NULL,
{configuration.SerializerIdColumnName} INTEGER NULL
CONSTRAINT PK_{configuration.SnapshotTableName} PRIMARY KEY ({
configuration.PersistenceIdColumnName
}, {configuration.SequenceNrColumnName})
);
CREATE INDEX IX_{configuration.SnapshotTableName}_{configuration.SequenceNrColumnName} ON {
configuration.FullSnapshotTableName
}({configuration.SequenceNrColumnName});
CREATE INDEX IX_{configuration.SnapshotTableName}_{configuration.TimestampColumnName} ON {
configuration.FullSnapshotTableName
}({configuration.TimestampColumnName});
END
";
InsertSnapshotSql = $@"
IF (
SELECT COUNT(*)
FROM {configuration.FullSnapshotTableName}
WHERE {configuration.SequenceNrColumnName} = @SequenceNr
AND {configuration.PersistenceIdColumnName} = @PersistenceId) > 0
UPDATE {configuration.FullSnapshotTableName}
SET
{configuration.PersistenceIdColumnName} = @PersistenceId,
{configuration.SequenceNrColumnName} = @SequenceNr,
{configuration.TimestampColumnName} = @Timestamp,
{configuration.ManifestColumnName} = @Manifest,
{configuration.PayloadColumnName} = @Payload,
{configuration.SerializerIdColumnName} = @SerializerId
WHERE {configuration.SequenceNrColumnName} = @SequenceNr
AND {configuration.PersistenceIdColumnName} = @PersistenceId ELSE
INSERT INTO {configuration.FullSnapshotTableName} (
{configuration.PersistenceIdColumnName},
{configuration.SequenceNrColumnName},
{configuration.TimestampColumnName},
{configuration.ManifestColumnName},
{configuration.PayloadColumnName},
{configuration.SerializerIdColumnName})
VALUES (@PersistenceId, @SequenceNr, @Timestamp, @Manifest, @Payload, @SerializerId);";
SelectSnapshotSql = $@"
SELECT TOP 1 {Configuration.PersistenceIdColumnName},
{Configuration.SequenceNrColumnName},
{Configuration.TimestampColumnName},
{Configuration.ManifestColumnName},
{Configuration.PayloadColumnName},
{Configuration.SerializerIdColumnName}
FROM {Configuration.FullSnapshotTableName}
WHERE {Configuration.PersistenceIdColumnName} = @PersistenceId
AND {Configuration.SequenceNrColumnName} <= @SequenceNr
AND {Configuration.TimestampColumnName} <= @Timestamp
ORDER BY {Configuration.SequenceNrColumnName} DESC";
}
public IEnumerable <Coverage> GetCoverages(TrackData trackData, GroupedQueryResults groupedQueryResults, QueryConfiguration configuration)
{
var fingerprintConfiguration = configuration.FingerprintConfiguration;
var matches = groupedQueryResults.GetMatchesForTrackOrderedByQueryAt(trackData.TrackReference);
double queryLength = groupedQueryResults.GetQueryLength(fingerprintConfiguration);
if (configuration.AllowMultipleMatchesOfTheSameTrackInQuery)
{
var sequences = longestIncreasingTrackSequence.FindAllIncreasingTrackSequences(matches);
var filtered = OverlappingRegionFilter.FilterOverlappingSequences(sequences);
return(filtered.Select(matchedSequence => GetCoverage(matchedSequence, queryLength, fingerprintConfiguration.FingerprintLengthInSeconds)));
}
return(new List <Coverage>
{
GetCoverage(matches, queryLength, fingerprintConfiguration.FingerprintLengthInSeconds)
});
}
public SqlServerQueryExecutor(QueryConfiguration configuration, Akka.Serialization.Serialization serialization,
ITimestampProvider timestampProvider)
: base(configuration, serialization, timestampProvider)
{
ByTagSql = $@"
SELECT TOP (@Take)
e.{Configuration.PersistenceIdColumnName} as PersistenceId,
e.{Configuration.SequenceNrColumnName} as SequenceNr,
e.{Configuration.TimestampColumnName} as Timestamp,
e.{Configuration.IsDeletedColumnName} as IsDeleted,
e.{Configuration.ManifestColumnName} as Manifest,
e.{Configuration.PayloadColumnName} as Payload,
e.{Configuration.SerializerIdColumnName} as SerializerId,
e.{Configuration.OrderingColumnName} as Ordering
FROM {Configuration.FullJournalTableName} e
WHERE e.{Configuration.OrderingColumnName} > @Ordering AND e.{Configuration.TagsColumnName} LIKE @Tag
ORDER BY {Configuration.OrderingColumnName} ASC
";
CreateEventsJournalSql = $@"
IF NOT EXISTS (SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '{
configuration.SchemaName
}' AND TABLE_NAME = '{configuration.JournalEventsTableName}')
BEGIN
CREATE TABLE {configuration.FullJournalTableName} (
{configuration.OrderingColumnName} BIGINT IDENTITY(1,1) NOT NULL,
{configuration.PersistenceIdColumnName} NVARCHAR(255) NOT NULL,
{configuration.SequenceNrColumnName} BIGINT NOT NULL,
{configuration.TimestampColumnName} BIGINT NOT NULL,
{configuration.IsDeletedColumnName} BIT NOT NULL,
{configuration.ManifestColumnName} NVARCHAR(500) NOT NULL,
{configuration.PayloadColumnName} VARBINARY(MAX) NOT NULL,
{configuration.TagsColumnName} NVARCHAR(100) NULL,
{configuration.SerializerIdColumnName} INTEGER NULL,
CONSTRAINT PK_{configuration.JournalEventsTableName} PRIMARY KEY ({
configuration.OrderingColumnName
}),
CONSTRAINT UQ_{configuration.JournalEventsTableName} UNIQUE ({
configuration.PersistenceIdColumnName
}, {configuration.SequenceNrColumnName})
);
CREATE INDEX IX_{configuration.JournalEventsTableName}_{configuration.SequenceNrColumnName} ON {
configuration.FullJournalTableName
}({configuration.SequenceNrColumnName});
CREATE INDEX IX_{configuration.JournalEventsTableName}_{configuration.TimestampColumnName} ON {
configuration.FullJournalTableName
}({configuration.TimestampColumnName});
END
";
CreateMetaTableSql = $@"
IF NOT EXISTS (SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '{
configuration.SchemaName
}' AND TABLE_NAME = '{configuration.MetaTableName}')
BEGIN
CREATE TABLE {configuration.FullMetaTableName} (
{configuration.PersistenceIdColumnName} NVARCHAR(255) NOT NULL,
{configuration.SequenceNrColumnName} BIGINT NOT NULL,
CONSTRAINT PK_{configuration.MetaTableName} PRIMARY KEY ({configuration.PersistenceIdColumnName}, {
configuration.SequenceNrColumnName
})
);
END
";
}
/// <summary>
/// Initializes a new instance of the <see cref="BatchingSqliteJournalSetup" /> class.
/// </summary>
/// <param name="connectionString">The connection string used to connect to the database.</param>
/// <param name="maxConcurrentOperations">The maximum number of batch operations allowed to be executed at the same time.</param>
/// <param name="maxBatchSize">The maximum size of single batch of operations to be executed over a single <see cref="DbConnection"/>.</param>
/// <param name="maxBufferSize">The maximum size of requests stored in journal buffer.</param>
/// <param name="autoInitialize">
/// If set to <c>true</c>, the journal executes all SQL scripts stored under the
/// <see cref="BatchingSqlJournal{TConnection,TCommand}.Initializers"/> collection prior
/// to starting executing any requests.
/// </param>
/// <param name="connectionTimeout">The maximum time given for executed <see cref="DbCommand"/> to complete.</param>
/// <param name="isolationLevel">The isolation level of transactions used during query execution.</param>
/// <param name="circuitBreakerSettings">
/// The settings used by the <see cref="CircuitBreaker"/> when for executing request batches.
/// </param>
/// <param name="replayFilterSettings">The settings used when replaying events from database back to the persistent actors.</param>
/// <param name="namingConventions">The naming conventions used by the database to construct valid SQL statements.</param>
/// <param name="defaultSerializer">The serializer used when no specific type matching can be found.</param>
public BatchingSqliteJournalSetup(string connectionString, int maxConcurrentOperations, int maxBatchSize, int maxBufferSize, bool autoInitialize,
TimeSpan connectionTimeout, IsolationLevel isolationLevel, CircuitBreakerSettings circuitBreakerSettings, ReplayFilterSettings replayFilterSettings, QueryConfiguration namingConventions, string defaultSerializer)
: base(connectionString, maxConcurrentOperations, maxBatchSize, maxBufferSize, autoInitialize, connectionTimeout, isolationLevel, circuitBreakerSettings, replayFilterSettings, namingConventions, defaultSerializer)
{
}
public virtual IList <SubFingerprintData> ReadSubFingerprints(int[] hashBins, QueryConfiguration config)
{
return(subFingerprintDao.ReadSubFingerprints(hashBins, config.ThresholdVotes, config.Clusters).ToList());
}
public virtual ISet <SubFingerprintData> ReadSubFingerprints(IEnumerable <int[]> hashes, QueryConfiguration config)
{
return(subFingerprintDao.ReadSubFingerprints(hashes, config.ThresholdVotes, config.Clusters));
}
private GroupedQueryResults GetSimilaritiesUsingBatchedStrategy(IEnumerable <HashedFingerprint> queryFingerprints, QueryConfiguration configuration, DateTime relativeTo, IModelService modelService)
{
var hashedFingerprints = queryFingerprints as List <HashedFingerprint> ?? queryFingerprints.ToList();
var result = modelService.Query(hashedFingerprints.Select(hashedFingerprint => hashedFingerprint.HashBins), configuration);
double queryLength = hashedFingerprints.QueryLength(configuration.FingerprintConfiguration);
var groupedResults = new GroupedQueryResults(queryLength, relativeTo);
Parallel.ForEach(hashedFingerprints, queryFingerprint =>
{
var subFingerprints = result.Where(queryResult => QueryMath.IsCandidatePassingThresholdVotes(queryFingerprint.HashBins, queryResult.Hashes, configuration.ThresholdVotes));
foreach (var subFingerprint in subFingerprints)
{
double score = scoreAlgorithm.GetScore(queryFingerprint, subFingerprint, configuration);
groupedResults.Add(queryFingerprint, subFingerprint, score);
}
});
return(groupedResults);
}
public FakeDomainObjectsConfiguration(MappingLoaderConfiguration mappingLoader = null, StorageConfiguration storage = null, QueryConfiguration query = null)
{
_mappingLoader = mappingLoader ?? new MappingLoaderConfiguration();
_storage = storage ?? new StorageConfiguration();
_query = query ?? new QueryConfiguration();
}
public IEnumerable <SubFingerprintData> ReadSubFingerprints(IEnumerable <int[]> hashes, QueryConfiguration queryConfiguration)
{
var allSubs = new ConcurrentBag <SubFingerprintData>();
using var tx = databaseContext.OpenReadOnlyTransaction();
Parallel.ForEach(hashes, hashedFingerprint =>
{
foreach (var subFingerprint in ReadSubFingerprints(
hashedFingerprint,
queryConfiguration.ThresholdVotes,
queryConfiguration.MetaFieldsFilter,
tx
))
{
allSubs.Add(subFingerprint);
}
});
return(allSubs.Distinct());
}
public QueryProvider(QueryConfiguration <TContext> configuration)
{
_configuration = configuration;
}
public IEnumerable <SubFingerprintData> Query(Hashes hashes, QueryConfiguration config)
{
var queryHashes = hashes.Select(_ => _.HashBins).ToList();
return(queryHashes.Any() ? SubFingerprintDao.ReadSubFingerprints(queryHashes, config) : Enumerable.Empty <SubFingerprintData>());
}
public ThreadSafeWorkStack setWorkList()
{
_allConfigs = new ThreadSafeWorkStack();
getConfigs();
// cleanup from last batch - delete IEN tracking table records since IEN would be counted otherwise
cleanupFromLastRun();
ISqlDao sqlDao = _sqlDao;
Dictionary <String, String> lastIenTable = _sqlDao.getLastIenTable(); // grab the whole table - much better performance than a query per config/per site
// loop through sites and jobs - see if job was already reported as started by database
foreach (ExtractorConfiguration ec in _configurations)
{
QueryConfiguration qc = ec.QueryConfigurations.RootNode.Value;
foreach (string sitecode in _vhaSites)
{
ec.SiteCode = sitecode;
string currentKey = sitecode + "_" + qc.File;
if (!String.IsNullOrEmpty(ec.Sites))
{
bool included = false;
string[] configSites = ec.Sites.Split(new char[] { ';' });
if (configSites != null && configSites.Length > 0)
{
foreach (string s in configSites)
{
if (sitecode == s)
{
ExtractorConfiguration cloned = ec.Clone();
cloned.BatchId = this.Report.BatchId;
_allConfigs.Push(cloned);
included = true;
}
}
}
if (!included)
{
continue;
}
}
else
{
ExtractorConfiguration cloned = ec.Clone();
cloned.BatchId = this.Report.BatchId;
_allConfigs.Push(cloned);
}
// we will give the start point to the extractor for incremental extractions so we need to fetch it
ExtractorConfiguration newConfig = ec.Clone();
newConfig.BatchId = this.Report.BatchId;
if (ec.ExtractMode == ExtractorMode.INCREMENTAL)
{
try
{
String lastSqlIen = "0";
if (lastIenTable.ContainsKey(currentKey))
{
lastSqlIen = lastIenTable[currentKey]; // if we found this config/site key, use last IEN from last IEN table
_report.addDebug("Found incremental IEN in tracking table for key: " + currentKey + " - IEN: " + lastSqlIen);
}
//string lastSqlIen = _sqlDao.getLastIen(sitecode, config.File);
string fromConfig = newConfig.QueryConfigurations.RootNode.Value.From;
if (lastSqlIen.Equals("0") && !String.IsNullOrEmpty(fromConfig))
{
newConfig.StartIen = fromConfig;
}
else
{
if (String.IsNullOrEmpty(fromConfig)) // fromConfig == null || fromConfig.Equals(String.Empty))
{
newConfig.StartIen = lastSqlIen;
}
else
{
decimal incIen = Convert.ToDecimal(lastSqlIen);
decimal fromIen = Convert.ToDecimal(fromConfig);
if (incIen > fromIen)
{
newConfig.StartIen = lastSqlIen;
newConfig.QueryConfigurations.RootNode.Value.From = lastSqlIen;
}
else
{
// _report.addInfo(String.Format("The current configuration specified a start IEN ({0}) that is greater than the last IEN in the tracking table ({1}) - Site: {2}, File: {3}", fromConfig, lastSqlIen, sitecode, qc.File));
newConfig.StartIen = fromConfig;
newConfig.QueryConfigurations.RootNode.Value.From = fromConfig;
}
}
}
// do this at the end so we don't disturb any of the above code - don't care what's in IEN tracking table for these files, always start at beginning
if (String.Equals(newConfig.QueryConfigurations.RootNode.Value.File, "63") ||
String.Equals(newConfig.QueryConfigurations.RootNode.Value.File, "55")) // not crazy about this hard coding but at least it's just the Orchestrator handling it...
{
// logging.Log.LOG("Found a special config that requires a full Vista file traversal - setting start IEN to '0'");
// _report.addInfo("Found a special config that requires a full Vista file traversal - setting start IEN to '0'");
newConfig.StartIen = newConfig.QueryConfigurations.RootNode.Value.From = "0";
}
if (String.Equals(newConfig.QueryConfigurations.RootNode.Value.File, "69"))
{
// logging.Log.LOG("Updated file 69 config - set config start IEN to " + _file69From);
newConfig.StartIen = newConfig.QueryConfigurations.RootNode.Value.From = _file69From;
}
}
catch (Exception exc)
{
_report.addError("Unable to retrieve the last SQL IEN for the extractor!", exc);
_report.HasError = "T";
continue;
}
}
else if (ec.ExtractMode == ExtractorMode.DIFF)
{
newConfig.SqlIens = setParamsForDiff(sitecode, ec);
}
_report.addDebug("Adding a new job to the workstack: " + newConfig.ToString());
_workStack.PushOrUpdate(newConfig);
}
}
_report.addInfo(_workStack.Count() + " total jobs are on the work stack");
_report.addDebug(_workStack.Count() + " total jobs on the work stack");
preProcess(_configurations);
_workStack.SortBySiteCode(); // we want jobs for a site to run as a group
RequestHandler.getInstance().WorkStack = _workStack;
return(_workStack);
}
public StatefulRealtimeResultEntryAggregator(IRealtimeResultEntryFilter realtimeResultEntryFilter, QueryConfiguration queryConfiguration)
{
this.realtimeResultEntryFilter = realtimeResultEntryFilter;
completionStrategy = new ResultEntryCompletionStrategy(queryConfiguration.PermittedGap);
concatenator = new ResultEntryConcatenator();
}