/// <summary> /// Configures the specified property info. /// </summary> /// <param name="propertyInfo">The property info.</param> /// <param name="config">The config.</param> public void Configure(PropertyInfo propertyInfo, QueryConfiguration config) { config.Query = Query; config.IsLazy = IsLazy; config.IsRelative = IsRelative; config.InferType = InferType; base.Configure(propertyInfo, config); }
public SqlServerQueryExecutor(QueryConfiguration configuration, Akka.Serialization.Serialization serialization) : base(configuration, serialization) { CreateSnapshotTableSql = $@" IF NOT EXISTS (SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '{ configuration.SchemaName }' AND TABLE_NAME = '{configuration.SnapshotTableName}') BEGIN CREATE TABLE {configuration.FullSnapshotTableName} ( {configuration.PersistenceIdColumnName} NVARCHAR(255) NOT NULL, {configuration.SequenceNrColumnName} BIGINT NOT NULL, {configuration.TimestampColumnName} DATETIME2 NOT NULL, {configuration.ManifestColumnName} NVARCHAR(500) NOT NULL, {configuration.PayloadColumnName} VARBINARY(MAX) NOT NULL, {configuration.SerializerIdColumnName} INTEGER NULL CONSTRAINT PK_{configuration.SnapshotTableName} PRIMARY KEY ({ configuration.PersistenceIdColumnName }, {configuration.SequenceNrColumnName}) ); CREATE INDEX IX_{configuration.SnapshotTableName}_{configuration.SequenceNrColumnName} ON { configuration.FullSnapshotTableName }({configuration.SequenceNrColumnName}); CREATE INDEX IX_{configuration.SnapshotTableName}_{configuration.TimestampColumnName} ON { configuration.FullSnapshotTableName }({configuration.TimestampColumnName}); END "; InsertSnapshotSql = $@" IF ( SELECT COUNT(*) FROM {configuration.FullSnapshotTableName} WHERE {configuration.SequenceNrColumnName} = @SequenceNr AND {configuration.PersistenceIdColumnName} = @PersistenceId) > 0 UPDATE {configuration.FullSnapshotTableName} SET {configuration.PersistenceIdColumnName} = @PersistenceId, {configuration.SequenceNrColumnName} = @SequenceNr, {configuration.TimestampColumnName} = @Timestamp, {configuration.ManifestColumnName} = @Manifest, {configuration.PayloadColumnName} = @Payload, {configuration.SerializerIdColumnName} = @SerializerId WHERE {configuration.SequenceNrColumnName} = @SequenceNr AND {configuration.PersistenceIdColumnName} = @PersistenceId ELSE INSERT INTO {configuration.FullSnapshotTableName} ( {configuration.PersistenceIdColumnName}, {configuration.SequenceNrColumnName}, {configuration.TimestampColumnName}, {configuration.ManifestColumnName}, {configuration.PayloadColumnName}, {configuration.SerializerIdColumnName}) VALUES (@PersistenceId, @SequenceNr, @Timestamp, @Manifest, @Payload, @SerializerId);"; SelectSnapshotSql = $@" SELECT TOP 1 {Configuration.PersistenceIdColumnName}, {Configuration.SequenceNrColumnName}, {Configuration.TimestampColumnName}, {Configuration.ManifestColumnName}, {Configuration.PayloadColumnName}, {Configuration.SerializerIdColumnName} FROM {Configuration.FullSnapshotTableName} WHERE {Configuration.PersistenceIdColumnName} = @PersistenceId AND {Configuration.SequenceNrColumnName} <= @SequenceNr AND {Configuration.TimestampColumnName} <= @Timestamp ORDER BY {Configuration.SequenceNrColumnName} DESC"; }
public IEnumerable <Coverage> GetCoverages(TrackData trackData, GroupedQueryResults groupedQueryResults, QueryConfiguration configuration) { var fingerprintConfiguration = configuration.FingerprintConfiguration; var matches = groupedQueryResults.GetMatchesForTrackOrderedByQueryAt(trackData.TrackReference); double queryLength = groupedQueryResults.GetQueryLength(fingerprintConfiguration); if (configuration.AllowMultipleMatchesOfTheSameTrackInQuery) { var sequences = longestIncreasingTrackSequence.FindAllIncreasingTrackSequences(matches); var filtered = OverlappingRegionFilter.FilterOverlappingSequences(sequences); return(filtered.Select(matchedSequence => GetCoverage(matchedSequence, queryLength, fingerprintConfiguration.FingerprintLengthInSeconds))); } return(new List <Coverage> { GetCoverage(matches, queryLength, fingerprintConfiguration.FingerprintLengthInSeconds) }); }
public SqlServerQueryExecutor(QueryConfiguration configuration, Akka.Serialization.Serialization serialization, ITimestampProvider timestampProvider) : base(configuration, serialization, timestampProvider) { ByTagSql = $@" SELECT TOP (@Take) e.{Configuration.PersistenceIdColumnName} as PersistenceId, e.{Configuration.SequenceNrColumnName} as SequenceNr, e.{Configuration.TimestampColumnName} as Timestamp, e.{Configuration.IsDeletedColumnName} as IsDeleted, e.{Configuration.ManifestColumnName} as Manifest, e.{Configuration.PayloadColumnName} as Payload, e.{Configuration.SerializerIdColumnName} as SerializerId, e.{Configuration.OrderingColumnName} as Ordering FROM {Configuration.FullJournalTableName} e WHERE e.{Configuration.OrderingColumnName} > @Ordering AND e.{Configuration.TagsColumnName} LIKE @Tag ORDER BY {Configuration.OrderingColumnName} ASC "; CreateEventsJournalSql = $@" IF NOT EXISTS (SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '{ configuration.SchemaName }' AND TABLE_NAME = '{configuration.JournalEventsTableName}') BEGIN CREATE TABLE {configuration.FullJournalTableName} ( {configuration.OrderingColumnName} BIGINT IDENTITY(1,1) NOT NULL, {configuration.PersistenceIdColumnName} NVARCHAR(255) NOT NULL, {configuration.SequenceNrColumnName} BIGINT NOT NULL, {configuration.TimestampColumnName} BIGINT NOT NULL, {configuration.IsDeletedColumnName} BIT NOT NULL, {configuration.ManifestColumnName} NVARCHAR(500) NOT NULL, {configuration.PayloadColumnName} VARBINARY(MAX) NOT NULL, {configuration.TagsColumnName} NVARCHAR(100) NULL, {configuration.SerializerIdColumnName} INTEGER NULL, CONSTRAINT PK_{configuration.JournalEventsTableName} PRIMARY KEY ({ configuration.OrderingColumnName }), CONSTRAINT UQ_{configuration.JournalEventsTableName} UNIQUE ({ configuration.PersistenceIdColumnName }, {configuration.SequenceNrColumnName}) ); CREATE INDEX IX_{configuration.JournalEventsTableName}_{configuration.SequenceNrColumnName} ON { configuration.FullJournalTableName }({configuration.SequenceNrColumnName}); CREATE INDEX IX_{configuration.JournalEventsTableName}_{configuration.TimestampColumnName} ON { configuration.FullJournalTableName }({configuration.TimestampColumnName}); END "; CreateMetaTableSql = $@" IF NOT EXISTS (SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '{ configuration.SchemaName }' AND TABLE_NAME = '{configuration.MetaTableName}') BEGIN CREATE TABLE {configuration.FullMetaTableName} ( {configuration.PersistenceIdColumnName} NVARCHAR(255) NOT NULL, {configuration.SequenceNrColumnName} BIGINT NOT NULL, CONSTRAINT PK_{configuration.MetaTableName} PRIMARY KEY ({configuration.PersistenceIdColumnName}, { configuration.SequenceNrColumnName }) ); END "; }
/// <summary> /// Initializes a new instance of the <see cref="BatchingSqliteJournalSetup" /> class. /// </summary> /// <param name="connectionString">The connection string used to connect to the database.</param> /// <param name="maxConcurrentOperations">The maximum number of batch operations allowed to be executed at the same time.</param> /// <param name="maxBatchSize">The maximum size of single batch of operations to be executed over a single <see cref="DbConnection"/>.</param> /// <param name="maxBufferSize">The maximum size of requests stored in journal buffer.</param> /// <param name="autoInitialize"> /// If set to <c>true</c>, the journal executes all SQL scripts stored under the /// <see cref="BatchingSqlJournal{TConnection,TCommand}.Initializers"/> collection prior /// to starting executing any requests. /// </param> /// <param name="connectionTimeout">The maximum time given for executed <see cref="DbCommand"/> to complete.</param> /// <param name="isolationLevel">The isolation level of transactions used during query execution.</param> /// <param name="circuitBreakerSettings"> /// The settings used by the <see cref="CircuitBreaker"/> when for executing request batches. /// </param> /// <param name="replayFilterSettings">The settings used when replaying events from database back to the persistent actors.</param> /// <param name="namingConventions">The naming conventions used by the database to construct valid SQL statements.</param> /// <param name="defaultSerializer">The serializer used when no specific type matching can be found.</param> public BatchingSqliteJournalSetup(string connectionString, int maxConcurrentOperations, int maxBatchSize, int maxBufferSize, bool autoInitialize, TimeSpan connectionTimeout, IsolationLevel isolationLevel, CircuitBreakerSettings circuitBreakerSettings, ReplayFilterSettings replayFilterSettings, QueryConfiguration namingConventions, string defaultSerializer) : base(connectionString, maxConcurrentOperations, maxBatchSize, maxBufferSize, autoInitialize, connectionTimeout, isolationLevel, circuitBreakerSettings, replayFilterSettings, namingConventions, defaultSerializer) { }
public virtual IList <SubFingerprintData> ReadSubFingerprints(int[] hashBins, QueryConfiguration config) { return(subFingerprintDao.ReadSubFingerprints(hashBins, config.ThresholdVotes, config.Clusters).ToList()); }
public virtual ISet <SubFingerprintData> ReadSubFingerprints(IEnumerable <int[]> hashes, QueryConfiguration config) { return(subFingerprintDao.ReadSubFingerprints(hashes, config.ThresholdVotes, config.Clusters)); }
private GroupedQueryResults GetSimilaritiesUsingBatchedStrategy(IEnumerable <HashedFingerprint> queryFingerprints, QueryConfiguration configuration, DateTime relativeTo, IModelService modelService) { var hashedFingerprints = queryFingerprints as List <HashedFingerprint> ?? queryFingerprints.ToList(); var result = modelService.Query(hashedFingerprints.Select(hashedFingerprint => hashedFingerprint.HashBins), configuration); double queryLength = hashedFingerprints.QueryLength(configuration.FingerprintConfiguration); var groupedResults = new GroupedQueryResults(queryLength, relativeTo); Parallel.ForEach(hashedFingerprints, queryFingerprint => { var subFingerprints = result.Where(queryResult => QueryMath.IsCandidatePassingThresholdVotes(queryFingerprint.HashBins, queryResult.Hashes, configuration.ThresholdVotes)); foreach (var subFingerprint in subFingerprints) { double score = scoreAlgorithm.GetScore(queryFingerprint, subFingerprint, configuration); groupedResults.Add(queryFingerprint, subFingerprint, score); } }); return(groupedResults); }
public FakeDomainObjectsConfiguration(MappingLoaderConfiguration mappingLoader = null, StorageConfiguration storage = null, QueryConfiguration query = null) { _mappingLoader = mappingLoader ?? new MappingLoaderConfiguration(); _storage = storage ?? new StorageConfiguration(); _query = query ?? new QueryConfiguration(); }
public IEnumerable <SubFingerprintData> ReadSubFingerprints(IEnumerable <int[]> hashes, QueryConfiguration queryConfiguration) { var allSubs = new ConcurrentBag <SubFingerprintData>(); using var tx = databaseContext.OpenReadOnlyTransaction(); Parallel.ForEach(hashes, hashedFingerprint => { foreach (var subFingerprint in ReadSubFingerprints( hashedFingerprint, queryConfiguration.ThresholdVotes, queryConfiguration.MetaFieldsFilter, tx )) { allSubs.Add(subFingerprint); } }); return(allSubs.Distinct()); }
public QueryProvider(QueryConfiguration <TContext> configuration) { _configuration = configuration; }
public IEnumerable <SubFingerprintData> Query(Hashes hashes, QueryConfiguration config) { var queryHashes = hashes.Select(_ => _.HashBins).ToList(); return(queryHashes.Any() ? SubFingerprintDao.ReadSubFingerprints(queryHashes, config) : Enumerable.Empty <SubFingerprintData>()); }
public ThreadSafeWorkStack setWorkList() { _allConfigs = new ThreadSafeWorkStack(); getConfigs(); // cleanup from last batch - delete IEN tracking table records since IEN would be counted otherwise cleanupFromLastRun(); ISqlDao sqlDao = _sqlDao; Dictionary <String, String> lastIenTable = _sqlDao.getLastIenTable(); // grab the whole table - much better performance than a query per config/per site // loop through sites and jobs - see if job was already reported as started by database foreach (ExtractorConfiguration ec in _configurations) { QueryConfiguration qc = ec.QueryConfigurations.RootNode.Value; foreach (string sitecode in _vhaSites) { ec.SiteCode = sitecode; string currentKey = sitecode + "_" + qc.File; if (!String.IsNullOrEmpty(ec.Sites)) { bool included = false; string[] configSites = ec.Sites.Split(new char[] { ';' }); if (configSites != null && configSites.Length > 0) { foreach (string s in configSites) { if (sitecode == s) { ExtractorConfiguration cloned = ec.Clone(); cloned.BatchId = this.Report.BatchId; _allConfigs.Push(cloned); included = true; } } } if (!included) { continue; } } else { ExtractorConfiguration cloned = ec.Clone(); cloned.BatchId = this.Report.BatchId; _allConfigs.Push(cloned); } // we will give the start point to the extractor for incremental extractions so we need to fetch it ExtractorConfiguration newConfig = ec.Clone(); newConfig.BatchId = this.Report.BatchId; if (ec.ExtractMode == ExtractorMode.INCREMENTAL) { try { String lastSqlIen = "0"; if (lastIenTable.ContainsKey(currentKey)) { lastSqlIen = lastIenTable[currentKey]; // if we found this config/site key, use last IEN from last IEN table _report.addDebug("Found incremental IEN in tracking table for key: " + currentKey + " - IEN: " + lastSqlIen); } //string lastSqlIen = _sqlDao.getLastIen(sitecode, config.File); string fromConfig = newConfig.QueryConfigurations.RootNode.Value.From; if (lastSqlIen.Equals("0") && !String.IsNullOrEmpty(fromConfig)) { newConfig.StartIen = fromConfig; } else { if (String.IsNullOrEmpty(fromConfig)) // fromConfig == null || fromConfig.Equals(String.Empty)) { newConfig.StartIen = lastSqlIen; } else { decimal incIen = Convert.ToDecimal(lastSqlIen); decimal fromIen = Convert.ToDecimal(fromConfig); if (incIen > fromIen) { newConfig.StartIen = lastSqlIen; newConfig.QueryConfigurations.RootNode.Value.From = lastSqlIen; } else { // _report.addInfo(String.Format("The current configuration specified a start IEN ({0}) that is greater than the last IEN in the tracking table ({1}) - Site: {2}, File: {3}", fromConfig, lastSqlIen, sitecode, qc.File)); newConfig.StartIen = fromConfig; newConfig.QueryConfigurations.RootNode.Value.From = fromConfig; } } } // do this at the end so we don't disturb any of the above code - don't care what's in IEN tracking table for these files, always start at beginning if (String.Equals(newConfig.QueryConfigurations.RootNode.Value.File, "63") || String.Equals(newConfig.QueryConfigurations.RootNode.Value.File, "55")) // not crazy about this hard coding but at least it's just the Orchestrator handling it... { // logging.Log.LOG("Found a special config that requires a full Vista file traversal - setting start IEN to '0'"); // _report.addInfo("Found a special config that requires a full Vista file traversal - setting start IEN to '0'"); newConfig.StartIen = newConfig.QueryConfigurations.RootNode.Value.From = "0"; } if (String.Equals(newConfig.QueryConfigurations.RootNode.Value.File, "69")) { // logging.Log.LOG("Updated file 69 config - set config start IEN to " + _file69From); newConfig.StartIen = newConfig.QueryConfigurations.RootNode.Value.From = _file69From; } } catch (Exception exc) { _report.addError("Unable to retrieve the last SQL IEN for the extractor!", exc); _report.HasError = "T"; continue; } } else if (ec.ExtractMode == ExtractorMode.DIFF) { newConfig.SqlIens = setParamsForDiff(sitecode, ec); } _report.addDebug("Adding a new job to the workstack: " + newConfig.ToString()); _workStack.PushOrUpdate(newConfig); } } _report.addInfo(_workStack.Count() + " total jobs are on the work stack"); _report.addDebug(_workStack.Count() + " total jobs on the work stack"); preProcess(_configurations); _workStack.SortBySiteCode(); // we want jobs for a site to run as a group RequestHandler.getInstance().WorkStack = _workStack; return(_workStack); }
public StatefulRealtimeResultEntryAggregator(IRealtimeResultEntryFilter realtimeResultEntryFilter, QueryConfiguration queryConfiguration) { this.realtimeResultEntryFilter = realtimeResultEntryFilter; completionStrategy = new ResultEntryCompletionStrategy(queryConfiguration.PermittedGap); concatenator = new ResultEntryConcatenator(); }