Ejemplo n.º 1
0
        /// <summary>
        /// Helper function to bring a Shard into a consistent state with a ShardMap.
        /// </summary>
        /// <param name="token">Token from DetectMappingDifferences</param>
        private void RestoreShardFromShardmap(RecoveryToken token)
        {
            IStoreShardMap ssmLocal;

            DefaultStoreShard dss = this.GetStoreShardFromToken("ResolveMappingDifferences", token, out ssmLocal);

            IStoreResults gsmMappings;

            using (IStoreOperationGlobal op = this.Manager.StoreOperationFactory.CreateGetMappingsByRangeGlobalOperation(
                       this.Manager,
                       "ResolveMappingDifferences",
                       ssmLocal,
                       dss,
                       null,
                       ShardManagementErrorCategory.Recovery,
                       false,
                       false))
            {
                gsmMappings = op.Do();
            }

            using (IStoreOperationLocal op = this.Manager.StoreOperationFactory.CreateReplaceMappingsLocalOperation(
                       this.Manager,
                       dss.Location,
                       "ResolveMappingDifferences",
                       ssmLocal,
                       dss,
                       null,
                       gsmMappings.StoreMappings))
            {
                op.Do();
            }
        }
        /// <summary>
        /// Tries to fetch the <see cref="SchemaInfo"/> with the given <see cref="ShardMap"/> name without
        /// raising any exception if data doesn't exist.
        /// </summary>
        /// <param name="shardMapName">The name of the <see cref="ShardMap"/> whose <see cref="SchemaInfo"/>
        /// will be fetched</param>
        /// <param name="schemaInfo">The <see cref="SchemaInfo"/> that was fetched or null if retrieval failed</param>
        /// <returns>true if schema info exists with given name, false otherwise.</returns>
        public bool TryGet(string shardMapName, out SchemaInfo schemaInfo)
        {
            ExceptionUtils.DisallowNullOrEmptyStringArgument(shardMapName, "shardMapName");

            schemaInfo = null;

            IStoreResults result;

            using (IStoreOperationGlobal op = this.Manager.StoreOperationFactory.CreateFindShardingSchemaInfoGlobalOperation(
                       this.Manager,
                       "TryGet",
                       shardMapName))
            {
                result = op.Do();
            }

            if (result.Result == StoreResult.SchemaInfoNameDoesNotExist)
            {
                return(false);
            }

            schemaInfo = result.StoreSchemaInfoCollection
                         .Select(si => SerializationHelper.DeserializeXmlData <SchemaInfo>(si.ShardingSchemaInfo))
                         .Single();

            return(true);
        }
 /// <summary>
 /// Upgrades store hosting GSM.
 /// </summary>
 /// <param name="targetVersion">Target version for store to upgrade to.</param>
 private void UpgradeStoreGlobal(Version targetVersion)
 {
     using (IStoreOperationGlobal op = this.StoreOperationFactory.CreateUpgradeStoreGlobalOperation(this, "UpgradeStoreGlobal", targetVersion))
     {
         op.Do();
     }
 }
 /// <summary>
 /// Removes a shard map from global shard map.
 /// </summary>
 /// <param name="ssm">Shard map to remove.</param>
 private void RemoveShardMapFromStore(IStoreShardMap ssm)
 {
     using (IStoreOperationGlobal op = this.StoreOperationFactory.CreateRemoveShardMapGlobalOperation(this, "DeleteShardMap", ssm))
     {
         op.Do();
     }
 }
 /// <summary>
 /// Adds a shard to global shard map.
 /// </summary>
 /// <param name="operationName">Operation name, useful for diagnostics.</param>
 /// <param name="ssm">Storage representation of shard map object.</param>
 private void AddShardMapToStore(string operationName, IStoreShardMap ssm)
 {
     using (IStoreOperationGlobal op = this.StoreOperationFactory.CreateAddShardMapGlobalOperation(this, operationName, ssm))
     {
         op.Do();
     }
 }
        /// <summary>
        /// Fetches the <see cref="SchemaInfo"/> stored with the supplied <see cref="ShardMap"/> name.
        /// </summary>
        /// <param name="shardMapName">The name of the <see cref="ShardMap"/> to get.</param>
        /// <returns>SchemaInfo object.</returns>
        public SchemaInfo Get(string shardMapName)
        {
            ExceptionUtils.DisallowNullOrEmptyStringArgument(shardMapName, "shardMapName");

            IStoreResults result;

            using (IStoreOperationGlobal op = this.Manager.StoreOperationFactory.CreateFindShardingSchemaInfoGlobalOperation(
                       this.Manager,
                       "Get",
                       shardMapName))
            {
                result = op.Do();
            }

            if (result.Result == StoreResult.SchemaInfoNameDoesNotExist)
            {
                throw new SchemaInfoException(
                          SchemaInfoErrorCode.SchemaInfoNameDoesNotExist,
                          Errors._Store_SchemaInfo_NameDoesNotExist,
                          "Get",
                          shardMapName);
            }

            return(result.StoreSchemaInfoCollection
                   .Select(si => SerializationHelper.DeserializeXmlData <SchemaInfo>(si.ShardingSchemaInfo))
                   .Single());
        }
Ejemplo n.º 7
0
        private static ShardMapManager GetSqlShardMapManager(
            string connectionString,
            SqlCredential secureCredential,
            ShardMapManagerLoadPolicy loadPolicy,
            RetryBehavior retryBehavior,
            EventHandler <RetryingEventArgs> retryEventHandler,
            bool throwOnFailure)
        {
            Debug.Assert(connectionString != null);
            Debug.Assert(retryBehavior != null);

            SqlShardMapManagerCredentials credentials = new SqlShardMapManagerCredentials(connectionString, secureCredential);

            StoreOperationFactory storeOperationFactory = new StoreOperationFactory();

            IStoreResults result;

            TransientFaultHandling.RetryPolicy retryPolicy = new TransientFaultHandling.RetryPolicy(
                new ShardManagementTransientErrorDetectionStrategy(retryBehavior),
                RetryPolicy.DefaultRetryPolicy.GetRetryStrategy());

            EventHandler <TransientFaultHandling.RetryingEventArgs> handler = (sender, args) =>
            {
                if (retryEventHandler != null)
                {
                    retryEventHandler(sender, new RetryingEventArgs(args));
                }
            };

            try
            {
                retryPolicy.Retrying += handler;

                using (IStoreOperationGlobal op = storeOperationFactory.CreateGetShardMapManagerGlobalOperation(
                           credentials,
                           retryPolicy,
                           throwOnFailure ? "GetSqlShardMapManager" : "TryGetSqlShardMapManager",
                           throwOnFailure))
                {
                    result = op.Do();
                }
            }
            finally
            {
                retryPolicy.Retrying -= handler;
            }

            return(result.Result == StoreResult.Success ?
                   new ShardMapManager(
                       credentials,
                       new SqlStoreConnectionFactory(),
                       storeOperationFactory,
                       new CacheStore(),
                       loadPolicy,
                       RetryPolicy.DefaultRetryPolicy,
                       retryBehavior,
                       retryEventHandler) :
                   null);
        }
Ejemplo n.º 8
0
        /// <summary>
        /// Looks up the key value and returns the corresponding mapping.
        /// </summary>
        /// <typeparam name="TMapping">Mapping type.</typeparam>
        /// <typeparam name="TKey">Key type.</typeparam>
        /// <param name="key">Input key value.</param>
        /// <param name="useCache">Whether to use cache for lookups.</param>
        /// <param name="constructMapping">Delegate to construct a mapping object.</param>
        /// <param name="errorCategory">Category under which errors must be thrown.</param>
        /// <returns>Mapping that contains the key value.</returns>
        protected TMapping Lookup <TMapping, TKey>(
            TKey key,
            bool useCache,
            Func <ShardMapManager, ShardMap, IStoreMapping, TMapping> constructMapping,
            ShardManagementErrorCategory errorCategory)
            where TMapping : class, IShardProvider
        {
            ShardKey sk = new ShardKey(ShardKey.ShardKeyTypeFromType(typeof(TKey)), key);

            if (useCache)
            {
                ICacheStoreMapping cachedMapping = this.Manager.Cache.LookupMappingByKey(this.ShardMap.StoreShardMap, sk);

                if (cachedMapping != null)
                {
                    return(constructMapping(this.Manager, this.ShardMap, cachedMapping.Mapping));
                }
            }

            // Cache-miss, find mapping for given key in GSM.
            TMapping m = null;

            IStoreResults gsmResult;

            Stopwatch stopwatch = Stopwatch.StartNew();

            using (IStoreOperationGlobal op = this.Manager.StoreOperationFactory.CreateFindMappingByKeyGlobalOperation(
                       this.Manager,
                       "Lookup",
                       this.ShardMap.StoreShardMap,
                       sk,
                       CacheStoreMappingUpdatePolicy.OverwriteExisting,
                       errorCategory,
                       true,
                       false))
            {
                gsmResult = op.Do();
            }

            stopwatch.Stop();

            Tracer.TraceVerbose(
                TraceSourceConstants.ComponentNames.BaseShardMapper,
                "Lookup",
                "Lookup key from GSM complete; Key type : {0}; Result: {1}; Duration: {2}",
                typeof(TKey),
                gsmResult.Result,
                stopwatch.Elapsed);

            // If we could not locate the mapping, we return null and do nothing here.
            if (gsmResult.Result != StoreResult.MappingNotFoundForKey)
            {
                return(gsmResult.StoreMappings.Select(sm => constructMapping(this.Manager, this.ShardMap, sm)).Single());
            }

            return(m);
        }
        /// <summary>
        /// Loads the shard map manager and shards from Store.
        /// </summary>
        private void LoadFromStore()
        {
            this.Cache.Clear();

            using (IStoreOperationGlobal op = this.StoreOperationFactory.CreateLoadShardMapManagerGlobalOperation(this, "GetShardMapManager"))
            {
                op.Do();
            }
        }
        /// <summary>
        /// Get distinct locations for the shard map manager from store.
        /// </summary>
        /// <returns>Distinct locations from shard map manager.</returns>
        private IEnumerable <ShardLocation> GetDistinctShardLocationsFromStore()
        {
            IStoreResults result;

            using (IStoreOperationGlobal op = this.StoreOperationFactory.CreateGetDistinctShardLocationsGlobalOperation(this, "GetDistinctShardLocations"))
            {
                result = op.Do();
            }

            return(result.StoreLocations
                   .Select(sl => sl.Location));
        }
        /// <summary>
        /// Obtains all ShardMaps associated with the shard map manager.
        /// </summary>
        /// <returns>Collection of shard maps associated with the shard map manager.</returns>
        private IEnumerable <ShardMap> GetShardMapsFromStore()
        {
            IStoreResults result;

            using (IStoreOperationGlobal op = this.StoreOperationFactory.CreateGetShardMapsGlobalOperation(this, "GetShardMaps"))
            {
                result = op.Do();
            }

            return(result.StoreShardMaps
                   .Select(ssm => ShardMapUtils.CreateShardMapFromStoreShardMap(this, ssm)));
        }
        /// <summary>
        /// Removes the <see cref="SchemaInfo"/> with the given <see cref="ShardMap"/> name.
        /// </summary>
        /// <param name="shardMapName">The name of the <see cref="ShardMap"/> whose <see cref="SchemaInfo"/>
        /// will be removed</param>
        public void Remove(string shardMapName)
        {
            ExceptionUtils.DisallowNullOrEmptyStringArgument(shardMapName, "shardMapName");

            using (IStoreOperationGlobal op = this.Manager.StoreOperationFactory.CreateRemoveShardingSchemaInfoGlobalOperation(
                       this.Manager,
                       "Remove",
                       shardMapName))
            {
                op.Do();
            }
        }
Ejemplo n.º 13
0
        /// <summary>
        /// Detaches the given shard from the shard map manager. Mappings pointing to the
        /// shard to be deleted will automatically be removed by this method.
        /// </summary>
        /// <param name="location">Location of the shard being detached.</param>
        /// <param name="shardMapName">Optional string to filter on shard map name.</param>
        /// <remarks>
        /// Note that this method can cause unrecoverable data loss. Make sure you have taken backups or copies
        /// of your databases and only then proceed with great care.
        /// </remarks>
        public void DetachShard(ShardLocation location, string shardMapName)
        {
            ExceptionUtils.DisallowNullArgument(location, "location");

            using (IStoreOperationGlobal op = this.Manager.StoreOperationFactory.CreateDetachShardGlobalOperation(
                       this.Manager,
                       "DetachShard",
                       location,
                       shardMapName))
            {
                op.Do();
            }
        }
        /// <summary>
        /// Finds shard map with given name in global shard map.
        /// </summary>
        /// <param name="operationName">Operation name, useful for diagnostics.</param>
        /// <param name="shardMapName">Name of shard map to search.</param>
        /// <returns>Shard map corresponding to given Id.</returns>
        private ShardMap LookupShardMapByNameInStore(string operationName, string shardMapName)
        {
            IStoreResults result;

            using (IStoreOperationGlobal op = this.StoreOperationFactory.CreateFindShardMapByNameGlobalOperation(this, operationName, shardMapName))
            {
                result = op.Do();
            }

            return(result.StoreShardMaps
                   .Select(ssm => ShardMapUtils.CreateShardMapFromStoreShardMap(this, ssm))
                   .SingleOrDefault());
        }
Ejemplo n.º 15
0
        /// <summary>
        /// Gets all shards for a shard map.
        /// </summary>
        /// <returns>All the shards belonging to the shard map.</returns>
        internal IEnumerable <Shard> GetShards()
        {
            IStoreResults result;

            using (IStoreOperationGlobal op = this.Manager.StoreOperationFactory.CreateGetShardsGlobalOperation(
                       "GetShards",
                       this.Manager,
                       this.ShardMap.StoreShardMap))
            {
                result = op.Do();
            }

            return(result.StoreShards
                   .Select(ss => new Shard(this.Manager, this.ShardMap, ss)));
        }
Ejemplo n.º 16
0
        /// <summary>
        /// Asynchronously finds the mapping in store for OpenConnectionForKey operation.
        /// </summary>
        /// <param name="sk">Key to find.</param>
        /// <param name="policy">Cache update policy.</param>
        /// <param name="errorCategory">Error category.</param>
        /// <returns>Task with the Mapping corresponding to the given key if found as the result.</returns>
        private async Task <IStoreMapping> LookupMappingForOpenConnectionForKeyAsync(
            ShardKey sk,
            CacheStoreMappingUpdatePolicy policy,
            ShardManagementErrorCategory errorCategory)
        {
            IStoreResults gsmResult;

            Stopwatch stopwatch = Stopwatch.StartNew();

            using (IStoreOperationGlobal op = this.Manager.StoreOperationFactory.CreateFindMappingByKeyGlobalOperation(
                       this.Manager,
                       "Lookup",
                       this.ShardMap.StoreShardMap,
                       sk,
                       policy,
                       errorCategory,
                       true,
                       false))
            {
                gsmResult = await op.DoAsync().ConfigureAwait(false);
            }

            stopwatch.Stop();

            Tracer.TraceVerbose(
                TraceSourceConstants.ComponentNames.BaseShardMapper,
                "LookupMappingForOpenConnectionForKeyAsync",
                "Lookup key from GSM complete; Key type : {0}; Result: {1}; Duration: {2}",
                sk.DataType,
                gsmResult.Result,
                stopwatch.Elapsed);

            // If we could not locate the mapping, we throw.
            if (gsmResult.Result == StoreResult.MappingNotFoundForKey)
            {
                throw new ShardManagementException(
                          errorCategory,
                          ShardManagementErrorCode.MappingNotFoundForKey,
                          Errors._Store_ShardMapper_MappingNotFoundForKeyGlobal,
                          this.ShardMap.Name,
                          StoreOperationRequestBuilder.SpFindShardMappingByKeyGlobal,
                          "LookupMappingForOpenConnectionForKeyAsync");
            }
            else
            {
                return(gsmResult.StoreMappings.Single());
            }
        }
Ejemplo n.º 17
0
        /// <summary>
        /// Gets all the mappings that exist within given range.
        /// </summary>
        /// <param name="range">Optional range value, if null, we cover everything.</param>
        /// <param name="shard">Optional shard parameter, if null, we cover all shards.</param>
        /// <param name="constructMapping">Delegate to construct a mapping object.</param>
        /// <param name="errorCategory">Category under which errors will be posted.</param>
        /// <param name="mappingType">Name of mapping type.</param>
        /// <returns>Read-only collection of mappings that overlap with given range.</returns>
        protected IReadOnlyList <TMapping> GetMappingsForRange <TMapping, TKey>(
            Range <TKey> range,
            Shard shard,
            Func <ShardMapManager, ShardMap, IStoreMapping, TMapping> constructMapping,
            ShardManagementErrorCategory errorCategory,
            string mappingType)
            where TMapping : class
        {
            ShardRange sr = null;

            if (shard != null)
            {
                ExceptionUtils.EnsureShardBelongsToShardMap(
                    this.Manager,
                    this.ShardMap,
                    shard,
                    "GetMappings",
                    mappingType);
            }

            if (range != null)
            {
                sr = new ShardRange(
                    new ShardKey(ShardKey.ShardKeyTypeFromType(typeof(TKey)), range.Low),
                    new ShardKey(ShardKey.ShardKeyTypeFromType(typeof(TKey)), range.HighIsMax ? null : (object)range.High));
            }

            IStoreResults result;

            using (IStoreOperationGlobal op = this.Manager.StoreOperationFactory.CreateGetMappingsByRangeGlobalOperation(
                       this.Manager,
                       "GetMappingsForRange",
                       this.ShardMap.StoreShardMap,
                       shard != null ? shard.StoreShard : null,
                       sr,
                       errorCategory,
                       true, // Always cache.
                       false))
            {
                result = op.Do();
            }

            return(result.StoreMappings
                   .Select(sm => constructMapping(this.Manager, this.ShardMap, sm))
                   .ToList()
                   .AsReadOnly());
        }
        /// <summary>
        /// Replaces the <see cref="SchemaInfo"/> with the given <see cref="ShardMap"/> name.
        /// </summary>
        /// <param name="shardMapName">The name of the <see cref="ShardMap"/> whose <see cref="SchemaInfo"/> will be replaced.</param>
        /// <param name="schemaInfo">Sharding schema information.</param>
        public void Replace(string shardMapName, SchemaInfo schemaInfo)
        {
            ExceptionUtils.DisallowNullOrEmptyStringArgument(shardMapName, "shardMapName");
            ExceptionUtils.DisallowNullArgument <SchemaInfo>(schemaInfo, "schemaInfo");

            DefaultStoreSchemaInfo dssi = new DefaultStoreSchemaInfo(
                shardMapName,
                SerializationHelper.SerializeXmlData <SchemaInfo>(schemaInfo));

            using (IStoreOperationGlobal op = this.Manager.StoreOperationFactory.CreateUpdateShardingSchemaInfoGlobalOperation(
                       this.Manager,
                       "Replace",
                       dssi))
            {
                op.Do();
            }
        }
Ejemplo n.º 19
0
        /// <summary>
        /// Gets the lock owner of a mapping.
        /// </summary>
        /// <param name="mapping">The mapping</param>
        /// <param name="errorCategory">Error category to use for the store operation</param>
        /// <returns>Lock owner for the mapping.</returns>
        internal Guid GetLockOwnerForMapping <TMapping>(TMapping mapping, ShardManagementErrorCategory errorCategory) where TMapping : class, IShardProvider, IMappingInfoProvider
        {
            this.EnsureMappingBelongsToShardMap <TMapping>(mapping, "LookupLockOwner", "mapping");

            IStoreResults result;

            using (IStoreOperationGlobal op = this.Manager.StoreOperationFactory.CreateFindMappingByIdGlobalOperation(
                       this.Manager,
                       "LookupLockOwner",
                       this.ShardMap.StoreShardMap,
                       mapping.StoreMapping,
                       errorCategory))
            {
                result = op.Do();
            }

            return(result.StoreMappings.Single().LockOwnerId);
        }
Ejemplo n.º 20
0
        /// <summary>
        /// Gets shard object based on given location.
        /// </summary>
        /// <param name="location">Input location.</param>
        /// <returns>Shard belonging to ShardMap.</returns>
        internal Shard GetShardByLocation(ShardLocation location)
        {
            Debug.Assert(location != null);

            IStoreResults result;

            using (IStoreOperationGlobal op = this.Manager.StoreOperationFactory.CreateFindShardByLocationGlobalOperation(
                       this.Manager,
                       "GetShardByLocation",
                       this.ShardMap.StoreShardMap,
                       location))
            {
                result = op.Do();
            }

            return(result.StoreShards
                   .Select(ss => new Shard(this.Manager, this.ShardMap, ss)).SingleOrDefault());
        }
        /// <summary>
        /// Returns an enumerator that iterates through the <see cref="SchemaInfoCollection"/>.
        /// </summary>
        /// <returns>Enumerator of key-value pairs of name and <see cref="SchemaInfo"/> objects.</returns>
        public IEnumerator <KeyValuePair <string, SchemaInfo> > GetEnumerator()
        {
            IStoreResults result;

            using (IStoreOperationGlobal op = this.Manager.StoreOperationFactory.CreateGetShardingSchemaInfosGlobalOperation(
                       this.Manager,
                       "GetEnumerator"))
            {
                result = op.Do();
            }

            Dictionary <string, SchemaInfo> mdCollection = new Dictionary <string, SchemaInfo>();

            foreach (IStoreSchemaInfo ssi in result.StoreSchemaInfoCollection)
            {
                mdCollection.Add(ssi.Name, SerializationHelper.DeserializeXmlData <SchemaInfo>(ssi.ShardingSchemaInfo));
            }

            return(mdCollection.GetEnumerator());
        }
Ejemplo n.º 22
0
        private void RestoreShardMapFromShard(RecoveryToken token)
        {
            IStoreShardMap ssmLocal;

            DefaultStoreShard dss = this.GetStoreShardFromToken("ResolveMappingDifferences", token, out ssmLocal);

            IStoreResults lsmMappingsToRemove;

            using (IStoreOperationLocal op = this.Manager.StoreOperationFactory.CreateGetMappingsByRangeLocalOperation(
                       this.Manager,
                       dss.Location,
                       "ResolveMappingDifferences",
                       ssmLocal,
                       dss,
                       null,
                       false))
            {
                lsmMappingsToRemove = op.Do();
            }

            IEnumerable <IStoreMapping> gsmMappingsToAdd = lsmMappingsToRemove.StoreMappings.Select(
                mapping => new DefaultStoreMapping(
                    mapping.Id,
                    mapping.ShardMapId,
                    dss,
                    mapping.MinValue,
                    mapping.MaxValue,
                    mapping.Status,
                    default(Guid)));

            using (IStoreOperationGlobal op = this.Manager.StoreOperationFactory.CreateReplaceMappingsGlobalOperation(
                       this.Manager,
                       "ResolveMappingDifferences",
                       ssmLocal,
                       dss,
                       lsmMappingsToRemove.StoreMappings,
                       gsmMappingsToAdd))
            {
                op.Do();
            }
        }
Ejemplo n.º 23
0
        internal void LockOrUnlockMappings <TMapping>(TMapping mapping, Guid lockOwnerId, LockOwnerIdOpType lockOwnerIdOpType, ShardManagementErrorCategory errorCategory) where TMapping : class, IShardProvider, IMappingInfoProvider
        {
            string operationName = lockOwnerIdOpType == LockOwnerIdOpType.Lock ? "Lock" : "UnLock";

            if (lockOwnerIdOpType != LockOwnerIdOpType.UnlockAllMappingsForId && lockOwnerIdOpType != LockOwnerIdOpType.UnlockAllMappings)
            {
                this.EnsureMappingBelongsToShardMap <TMapping>(mapping, operationName, "mapping");

                if (lockOwnerIdOpType == LockOwnerIdOpType.Lock &&
                    lockOwnerId == MappingLockToken.ForceUnlock.LockOwnerId)
                {
                    throw new ArgumentException(
                              StringUtils.FormatInvariant(
                                  Errors._ShardMapping_LockIdNotSupported,
                                  mapping.ShardInfo.Location,
                                  this.ShardMap.Name,
                                  lockOwnerId),
                              "lockOwnerId");
                }
            }
            else
            {
                Debug.Assert(mapping == null);
            }

            using (IStoreOperationGlobal op = this.Manager.StoreOperationFactory.CreateLockOrUnLockMappingsGlobalOperation(
                       this.Manager,
                       operationName,
                       this.ShardMap.StoreShardMap,
                       mapping != null ? mapping.StoreMapping : null,
                       lockOwnerId,
                       lockOwnerIdOpType,
                       errorCategory))
            {
                op.Do();
            }
        }
Ejemplo n.º 24
0
        public IEnumerable <RecoveryToken> DetectMappingDifferences(ShardLocation location, string shardMapName)
        {
            ExceptionUtils.DisallowNullArgument(location, "location");

            IList <RecoveryToken> listOfTokens = new List <RecoveryToken>();

            IStoreResults getShardsLocalResult;

            using (IStoreOperationLocal op = this.Manager.StoreOperationFactory.CreateGetShardsLocalOperation(
                       this.Manager,
                       location,
                       "DetectMappingDifferences"))
            {
                getShardsLocalResult = op.Do();
            }

            Debug.Assert(getShardsLocalResult.Result == StoreResult.Success);

            IEnumerable <IStoreShardMap> shardMaps =
                shardMapName == null ?
                getShardsLocalResult.StoreShardMaps :
                getShardsLocalResult.StoreShardMaps.Where(s => s.Name == shardMapName);

            IEnumerable <Tuple <IStoreShardMap, IStoreShard> > shardInfos =
                shardMaps
                .Select(sm => new Tuple <IStoreShardMap, IStoreShard>(
                            sm,
                            getShardsLocalResult.StoreShards.SingleOrDefault(s => s.ShardMapId == sm.Id)));

            foreach (Tuple <IStoreShardMap, IStoreShard> shardInfo in shardInfos)
            {
                IStoreShardMap ssmLocal = shardInfo.Item1;
                IStoreShard    ssLocal  = shardInfo.Item2;

                RecoveryToken token = new RecoveryToken();

                listOfTokens.Add(token);
                this.StoreShardMaps[token] = shardInfo;
                this.Locations[token]      = location;

                this.Inconsistencies[token] = new Dictionary <ShardRange, MappingDifference>();

                DefaultStoreShard dss = new DefaultStoreShard(
                    ssLocal.Id,
                    ssLocal.Version,
                    ssLocal.ShardMapId,
                    ssLocal.Location,
                    ssLocal.Status);

                // First get all local mappings.
                IStoreResults lsmMappings;

                using (IStoreOperationLocal op = this.Manager.StoreOperationFactory.CreateGetMappingsByRangeLocalOperation(
                           this.Manager,
                           location,
                           "DetectMappingDifferences",
                           ssmLocal,
                           dss,
                           null,
                           true))
                {
                    lsmMappings = op.Do();

                    if (lsmMappings.Result == StoreResult.ShardMapDoesNotExist)
                    {
                        // The shard needs to be re-attached. We are ignoring these errors in
                        // DetectMappingDifferences, since corruption is more profound than
                        // just inconsistent mappings.
                        // Alternatively, this shard belongs to a different shard map manager.
                        // Either way, we can't do anything about it here.
                        continue;
                    }
                }

                // Next build up a set of relevant global mappings.
                // This is the union of those mappings that are associated with this local shard
                // and those mappings which intersect with mappings found in the local shard.
                // We will partition these mappings based on ranges.
                IDictionary <ShardRange, IStoreMapping> relevantGsmMappings = new Dictionary <ShardRange, IStoreMapping>();

                IStoreResults gsmMappingsByMap;

                using (IStoreOperationGlobal op = this.Manager.StoreOperationFactory.CreateGetMappingsByRangeGlobalOperation(
                           this.Manager,
                           "DetectMappingDifferences",
                           ssmLocal,
                           dss,
                           null,
                           ShardManagementErrorCategory.Recovery,
                           false,
                           true /* ignore failures */))
                {
                    gsmMappingsByMap = op.Do();
                }

                if (gsmMappingsByMap.Result == StoreResult.ShardMapDoesNotExist)
                {
                    // The shard map is not properly attached to this GSM.
                    // This is beyond what we can handle resolving mappings.
                    continue;
                }

                foreach (IStoreMapping gsmMapping in gsmMappingsByMap.StoreMappings)
                {
                    ShardKey min = ShardKey.FromRawValue(ssmLocal.KeyType, gsmMapping.MinValue);

                    ShardKey max = null;

                    switch (ssmLocal.MapType)
                    {
                    case ShardMapType.Range:
                        max = ShardKey.FromRawValue(ssmLocal.KeyType, gsmMapping.MaxValue);
                        break;

                    default:
                        Debug.Assert(ssmLocal.MapType == ShardMapType.List);
                        max = ShardKey.FromRawValue(ssmLocal.KeyType, gsmMapping.MinValue).GetNextKey();
                        break;
                    }

                    ShardRange range = new ShardRange(min, max);

                    relevantGsmMappings[range] = gsmMapping;
                }

                // Next, for each of the mappings in lsmMappings, we need to augment
                // the gsmMappingsByMap by intersecting ranges.
                foreach (IStoreMapping lsmMapping in lsmMappings.StoreMappings)
                {
                    ShardKey min = ShardKey.FromRawValue(ssmLocal.KeyType, lsmMapping.MinValue);

                    IStoreResults gsmMappingsByRange;

                    if (ssmLocal.MapType == ShardMapType.Range)
                    {
                        ShardKey max = ShardKey.FromRawValue(ssmLocal.KeyType, lsmMapping.MaxValue);

                        ShardRange range = new ShardRange(min, max);

                        using (IStoreOperationGlobal op = this.Manager.StoreOperationFactory.CreateGetMappingsByRangeGlobalOperation(
                                   this.Manager,
                                   "DetectMappingDifferences",
                                   ssmLocal,
                                   null,
                                   range,
                                   ShardManagementErrorCategory.Recovery,
                                   false,
                                   true /* ignore failures */))
                        {
                            gsmMappingsByRange = op.Do();
                        }

                        if (gsmMappingsByRange.Result == StoreResult.ShardMapDoesNotExist)
                        {
                            // The shard was not properly attached.
                            // This is more than we can deal with in mapping resolution.
                            continue;
                        }
                    }
                    else
                    {
                        Debug.Assert(ssmLocal.MapType == ShardMapType.List);
                        using (IStoreOperationGlobal op = this.Manager.StoreOperationFactory.CreateFindMappingByKeyGlobalOperation(
                                   this.Manager,
                                   "DetectMappingDifferences",
                                   ssmLocal,
                                   min,
                                   CacheStoreMappingUpdatePolicy.OverwriteExisting,
                                   ShardManagementErrorCategory.Recovery,
                                   false,
                                   true /* ignore failures */))
                        {
                            gsmMappingsByRange = op.Do();

                            if (gsmMappingsByRange.Result == StoreResult.MappingNotFoundForKey ||
                                gsmMappingsByRange.Result == StoreResult.ShardMapDoesNotExist)
                            {
                                // * No intersections being found is fine. Skip to the next mapping.
                                // * The shard was not properly attached.
                                // This is more than we can deal with in mapping resolution.
                                continue;
                            }
                        }
                    }

                    foreach (IStoreMapping gsmMapping in gsmMappingsByRange.StoreMappings)
                    {
                        ShardKey retrievedMin = ShardKey.FromRawValue(ssmLocal.KeyType, gsmMapping.MinValue);

                        ShardRange retrievedRange = null;

                        switch (ssmLocal.MapType)
                        {
                        case ShardMapType.Range:
                            ShardKey retrievedMax = ShardKey.FromRawValue(ssmLocal.KeyType, gsmMapping.MaxValue);
                            retrievedRange = new ShardRange(retrievedMin, retrievedMax);
                            break;

                        default:
                            Debug.Assert(ssmLocal.MapType == ShardMapType.List);
                            retrievedMax   = ShardKey.FromRawValue(ssmLocal.KeyType, gsmMapping.MinValue).GetNextKey();
                            retrievedRange = new ShardRange(retrievedMin, retrievedMax);
                            break;
                        }

                        relevantGsmMappings[retrievedRange] = gsmMapping;
                    }
                }

                IList <MappingComparisonResult> comparisonResults = null;

                switch (ssmLocal.MapType)
                {
                case ShardMapType.Range:
                    comparisonResults = MappingComparisonUtils.CompareRangeMappings(
                        ssmLocal,
                        relevantGsmMappings.Values,
                        lsmMappings.StoreMappings);
                    break;

                default:
                    Debug.Assert(ssmLocal.MapType == ShardMapType.List);
                    comparisonResults = MappingComparisonUtils.ComparePointMappings(
                        ssmLocal,
                        relevantGsmMappings.Values,
                        lsmMappings.StoreMappings);
                    break;
                }

                // Now we have 2 sets of mappings. Each submapping generated from this function is
                //  1.) in the GSM only: report.
                //  2.) in the LSM only: report.
                //  3.) in both but with different version number: report.
                //  4.) in both with the same version number: skip.
                foreach (MappingComparisonResult r in comparisonResults)
                {
                    switch (r.MappingLocation)
                    {
                    case MappingLocation.MappingInShardMapOnly:
                    case MappingLocation.MappingInShardOnly:
                        break;

                    default:
                        Debug.Assert(r.MappingLocation == MappingLocation.MappingInShardMapAndShard);

                        if (r.ShardMapManagerMapping.Id == r.ShardMapping.Id)
                        {
                            // No conflict found, skip to the next range.
                            continue;
                        }
                        break;
                    }

                    // Store the inconsistency for later reporting.
                    this.Inconsistencies[token][r.Range] = new MappingDifference(
                        type: MappingDifferenceType.Range,
                        location: r.MappingLocation,
                        shardMap: r.ShardMap,
                        mappingForShard: r.ShardMapping,
                        mappingForShardMap: r.ShardMapManagerMapping);
                }
            }

            return(listOfTokens);
        }
Ejemplo n.º 25
0
        /// <summary>
        /// Looks up the key value and returns the corresponding mapping.
        /// </summary>
        /// <typeparam name="TMapping">Mapping type.</typeparam>
        /// <typeparam name="TKey">Key type.</typeparam>
        /// <param name="key">Input key value.</param>
        /// <param name="lookupOptions">Whether to use cache and/or storage for lookups.</param>
        /// <param name="constructMapping">Delegate to construct a mapping object.</param>
        /// <param name="errorCategory">Category under which errors must be thrown.</param>
        /// <returns>Mapping that contains the key value.</returns>
        protected TMapping Lookup <TMapping, TKey>(
            TKey key,
            LookupOptions lookupOptions,
            Func <ShardMapManager, ShardMap, IStoreMapping, TMapping> constructMapping,
            ShardManagementErrorCategory errorCategory)
            where TMapping : class, IShardProvider
        {
            ShardKey sk = new ShardKey(ShardKey.ShardKeyTypeFromType(typeof(TKey)), key);

            // Try to lookup in cache. Note the interation with cache removal later in this method.
            bool tryLookupInCache = lookupOptions.HasFlag(LookupOptions.LookupInCache);

            if (tryLookupInCache)
            {
                ICacheStoreMapping cachedMapping = this.Manager.Cache.LookupMappingByKey(this.ShardMap.StoreShardMap, sk);
                if (cachedMapping != null)
                {
                    return(constructMapping(this.Manager, this.ShardMap, cachedMapping.Mapping));
                }
            }

            // Cache-miss (or didn't use cache), find mapping for given key in GSM.
            if (lookupOptions.HasFlag(LookupOptions.LookupInStore))
            {
                IStoreResults gsmResult;

                Stopwatch stopwatch = Stopwatch.StartNew();

                using (IStoreOperationGlobal op = this.Manager.StoreOperationFactory.CreateFindMappingByKeyGlobalOperation(
                           this.Manager,
                           "Lookup",
                           this.ShardMap.StoreShardMap,
                           sk,
                           CacheStoreMappingUpdatePolicy.OverwriteExisting,
                           errorCategory,
                           true,
                           false))
                {
                    gsmResult = op.Do();
                }

                stopwatch.Stop();

                Tracer.TraceVerbose(
                    TraceSourceConstants.ComponentNames.BaseShardMapper,
                    "Lookup",
                    "Lookup key from GSM complete; Key type : {0}; Result: {1}; Duration: {2}",
                    typeof(TKey),
                    gsmResult.Result,
                    stopwatch.Elapsed);

                // If mapping was found, return it.
                if (gsmResult.Result != StoreResult.MappingNotFoundForKey)
                {
                    return(gsmResult.StoreMappings.Select(sm => constructMapping(this.Manager, this.ShardMap, sm)).Single());
                }

                // If we could not locate the mapping, then we might need to update the cache to remove it.
                //
                // Only do this if we didn't already try to return the mapping from the cache (since, if it was found,
                // we would have already returned it earlier).
                if (!tryLookupInCache)
                {
                    ICacheStoreMapping cachedMapping =
                        this.Manager.Cache.LookupMappingByKey(this.ShardMap.StoreShardMap, sk);
                    if (cachedMapping != null)
                    {
                        this.Manager.Cache.DeleteMapping(cachedMapping.Mapping);
                    }
                }
            }

            // Mapping not found - return null
            return(null);
        }