Beispiel #1
0
        /// <summary>
        /// Selects one of the shard maps (either local or global) as a source of truth and brings
        /// mappings on both shard maps in sync.
        /// </summary>
        /// <param name="token">Recovery token returned from <see cref="DetectMappingDifferences(ShardLocation, string)"/>.</param>
        /// <param name="resolution">The resolution strategy to be used for resolution.</param>
        /// <remarks>
        /// Note that this method can cause unrecoverable data loss. Make sure you have taken backups or copies
        /// of your databases and only then proceed with great care.
        /// </remarks>
        public void ResolveMappingDifferences(
            RecoveryToken token,
            MappingDifferenceResolution resolution)
        {
            switch (resolution)
            {
            case MappingDifferenceResolution.KeepShardMapMapping:
                this.RestoreShardFromShardmap(token);
                break;

            case MappingDifferenceResolution.KeepShardMapping:
                this.RestoreShardMapFromShard(token);
                break;

            case MappingDifferenceResolution.Ignore:
                break;

            default:
                Debug.Fail("Unexpected value for MappingDifferenceResolution.");
                return;
            }

            this.StoreShardMaps.Remove(token);
            this.Locations.Remove(token);
            this.Inconsistencies.Remove(token);
        }
Beispiel #2
0
        /// <summary>
        /// Helper function to bring a Shard into a consistent state with a ShardMap.
        /// </summary>
        /// <param name="token">Token from DetectMappingDifferences</param>
        private void RestoreShardFromShardmap(RecoveryToken token)
        {
            IStoreShardMap ssmLocal;

            DefaultStoreShard dss = this.GetStoreShardFromToken("ResolveMappingDifferences", token, out ssmLocal);

            IStoreResults gsmMappings;

            using (IStoreOperationGlobal op = this.Manager.StoreOperationFactory.CreateGetMappingsByRangeGlobalOperation(
                       this.Manager,
                       "ResolveMappingDifferences",
                       ssmLocal,
                       dss,
                       null,
                       ShardManagementErrorCategory.Recovery,
                       false,
                       false))
            {
                gsmMappings = op.Do();
            }

            using (IStoreOperationLocal op = this.Manager.StoreOperationFactory.CreateReplaceMappingsLocalOperation(
                       this.Manager,
                       dss.Location,
                       "ResolveMappingDifferences",
                       ssmLocal,
                       dss,
                       null,
                       gsmMappings.StoreMappings))
            {
                op.Do();
            }
        }
Beispiel #3
0
        public void GetShardInfo(RecoveryToken token, out ShardMapType mapType, out string shardMapName, out ShardLocation shardLocation)
        {
            ExceptionUtils.DisallowNullArgument(token, "token");

            Tuple <IStoreShardMap, IStoreShard> shardInfoLocal;

            if (!this.StoreShardMaps.TryGetValue(token, out shardInfoLocal))
            {
                throw new ArgumentException(
                          StringUtils.FormatInvariant(
                              Errors._Recovery_InvalidRecoveryToken,
                              token),
                          "token");
            }

            mapType      = shardInfoLocal.Item1.MapType;
            shardMapName = shardInfoLocal.Item1.Name;

            if (!this.Locations.TryGetValue(token, out shardLocation))
            {
                throw new ArgumentException(
                          StringUtils.FormatInvariant(
                              Errors._Recovery_InvalidRecoveryToken,
                              token),
                          "token");
            }
        }
Beispiel #4
0
        /// <summary>
        /// Helper function to obtain a store shard object from given recovery token.
        /// </summary>
        /// <param name="operationName">Operation name.</param>
        /// <param name="token">Token from DetectMappingDifferences.</param>
        /// <param name="ssmLocal">Reference to store shard map corresponding to the token.</param>
        /// <returns>Store shard object corresponding to given token, or null if shard map is default shard map.</returns>
        private DefaultStoreShard GetStoreShardFromToken(string operationName, RecoveryToken token, out IStoreShardMap ssmLocal)
        {
            Tuple <IStoreShardMap, IStoreShard> shardInfoLocal;

            if (!this.StoreShardMaps.TryGetValue(token, out shardInfoLocal))
            {
                throw new ArgumentException(
                          StringUtils.FormatInvariant(
                              Errors._Recovery_InvalidRecoveryToken,
                              token),
                          "token");
            }

            ssmLocal = shardInfoLocal.Item1;

            IStoreShard ssLocal = shardInfoLocal.Item2;

            ShardLocation location = this.GetShardLocation(token);

            using (IStoreOperationLocal op = this.Manager.StoreOperationFactory.CreateCheckShardLocalOperation(
                       operationName,
                       this.Manager,
                       location))
            {
                op.Do();
            }

            return(new DefaultStoreShard(
                       ssLocal.Id,
                       ssLocal.Version,
                       ssLocal.ShardMapId,
                       ssLocal.Location,
                       ssLocal.Status));
        }
Beispiel #5
0
        /// <summary>
        /// Given a collection of shard locations, reconstructs the shard map manager based on information
        /// stored in the individual shards.
        /// If the information in the individual shard maps is or becomes inconsistent, behavior is undefined.
        /// No cross shard locks are taken, so if any shards become inconsistent during the execution of this
        /// method, the final state of the global shard map may be corrupt.
        /// </summary>
        /// <param name="operationName">Operation name.</param>
        /// <param name="shardLocations">Collection of shard locations.</param>
        /// <param name="resolutionStrategy">Strategy for resolving the mapping differences.</param>
        /// <param name="shardMapName">Optional name of shard map. If omitted, will attempt to recover from all shard maps present on each shard.</param>
        private void RebuildMappingsHelper(
            string operationName,
            IEnumerable <ShardLocation> shardLocations,
            MappingDifferenceResolution resolutionStrategy,
            string shardMapName = null)
        {
            Debug.Assert(shardLocations != null);

            IList <RecoveryToken> idsToProcess = new List <RecoveryToken>();

            // Collect the shard map-shard pairings to recover. Give each of these pairings a token.
            foreach (ShardLocation shardLocation in shardLocations)
            {
                IStoreResults getShardsLocalResult;

                using (IStoreOperationLocal op = this.Manager.StoreOperationFactory.CreateGetShardsLocalOperation(
                           this.Manager,
                           shardLocation,
                           operationName))
                {
                    getShardsLocalResult = op.Do();
                }

                Debug.Assert(getShardsLocalResult.Result == StoreResult.Success);

                IEnumerable <IStoreShardMap> shardMaps =
                    shardMapName == null ?
                    getShardsLocalResult.StoreShardMaps :
                    getShardsLocalResult.StoreShardMaps.Where(s => s.Name == shardMapName);

                IEnumerable <Tuple <IStoreShardMap, IStoreShard> > shardInfos =
                    shardMaps
                    .Select(sm => new Tuple <IStoreShardMap, IStoreShard>(
                                sm,
                                getShardsLocalResult.StoreShards.SingleOrDefault(s => s.ShardMapId == sm.Id)));

                foreach (Tuple <IStoreShardMap, IStoreShard> shardInfo in shardInfos)
                {
                    RecoveryToken token = new RecoveryToken();

                    idsToProcess.Add(token);

                    this.StoreShardMaps[token] = shardInfo;
                    this.Locations[token]      = shardLocation;
                }
            }

            // Recover from the shard map-shard pairing corresponding to the collected token.
            foreach (RecoveryToken token in idsToProcess)
            {
                this.ResolveMappingDifferences(token, resolutionStrategy);

                this.StoreShardMaps.Remove(token);
                this.Locations.Remove(token);
            }
        }
Beispiel #6
0
        /// <summary>
        /// Returns a dictionary of range-to-location key-value pairs. The location returned is an enumerator stating
        /// whether a given range (or point) is present only in the local shard map, only in the global shard map, or both.
        /// Ranges not contained in either shard map cannot contain differences so those ranges are not shown.
        /// </summary>
        /// <param name="token">Recovery token returned from <see cref="DetectMappingDifferences(ShardLocation, string)"/>.</param>
        /// <returns>The set of ranges and their corresponding <see cref="MappingLocation"/>.</returns>
        /// <remarks>
        /// This method assumes a previous call to <see cref="DetectMappingDifferences(ShardLocation, string)"/> that provides the recovery token parameter.
        /// The result of this method is typically used in subsequent calls to resolve inconsistencies such as
        /// <see cref="ResolveMappingDifferences"/> or <see cref="RebuildMappingsOnShard"/>.
        /// </remarks>
        public IDictionary <ShardRange, MappingLocation> GetMappingDifferences(RecoveryToken token)
        {
            ExceptionUtils.DisallowNullArgument(token, "token");

            if (!this.Inconsistencies.ContainsKey(token))
            {
                throw new ArgumentException(
                          StringUtils.FormatInvariant(
                              Errors._Recovery_InvalidRecoveryToken,
                              token),
                          "token");
            }

            return(this.Inconsistencies[token].ToDictionary(i => i.Key, i => i.Value.Location));
        }
Beispiel #7
0
        /// <summary>
        /// Returns the shard location of the local shard map processed by <see cref="DetectMappingDifferences(ShardLocation, string)"/>.
        /// </summary>
        /// <param name="token">Recovery token returned from <see cref="DetectMappingDifferences(ShardLocation, string)"/></param>
        /// <returns>Location of the shard corresponding to the set of mapping differences detected in <see cref="DetectMappingDifferences(ShardLocation, string)"/>.</returns>
        public ShardLocation GetShardLocation(RecoveryToken token)
        {
            ExceptionUtils.DisallowNullArgument(token, "token");

            ShardLocation shardLocation;

            if (!this.Locations.TryGetValue(token, out shardLocation))
            {
                throw new ArgumentException(
                          StringUtils.FormatInvariant(
                              Errors._Recovery_InvalidRecoveryToken,
                              token),
                          "token");
            }

            return(shardLocation);
        }
Beispiel #8
0
        /// <summary>
        /// Returns the shard map name of the shard map processed by <see cref="DetectMappingDifferences(ShardLocation, string)"/>.
        /// </summary>
        /// <param name="token">Recovery token returned from <see cref="DetectMappingDifferences(ShardLocation, string)"/>.</param>
        /// <returns>The name of the shard map for the given recovery token.</returns>
        public string GetShardMapName(RecoveryToken token)
        {
            ExceptionUtils.DisallowNullArgument(token, "token");

            Tuple <IStoreShardMap, IStoreShard> shardInfoLocal;

            if (!this.StoreShardMaps.TryGetValue(token, out shardInfoLocal))
            {
                throw new ArgumentException(
                          StringUtils.FormatInvariant(
                              Errors._Recovery_InvalidRecoveryToken,
                              token),
                          "token");
            }

            return(shardInfoLocal.Item1.Name);
        }
Beispiel #9
0
        private void RestoreShardMapFromShard(RecoveryToken token)
        {
            IStoreShardMap ssmLocal;

            DefaultStoreShard dss = this.GetStoreShardFromToken("ResolveMappingDifferences", token, out ssmLocal);

            IStoreResults lsmMappingsToRemove;

            using (IStoreOperationLocal op = this.Manager.StoreOperationFactory.CreateGetMappingsByRangeLocalOperation(
                       this.Manager,
                       dss.Location,
                       "ResolveMappingDifferences",
                       ssmLocal,
                       dss,
                       null,
                       false))
            {
                lsmMappingsToRemove = op.Do();
            }

            IEnumerable <IStoreMapping> gsmMappingsToAdd = lsmMappingsToRemove.StoreMappings.Select(
                mapping => new DefaultStoreMapping(
                    mapping.Id,
                    mapping.ShardMapId,
                    dss,
                    mapping.MinValue,
                    mapping.MaxValue,
                    mapping.Status,
                    default(Guid)));

            using (IStoreOperationGlobal op = this.Manager.StoreOperationFactory.CreateReplaceMappingsGlobalOperation(
                       this.Manager,
                       "ResolveMappingDifferences",
                       ssmLocal,
                       dss,
                       lsmMappingsToRemove.StoreMappings,
                       gsmMappingsToAdd))
            {
                op.Do();
            }
        }
Beispiel #10
0
        public IEnumerable <RecoveryToken> DetectMappingDifferences(ShardLocation location, string shardMapName)
        {
            ExceptionUtils.DisallowNullArgument(location, "location");

            IList <RecoveryToken> listOfTokens = new List <RecoveryToken>();

            IStoreResults getShardsLocalResult;

            using (IStoreOperationLocal op = this.Manager.StoreOperationFactory.CreateGetShardsLocalOperation(
                       this.Manager,
                       location,
                       "DetectMappingDifferences"))
            {
                getShardsLocalResult = op.Do();
            }

            Debug.Assert(getShardsLocalResult.Result == StoreResult.Success);

            IEnumerable <IStoreShardMap> shardMaps =
                shardMapName == null ?
                getShardsLocalResult.StoreShardMaps :
                getShardsLocalResult.StoreShardMaps.Where(s => s.Name == shardMapName);

            IEnumerable <Tuple <IStoreShardMap, IStoreShard> > shardInfos =
                shardMaps
                .Select(sm => new Tuple <IStoreShardMap, IStoreShard>(
                            sm,
                            getShardsLocalResult.StoreShards.SingleOrDefault(s => s.ShardMapId == sm.Id)));

            foreach (Tuple <IStoreShardMap, IStoreShard> shardInfo in shardInfos)
            {
                IStoreShardMap ssmLocal = shardInfo.Item1;
                IStoreShard    ssLocal  = shardInfo.Item2;

                RecoveryToken token = new RecoveryToken();

                listOfTokens.Add(token);
                this.StoreShardMaps[token] = shardInfo;
                this.Locations[token]      = location;

                this.Inconsistencies[token] = new Dictionary <ShardRange, MappingDifference>();

                DefaultStoreShard dss = new DefaultStoreShard(
                    ssLocal.Id,
                    ssLocal.Version,
                    ssLocal.ShardMapId,
                    ssLocal.Location,
                    ssLocal.Status);

                // First get all local mappings.
                IStoreResults lsmMappings;

                using (IStoreOperationLocal op = this.Manager.StoreOperationFactory.CreateGetMappingsByRangeLocalOperation(
                           this.Manager,
                           location,
                           "DetectMappingDifferences",
                           ssmLocal,
                           dss,
                           null,
                           true))
                {
                    lsmMappings = op.Do();

                    if (lsmMappings.Result == StoreResult.ShardMapDoesNotExist)
                    {
                        // The shard needs to be re-attached. We are ignoring these errors in
                        // DetectMappingDifferences, since corruption is more profound than
                        // just inconsistent mappings.
                        // Alternatively, this shard belongs to a different shard map manager.
                        // Either way, we can't do anything about it here.
                        continue;
                    }
                }

                // Next build up a set of relevant global mappings.
                // This is the union of those mappings that are associated with this local shard
                // and those mappings which intersect with mappings found in the local shard.
                // We will partition these mappings based on ranges.
                IDictionary <ShardRange, IStoreMapping> relevantGsmMappings = new Dictionary <ShardRange, IStoreMapping>();

                IStoreResults gsmMappingsByMap;

                using (IStoreOperationGlobal op = this.Manager.StoreOperationFactory.CreateGetMappingsByRangeGlobalOperation(
                           this.Manager,
                           "DetectMappingDifferences",
                           ssmLocal,
                           dss,
                           null,
                           ShardManagementErrorCategory.Recovery,
                           false,
                           true /* ignore failures */))
                {
                    gsmMappingsByMap = op.Do();
                }

                if (gsmMappingsByMap.Result == StoreResult.ShardMapDoesNotExist)
                {
                    // The shard map is not properly attached to this GSM.
                    // This is beyond what we can handle resolving mappings.
                    continue;
                }

                foreach (IStoreMapping gsmMapping in gsmMappingsByMap.StoreMappings)
                {
                    ShardKey min = ShardKey.FromRawValue(ssmLocal.KeyType, gsmMapping.MinValue);

                    ShardKey max = null;

                    switch (ssmLocal.MapType)
                    {
                    case ShardMapType.Range:
                        max = ShardKey.FromRawValue(ssmLocal.KeyType, gsmMapping.MaxValue);
                        break;

                    default:
                        Debug.Assert(ssmLocal.MapType == ShardMapType.List);
                        max = ShardKey.FromRawValue(ssmLocal.KeyType, gsmMapping.MinValue).GetNextKey();
                        break;
                    }

                    ShardRange range = new ShardRange(min, max);

                    relevantGsmMappings[range] = gsmMapping;
                }

                // Next, for each of the mappings in lsmMappings, we need to augment
                // the gsmMappingsByMap by intersecting ranges.
                foreach (IStoreMapping lsmMapping in lsmMappings.StoreMappings)
                {
                    ShardKey min = ShardKey.FromRawValue(ssmLocal.KeyType, lsmMapping.MinValue);

                    IStoreResults gsmMappingsByRange;

                    if (ssmLocal.MapType == ShardMapType.Range)
                    {
                        ShardKey max = ShardKey.FromRawValue(ssmLocal.KeyType, lsmMapping.MaxValue);

                        ShardRange range = new ShardRange(min, max);

                        using (IStoreOperationGlobal op = this.Manager.StoreOperationFactory.CreateGetMappingsByRangeGlobalOperation(
                                   this.Manager,
                                   "DetectMappingDifferences",
                                   ssmLocal,
                                   null,
                                   range,
                                   ShardManagementErrorCategory.Recovery,
                                   false,
                                   true /* ignore failures */))
                        {
                            gsmMappingsByRange = op.Do();
                        }

                        if (gsmMappingsByRange.Result == StoreResult.ShardMapDoesNotExist)
                        {
                            // The shard was not properly attached.
                            // This is more than we can deal with in mapping resolution.
                            continue;
                        }
                    }
                    else
                    {
                        Debug.Assert(ssmLocal.MapType == ShardMapType.List);
                        using (IStoreOperationGlobal op = this.Manager.StoreOperationFactory.CreateFindMappingByKeyGlobalOperation(
                                   this.Manager,
                                   "DetectMappingDifferences",
                                   ssmLocal,
                                   min,
                                   CacheStoreMappingUpdatePolicy.OverwriteExisting,
                                   ShardManagementErrorCategory.Recovery,
                                   false,
                                   true /* ignore failures */))
                        {
                            gsmMappingsByRange = op.Do();

                            if (gsmMappingsByRange.Result == StoreResult.MappingNotFoundForKey ||
                                gsmMappingsByRange.Result == StoreResult.ShardMapDoesNotExist)
                            {
                                // * No intersections being found is fine. Skip to the next mapping.
                                // * The shard was not properly attached.
                                // This is more than we can deal with in mapping resolution.
                                continue;
                            }
                        }
                    }

                    foreach (IStoreMapping gsmMapping in gsmMappingsByRange.StoreMappings)
                    {
                        ShardKey retrievedMin = ShardKey.FromRawValue(ssmLocal.KeyType, gsmMapping.MinValue);

                        ShardRange retrievedRange = null;

                        switch (ssmLocal.MapType)
                        {
                        case ShardMapType.Range:
                            ShardKey retrievedMax = ShardKey.FromRawValue(ssmLocal.KeyType, gsmMapping.MaxValue);
                            retrievedRange = new ShardRange(retrievedMin, retrievedMax);
                            break;

                        default:
                            Debug.Assert(ssmLocal.MapType == ShardMapType.List);
                            retrievedMax   = ShardKey.FromRawValue(ssmLocal.KeyType, gsmMapping.MinValue).GetNextKey();
                            retrievedRange = new ShardRange(retrievedMin, retrievedMax);
                            break;
                        }

                        relevantGsmMappings[retrievedRange] = gsmMapping;
                    }
                }

                IList <MappingComparisonResult> comparisonResults = null;

                switch (ssmLocal.MapType)
                {
                case ShardMapType.Range:
                    comparisonResults = MappingComparisonUtils.CompareRangeMappings(
                        ssmLocal,
                        relevantGsmMappings.Values,
                        lsmMappings.StoreMappings);
                    break;

                default:
                    Debug.Assert(ssmLocal.MapType == ShardMapType.List);
                    comparisonResults = MappingComparisonUtils.ComparePointMappings(
                        ssmLocal,
                        relevantGsmMappings.Values,
                        lsmMappings.StoreMappings);
                    break;
                }

                // Now we have 2 sets of mappings. Each submapping generated from this function is
                //  1.) in the GSM only: report.
                //  2.) in the LSM only: report.
                //  3.) in both but with different version number: report.
                //  4.) in both with the same version number: skip.
                foreach (MappingComparisonResult r in comparisonResults)
                {
                    switch (r.MappingLocation)
                    {
                    case MappingLocation.MappingInShardMapOnly:
                    case MappingLocation.MappingInShardOnly:
                        break;

                    default:
                        Debug.Assert(r.MappingLocation == MappingLocation.MappingInShardMapAndShard);

                        if (r.ShardMapManagerMapping.Id == r.ShardMapping.Id)
                        {
                            // No conflict found, skip to the next range.
                            continue;
                        }
                        break;
                    }

                    // Store the inconsistency for later reporting.
                    this.Inconsistencies[token][r.Range] = new MappingDifference(
                        type: MappingDifferenceType.Range,
                        location: r.MappingLocation,
                        shardMap: r.ShardMap,
                        mappingForShard: r.ShardMapping,
                        mappingForShardMap: r.ShardMapManagerMapping);
                }
            }

            return(listOfTokens);
        }
Beispiel #11
0
        public void RebuildMappingsOnShard(RecoveryToken token, IEnumerable <ShardRange> ranges)
        {
            ExceptionUtils.DisallowNullArgument(token, "token");
            ExceptionUtils.DisallowNullArgument(ranges, "ranges");

            ShardLocation location = this.GetShardLocation(token);

            if (!this.Inconsistencies.ContainsKey(token))
            {
                throw new ArgumentException(
                          StringUtils.FormatInvariant(
                              Errors._Recovery_InvalidRecoveryToken,
                              token),
                          "token");
            }

            IStoreShardMap ssmLocal;

            DefaultStoreShard dss = this.GetStoreShardFromToken("RebuildMappingsOnShard", token, out ssmLocal);

            IList <IStoreMapping> mappingsToAdd = new List <IStoreMapping>();

            // Determine the ranges we want to keep based on input keeps list.
            foreach (ShardRange range in ranges)
            {
                MappingDifference difference;

                if (!this.Inconsistencies[token].TryGetValue(range, out difference))
                {
                    throw new ArgumentException(
                              StringUtils.FormatInvariant(
                                  Errors._Recovery_InvalidRebuildShardSpecification,
                                  range,
                                  location),
                              "ranges");
                }

                // The storeMapping we will use as a template.
                IStoreMapping storeMappingTemplate = difference.Location == MappingLocation.MappingInShardMapOnly ?
                                                     difference.MappingForShardMap :
                                                     difference.MappingForShard;

                IStoreMapping storeMappingToAdd = new DefaultStoreMapping(
                    Guid.NewGuid(),
                    storeMappingTemplate.ShardMapId,
                    dss,
                    range.Low.RawValue,
                    range.High.RawValue,
                    storeMappingTemplate.Status,
                    default(Guid)
                    );

                mappingsToAdd.Add(storeMappingToAdd);
            }

            using (IStoreOperationLocal op = this.Manager.StoreOperationFactory.CreateReplaceMappingsLocalOperation(
                       this.Manager,
                       location,
                       "RebuildMappingsOnShard",
                       ssmLocal,
                       dss,
                       this.Inconsistencies[token].Keys,
                       mappingsToAdd))
            {
                op.Do();
            }

            this.StoreShardMaps.Remove(token);
            this.Locations.Remove(token);
            this.Inconsistencies.Remove(token);
        }