private void AddPointerShards(DataElasticityEntities context, ShardSetConfig shardSetConfig)
        {
            foreach (var ps in shardSetConfig.Shards)
            {
                if (ps.PointerShardID == -1)
                {
                    AddShardToShardSet(ps, shardSetConfig);
                }
                else
                {
                    ModifyShard(ps);
                }
            }

            var deleteShards =
                context.Shards.Where(p => p.ShardSetID == shardSetConfig.ShardSetID);

            foreach (var ps in deleteShards)
            {
                if (shardSetConfig.Shards.Count(x => x.PointerShardID == ps.ShardID) == 0)
                {
                    context.Shards.Remove(ps);
                }
            }
        }
 private static void CompleteShardMapPublishingRequest(ShardSetConfig config,
                                                       ShardMapPublishingRequest shardMapPublishingRequest)
 {
     //todo: this is saving a whole'nother config rathr thanjust incrementing the "current" config map in force
     config.CurrentPublishedShardMapID = shardMapPublishingRequest.NewShardMapID;
     config.Save();
 }
        private static void ProcessShardMapPublishingRequest(ShardMapPublishingRequest shardMapPublishingRequest,
                                                             ShardSetConfig shardSetConfig)
        {
            var shardSetName             = shardMapPublishingRequest.ShardSetName;
            var shardSetConnectionDriver = GetShardSetConnectionDriver(shardSetName);

            // to do - may need to page
            var shardlets = shardSetConnectionDriver.GetShardlets(shardSetName);

            foreach (var shardlet in shardlets)
            {
                // skip if shardlet is pinned
                if (shardlet.Pinned)
                {
                    continue;
                }

                // get the matching range shard for this shardlet based on the range map
                var rangeShard =
                    shardSetConfig.ShardMap.Shards
                    .FirstOrDefault(rs => rs.HighDistributionKey > shardlet.DistributionKey);

                // skip if we cannot find a range.
                if (rangeShard == null)
                {
                    continue;
                }

                // determine if the shardlet needs to be moved
                if (rangeShard.ServerInstanceName == shardlet.ServerInstanceName &&
                    rangeShard.Catalog == shardlet.Catalog)
                {
                    continue;
                }

                var request =
                    new ShardletMoveRequest
                {
                    DestinationCatalog            = rangeShard.Catalog,
                    DestinationServerInstanceName = shardlet.ServerInstanceName,
                    SourceCatalog            = shardlet.Catalog,
                    SourceServerInstanceName = shardlet.ServerInstanceName,
                    DistributionKey          = shardlet.DistributionKey,
                    ShardingKey  = shardlet.ShardingKey,
                    ShardSetName = shardlet.ShardSetName,
                    DeleteOnMove = true,
                    Pin          = false
                };

                request.Save();
            }

            // todo: do this save in parallel
            //var parallelOptions = GetParallelOptions();

            //Parallel.ForEach(shardletMoveRequests, parallelOptions,
            //    shardletMoveRequest => shardletMoveRequest.Save());
        }
        /// <summary>
        ///     Check the shard set action queues for the next request and, if found,
        ///     process the associated action off the queue.
        /// </summary>
        /// <param name="uniqueProcessID">A unique process identifier pass to the table driver to identify the caller.</param>
        public virtual void CheckAndProcessQueue(Guid uniqueProcessID)
        {
            SendQueueProcessingEvent("Queue Processing Started");

            //Get the next queue request in order of execution
            var request = GetNextQueueRequest();

            // identity the request type and call the appropriate method to handle the request
            while (request != null)
            {
                switch (request.GetType().Name)
                {
                case _shardCreationRequest:
                {
                    SendQueueProcessingEvent(_shardCreationRequest, request);
                    ShardSetConfig.HandleShardCreationRequest(request as ShardCreationRequest, this);
                    break;
                }

                case _shardMapPublishingRequest:
                {
                    SendQueueProcessingEvent(_shardMapPublishingRequest, request);
                    ShardSetConfig.HandleShardMapPublishingRequest(request as ShardMapPublishingRequest,
                                                                   this);
                    break;
                }

                case _shardletMoveRequest:
                {
                    SendQueueProcessingEvent(_shardletMoveRequest, request);
                    Shardlet.HandleShardletMoveRequest(request as ShardletMoveRequest, this, uniqueProcessID);
                    break;
                }

                case _shardDeletionRequest:
                {
                    SendQueueProcessingEvent(_shardDeletionRequest, request);
                    ShardSetConfig.HandleShardDeletionRequest(request as ShardDeletionRequest, this);
                    break;
                }

                case _shardMapSyncRequest:
                {
                    SendQueueProcessingEvent(_shardMapSyncRequest, request);
                    ShardSetConfig.HandleShardSyncRequest(request as ShardSyncRequest, this);
                    break;
                }
                }

                request = GetNextQueueRequest();
            }

            SendQueueProcessingEvent("Queue Processing Completed");
        }
 public IList <Shard> GetShardsForShardSet(
     ShardSetConfig shardSetConfig)
 {
     using (var context = GetConnection(ConnectionString))
     {
         return(context.Shards.Where(ps => ps.ShardSetID == shardSetConfig.ShardSetID)
                .Select(ps =>
                        new Shard
         {
             Catalog = ps.Database.DatabaseName,
             Description = ps.Description,
             ServerInstanceName = ps.Database.Server.ServerName,
             PointerShardID = ps.ShardID
         })
                .ToList());
     }
 }
        public ShardSetConfig GetShardSetConfigLatestVersion(string shardSetName)
        {
            ShardSetConfig returnVal = null;

            using (var context = GetConnection(ConnectionString))
            {
                var shardSetConfig =
                    context.ShardSetConfigs
                    .Where(x => x.ShardSet.Name == shardSetName)
                    .OrderByDescending(x => x.Version)
                    .FirstOrDefault();

                if (shardSetConfig != null)
                {
                    returnVal = MakeShardSetConfig(shardSetConfig);
                }
            }
            return(returnVal);
        }
        public Shard AddShardToShardSet(
            Shard shard, ShardSetConfig shardSetConfig)
        {
            using (var context = GetConnection(ConnectionString))
            {
                var dbShard = new Models.Shard
                {
                    Database =
                        context.Databases
                        .FirstOrDefault(
                            d =>
                            d.DatabaseName == shard.Catalog &&
                            d.Server.ServerName == shard.ServerInstanceName),
                    Description = shard.Description,
                    ShardSet    =
                        context.ShardSets
                        .FirstOrDefault(tg => tg.ShardSetID == shardSetConfig.ShardSetID)
                };

                if (dbShard.Database == null)
                {
                    dbShard.Database =
                        new Database
                    {
                        DatabaseName = shard.Catalog,
                        Server       =
                            context.Servers.FirstOrDefault(s => s.ServerName == shard.ServerInstanceName),
                    };

                    context.Databases.Add(dbShard.Database);
                }

                context.Shards.Add(dbShard);
                context.SaveChanges();
                shard.PointerShardID = dbShard.ShardID;
            }

            return(shard);
        }
 private static IShardSetDriver GetShardSetDriver(ShardSetConfig config)
 {
     return(ScaleOutConfigManager.GetManager().GetShardSetDriver(config));
 }
        private ShardSetConfig MakeShardSetConfig(Models.ShardSetConfig shardSetConfigModel)
        {
            var shardSetConfig =
                new ShardSetConfig
            {
                ShardSetConfigID     = shardSetConfigModel.ShardSetConfigID,
                ShardSetID           = shardSetConfigModel.ShardSetID,
                AllowDeployments     = shardSetConfigModel.AllowDeployment,
                CurrentShardCount    = shardSetConfigModel.ShardMap.RangeShards.Count(),
                MaxShardCount        = shardSetConfigModel.MaxShardCount,
                MaxShardSizeMb       = shardSetConfigModel.MaxShardSizeMB,
                MaxShardletsPerShard = shardSetConfigModel.MaxShardletsPerShard,
                MinShardSizeMb       = shardSetConfigModel.MinShardSizeMB,
                TargetShardCount     = shardSetConfigModel.TargetShardCount,
                Version      = shardSetConfigModel.Version,
                ShardSetName = shardSetConfigModel.ShardSet.Name,
                CurrentPublishedShardMapID = shardSetConfigModel.ShardSet.CurrentShardMapID,
                ShardMap = new ShardMap
                {
                    ShardMapID = shardSetConfigModel.ShardMapID,
                    Shards     =
                        shardSetConfigModel.ShardMap.RangeShards
                        .Select(s =>
                                new RangeShard
                    {
                        ServerInstanceName  = s.Database.Server.ServerName,
                        Catalog             = s.Database.DatabaseName,
                        HighDistributionKey = s.RangeHighValue,
                        LowDistributionKey  = s.RangeLowValue,
                        ShardID             = s.ShardID
                    })
                        .ToList()
                },
                Shards = shardSetConfigModel.ShardSet.Shards
                         .Select(x =>
                                 new Shard
                {
                    Catalog            = x.Database.DatabaseName,
                    ServerInstanceName = x.Database.Server.ServerName,
                    Description        = x.Description,
                    PointerShardID     = x.ShardID
                })
                         .ToList(),
                Servers = shardSetConfigModel.ShardSet.Servers
                          .Select(x =>
                                  new Server
                {
                    ServerID           = x.ServerID,
                    ServerInstanceName = x.ServerName,
                    MaxShardsAllowed   = x.MaxShardsAllowed,
                    AvailableShards    = x.MaxShardsAllowed - x.Databases.Count
                                         //TODO : Change to databases that are deployed or planned.. not all databases in history
                })
                          .ToList(),
                ShardSetConfigSettings = shardSetConfigModel.ShardSetConfigSettings
                                         .Select(x =>
                                                 new DataElasticity.Models.ShardSetConfigSetting
                {
                    ShardSetConfigSettingID = x.ShardSetConfigSettingID,
                    ShardSetConfigID        = x.ShardSetConfigID,
                    SettingKey   = x.SettingKey,
                    SettingValue = x.SettingValue
                })
                                         .ToList(),
            };

            return(shardSetConfig);
        }
        public ShardSetConfig AddShardSetConfig(
            ShardSetConfig shardSetConfig)
        {
            using (var context = GetConnection(ConnectionString))
            {
                using (var scope = new TransactionScope())
                {
                    ShardSet shardSet;

                    // Locate the associated shard set either by the key or the shard set name
                    // If it does not exist, create and save it.

                    // Check to see if they gave us a ShardSetID
                    if (shardSetConfig.ShardSetID == -1)
                    {
                        // Lets try to look it up by Name
                        shardSet =
                            context.ShardSets.FirstOrDefault(x => x.Name == shardSetConfig.ShardSetName);

                        //If it doesn't exist create it.
                        if (shardSet == null)
                        {
                            shardSet =
                                new ShardSet
                            {
                                Name = shardSetConfig.ShardSetName
                            };

                            context.ShardSets.Add(shardSet);
                        }
                    }
                    else
                    {
                        shardSet =
                            context.ShardSets.FirstOrDefault(
                                x => x.ShardSetID == shardSetConfig.ShardSetID);

                        if (shardSet == null)
                        {
                            throw new InvalidOperationException(string.Format("Shard Set ID {0} does not exist",
                                                                              shardSetConfig.ShardSetID));
                        }
                    }


                    // if the configuration is being saved with a new Id, record it on the config table
                    if (shardSetConfig.CurrentPublishedShardMapID.HasValue)
                    {
                        shardSet.CurrentShardMapID = shardSetConfig.CurrentPublishedShardMapID;
                    }

                    // Nuke the server list because we are going to reload it from the configuration
                    shardSet.Servers.Clear();

                    //Now read the servers
                    foreach (var server in shardSetConfig.Servers)
                    {
                        //is the server new?
                        if (server.ServerID == -1)
                        {
                            server.Save();
                        }

                        //Lets grab the server object from the DB.
                        var dbServer = context.Servers.FirstOrDefault(x => x.ServerID == server.ServerID);
                        if (dbServer == null)
                        {
                            throw new Exception("Server not found in the config database");
                        }
                        shardSet.Servers.Add(dbServer);
                    }

                    if (shardSetConfig.ShardMap.ShardMapID == -1)
                    {
                        shardSetConfig.ShardMap = AddShardMap(shardSetConfig.ShardMap, context);
                    }

                    var dbShardSetConfig =
                        new Models.ShardSetConfig
                    {
                        AllowDeployment      = shardSetConfig.AllowDeployments,
                        MaxShardCount        = shardSetConfig.MaxShardCount,
                        MaxShardSizeMB       = shardSetConfig.MaxShardSizeMb,
                        MaxShardletsPerShard = shardSetConfig.MaxShardletsPerShard,
                        MinShardSizeMB       = shardSetConfig.MinShardSizeMb,
                        TargetShardCount     = shardSetConfig.TargetShardCount,
                        ShardMapID           = shardSetConfig.ShardMap.ShardMapID,
                        ShardSet             = shardSet,
                        Version = (shardSet.ShardSetConfigs.Count() + 1)
                    };

                    // create settings on the new configuration
                    foreach (var shardSetConfigSetting in shardSetConfig.ShardSetConfigSettings)
                    {
                        var setting = new ShardSetConfigSetting
                        {
                            SettingKey     = shardSetConfigSetting.SettingKey,
                            SettingValue   = shardSetConfigSetting.SettingValue,
                            ShardSetConfig = dbShardSetConfig
                        };

                        dbShardSetConfig.ShardSetConfigSettings.Add(setting);
                    }

                    context.ShardSetConfigs.Add(dbShardSetConfig);
                    context.SaveChanges();

                    shardSetConfig.ShardSetConfigID = dbShardSetConfig.ShardSetConfigID;
                    shardSetConfig.ShardSetID       = shardSet.ShardSetID;

                    AddPointerShards(context, shardSetConfig);

                    scope.Complete();
                }
            }

            return(shardSetConfig);
        }
        /// <summary>
        /// Moves to shard.
        /// </summary>
        /// <param name="shard">The shard to move the ShardLet data to.</param>
        /// <param name="pin">if set to <c>true</c> add the shard to the pinned shard list.</param>
        /// <param name="deleteOnMove">if set to <c>true</c> delete the Shardlet data from the existing Shard after move.</param>
        /// <param name="uniqueProcessID">A unique process identifier that can be set to maintain state between moves.</param>
        /// <param name="queueRequest">if set to <c>true</c> queue the shard move rather than executing it immediately.</param>
        public void MoveToShard(ShardBase shard, bool pin, bool deleteOnMove, Guid uniqueProcessID,
                                bool queueRequest = false)
        {
            if (queueRequest)
            {
                var request =
                    new ShardletMoveRequest
                {
                    DestinationCatalog            = shard.Catalog,
                    DestinationServerInstanceName = shard.ServerInstanceName,
                    SourceCatalog            = Catalog,
                    SourceServerInstanceName = ServerInstanceName,
                    DistributionKey          = DistributionKey,
                    ShardingKey     = ShardingKey,
                    ShardSetName    = ShardSetName,
                    DeleteOnMove    = deleteOnMove,
                    UniqueProcessID = uniqueProcessID,
                    Pin             = pin
                };

                request.Save();

                return;
            }

            var shardSetConnectionDriver = ScaleOutShardletManager.GetManager().GetShardletConnectionDriver(ShardSetName);

            try
            {
                // change the state to read only in the map and update th epin status
                Status = ShardletStatus.Moving;
                Pinned = pin;
                shardSetConnectionDriver.PublishShardlet(this);

                // terminate all existing connections
                shardSetConnectionDriver.TerminateConnections(this);

                // copy the shardlet data
                var shardSetConfig = ShardSetConfig.LoadCurrent(ShardSetName);

                var currentShard =
                    new RangeShard
                {
                    Catalog            = Catalog,
                    ServerInstanceName = ServerInstanceName,
                };

                shardSetConfig.CopyShardlet(currentShard, shard, ShardingKey, uniqueProcessID);

                // update the status and set the new catalog and instance in the shard map
                Catalog            = shard.Catalog;
                ServerInstanceName = shard.ServerInstanceName;
                Status             = ShardletStatus.Active;

                shardSetConnectionDriver.PublishShardlet(this);

                if (deleteOnMove)
                {
                    shardSetConfig.DeleteShardlet(currentShard, ShardingKey);
                }
            }
            catch (Exception)
            {
                Status = ShardletStatus.Active;
                shardSetConnectionDriver.PublishShardlet(this);

                throw;
            }
        }