private void MultiWriteUsingBlobLease(WriteOp op) { // TODO: recover from failed clients that may leave a write partially completed // TODO: remove use of leases and replace with ETags // throw new Exception("Write to multiple primaries not yet implemented."); try { bool done = false; while (!done) { // grab lease on blob to guard against concurrent writers using (PrimaryCloudBlobLease lease = new PrimaryCloudBlobLease(blobName, configuration, true)) { if (lease.HasLease) { // TODO: fix code for writing to multiple primaries (and remove it from here) foreach (string server in configuration.PrimaryServers) { watch.Start(); ICloudBlob blob = ClientRegistry.GetCloudBlob(server, configuration.Name, blobName, false); op(blob); watch.Stop(); ServerState ss = slaEngine.Monitor.GetServerState(server); ss.AddRtt(watch.ElapsedMilliseconds); slaEngine.Session.RecordObjectWritten(blobName, Timestamp(blob), ss); } done = true; } } } } catch (StorageException se) { throw se; } catch (Exception ex) { throw ex; } }
private void DoUploadFromStream(System.IO.Stream source, Microsoft.WindowsAzure.Storage.AccessCondition accessCondition = null, Microsoft.WindowsAzure.Storage.Blob.BlobRequestOptions options = null, Microsoft.WindowsAzure.Storage.OperationContext operationContext = null) { try { bool done = false; while (!done) { using (PrimaryCloudBlobLease lease = new PrimaryCloudBlobLease(Name, configuration, true)) { if (lease.HasLease) { foreach (string server in configuration.PrimaryServers) { watch.Start(); ICloudBlob blob = ClientRegistry.GetCloudBlob(server, configuration.Name, Name, false); source.Position = 0; blob.UploadFromStream(source, lease.getAccessConditionWithLeaseId(accessCondition), options, operationContext); watch.Stop(); ServerState ss = slaEngine.Monitor.GetServerState(server); ss.AddRtt(watch.ElapsedMilliseconds); slaEngine.Session.RecordObjectWritten(Name, Timestamp(blob), ss); } done = true; } } } } catch (StorageException se) { throw se; } catch (Exception ex) { throw ex; } }
/// <summary> /// Selects the servers to use for Bounded staleness. /// This is a bit tricky since we may not have accurate information about the freshness of each server. /// </summary> /// <param name="objectName">The object being read</param> /// <param name="bound">Time bound for bounded staleness (in seconds)</param> /// <returns>The selected servers.</returns> public HashSet <ServerState> SelectServersForBoundedStaleness(string objectName, int bound) { // Note: This assumes that the client's clock is synchronized with the servers' clocks. // It also assumes that the server's highTime is an indication of when it last synced with a master // and that the server received all versions with a lower timestamp during its last sync. DateTimeOffset minHighTime = DateTimeOffset.Now - TimeSpan.FromSeconds(bound); // find servers that are not too stale HashSet <ServerState> set = SelectServersForStrongConsistency(objectName); foreach (string serverName in config.SecondaryServers) { ServerState server = monitor.GetServerState(serverName); // TODO: check timestamp on Azure secondary site if (server.HighTime >= minHighTime) { set.Add(server); } } return(set); }
public ServerUtility(ServerState s, float u) { ss = s; utility = u; }
private void MultiWrite(WriteOp op, AccessCondition access, List <SessionState> sessions = null, ServerMonitor monitor = null) { // This protocol uses three phases and ETags // It assumes that the client is in fast mode and remains so throughout the protocol // i.e. it assumes that the set of primaries does not change. // TODO: recover from failed clients that may leave a write partially completed ICloudBlob blob; Dictionary <string, string> eTags; // Phase 1. Mark intention to write with write-in-progress flags eTags = SetWiPFlags(); if (eTags == null) { // flags were not successfully set, so abort the protocol return; } // Phase 2. Perform write at all primaries bool didAtLeastOne = false; foreach (string server in configuration.PrimaryServers) { blob = ClientRegistry.GetCloudBlob(server, configuration.Name, blobName, false); access.IfMatchETag = eTags[server]; watch.Start(); try { op(blob); } catch (StorageException) { // If writing fails at some primary, then abort the protocol // It could be that a concurrent writer is in progress // Note that some writes may have already been performed // If so, then we leave the WiP flags set so the recovery process will kick in // or so a concurrent writer can complete its protocol and overwrite our writes // If not, then we can clear the WiP flags if (!didAtLeastOne) { ClearWiPFlags(eTags); } return; } watch.Stop(); eTags[server] = blob.Properties.ETag; didAtLeastOne = true; // update session and server state ServerState ss = (monitor == null) ? slaEngine.Monitor.GetServerState(server) : monitor.GetServerState(server); ss.AddRtt(watch.ElapsedMilliseconds); if (sessions == null) { slaEngine.Session.RecordObjectWritten(blobName, Timestamp(blob), ss); } else { foreach (SessionState session in sessions) { session.RecordObjectWritten(blobName, Timestamp(blob), ss); } } } // Phase 3. Clear write-in-progress flags to indicate that write has completed ClearWiPFlags(eTags); }
/// <summary> /// Perform read optimistically and then verify configuration. /// </summary> /// <param name="op">The read operation</param> /// <param name="blob">The blob being read</param> /// <param name="ss">The chosen server</param> /// <returns>whether the read succeeded; if not, then it should be tried again.</returns> private void SlowRead(ReadOp op) { // TODO: Deal with the case that we try to read from a replica that is no longer a replica and the read fails // In this case, the read should be retried after refreshing the configuration. ServerState ss = null; try { bool isDone = false; do // until we enter fast mode or succeed at reading in slow mode with the correct configuration { ss = slaEngine.FindServerToRead(blobName); blobForRead = ClientRegistry.GetCloudBlob(ss.Name, containerName, blobName); // TODO: this should really check if the reader wants strong consistency // It could be that eventual consistency is fine but the SLA Engine just happened to choose a primary server // In this case, we can do a fast mode read since we don't care if the chosen server is no longer a primary if (configuration.IsInFastMode() || !configuration.PrimaryServers.Contains(ss.Name)) { // it is fine to read from the selected secondary replica // or from a primary replica if we have now entered fast mode FastRead(op); isDone = true; } else { // read from the selected replica that we believe to be a primary replica watch.Start(); op(blobForRead); watch.Stop(); // then see if the configuration has changed and the selected replica is no longer primary configuration.SyncWithCloud(ClientRegistry.GetConfigurationAccount()); // TODO: check the epoch number on the configuration to make sure that we have not missed a configuration. // i.e. we need to make sure that the server from which we read did not become a secondary during our read, // and then back to primary when we checked. // TODO: maybe we should check that the read delivered a non-zero utility. // It is possible that the configuration was so stale that a much better server could have been chosen. if (configuration.PrimaryServers.Contains(ss.Name)) { //We have contacted the primary replica, hence we are good to go. slaEngine.RecordObjectRead(blobForRead.Name, Timestamp(blobForRead), ss, watch.ElapsedMilliseconds); isDone = true; } isDone = false; // not done } } while (!isDone); } // TODO: decide if we need to catch any exceptions here or just let then propagate through catch (StorageException se) { if (StorageExceptionCode.NotFound(se)) { //blob is not found. //this is fine because the replica might have removed. //it simply returns, so client need to re-execute if this happens. //We can also re-execute the read here, but for debugging purposes, it's simpler for the client to re-execute. //Of course in real system, we need to re-execute at this level. return; } // storage exceptions are passed through to the caller throw; } catch (Exception ex) { if (ex.Message.Contains("Object reference not set to an instance of an object")) { return; } throw ex; } }
public void RecordObjectRead(string name, DateTimeOffset timestamp, ServerState server, long duration) { server.AddRtt(duration); Session.RecordObjectRead(name, timestamp, server, Sla.Id); DetermineHitSubSLA(name, timestamp, server, duration); }