private PullResponse PullOneChunk(HgResumeApiParameters request)
{
var pullResponse = new PullResponse(PullStatus.Fail);
try
{
HgResumeApiResponse response = _apiServer.Execute("pullBundleChunk", request, TimeoutInSeconds);
if (response == null)
{
_progress.WriteVerbose("API REQ: {0} Timeout", _apiServer.Url);
pullResponse.Status = PullStatus.Timeout;
return(pullResponse);
}
/* API returns the following HTTP codes:
* 200 OK (SUCCESS)
* 304 Not Modified (NOCHANGE)
* 400 Bad Request (FAIL, UNKNOWNID)
*/
_progress.WriteVerbose("API REQ: {0} RSP: {1} in {2}ms", _apiServer.Url, response.HttpStatus, response.ResponseTimeInMilliseconds);
if (response.ResumableResponse.HasNote)
{
_progress.WriteMessage(String.Format("Server replied: {0}", response.ResumableResponse.Note));
}
if (response.HttpStatus == HttpStatusCode.ServiceUnavailable && response.Content.Length > 0)
{
var msg = String.Format("Server temporarily unavailable: {0}",
Encoding.UTF8.GetString(response.Content));
_progress.WriteError(msg);
pullResponse.Status = PullStatus.NotAvailable;
return(pullResponse);
}
if (response.HttpStatus == HttpStatusCode.NotModified)
{
pullResponse.Status = PullStatus.NoChange;
return(pullResponse);
}
if (response.HttpStatus == HttpStatusCode.Accepted)
{
pullResponse.Status = PullStatus.InProgress;
return(pullResponse);
}
// chunk pulled OK
if (response.HttpStatus == HttpStatusCode.OK)
{
pullResponse.BundleSize = response.ResumableResponse.BundleSize;
pullResponse.Status = PullStatus.OK;
pullResponse.ChunkSize = CalculateChunkSize(request.ChunkSize, response.ResponseTimeInMilliseconds);
pullResponse.Chunk = response.Content;
return(pullResponse);
}
if (response.HttpStatus == HttpStatusCode.BadRequest && response.ResumableResponse.Status == "UNKNOWNID")
{
// this is not implemented currently (feb 2012 cjh)
_progress.WriteError("The server {0} does not have the project '{1}'", _targetLabel, request.RepoId);
return(pullResponse);
}
if (response.HttpStatus == HttpStatusCode.BadRequest && response.ResumableResponse.Status == "RESET")
{
pullResponse.Status = PullStatus.Reset;
return(pullResponse);
}
if (response.HttpStatus == HttpStatusCode.BadRequest)
{
if (response.ResumableResponse.HasError)
{
if (response.ResumableResponse.Error == "invalid baseHash")
{
pullResponse.Status = PullStatus.InvalidHash;
}
else
{
_progress.WriteWarning("Server Error: {0}", response.ResumableResponse.Error);
}
}
return(pullResponse);
}
if (response.HttpStatus == HttpStatusCode.Unauthorized)
{
_progress.WriteWarning("There is an authorization problem accessing this project. Check the project ID as well as your username and password. Alternatively, you may not be authorized to access this project.");
pullResponse.Status = PullStatus.Unauthorized;
}
_progress.WriteWarning("Invalid Server Response '{0}'", response.HttpStatus);
return(pullResponse);
}
catch (WebException e)
{
_progress.WriteWarning(String.Format("Pull data chunk failed: {0}", e.Message));
return(pullResponse);
}
}
public bool Pull(string[] baseRevisions)
{
var tipRevision = _repo.GetTip();
string localTip = "0";
string errorMessage;
if (tipRevision != null)
{
localTip = tipRevision.Number.Hash;
}
if (baseRevisions.Length == 0)
{
errorMessage = "Pull failed: No base revision.";
_progress.WriteError(errorMessage);
throw new HgResumeOperationFailed(errorMessage);
}
string bundleId = "";
foreach (var revision in baseRevisions)
{
bundleId += revision + "_" + localTip + '-';
}
bundleId = bundleId.TrimEnd('-');
var bundleHelper = new PullStorageManager(PathToLocalStorage, bundleId);
var req = new HgResumeApiParameters
{
RepoId = _apiServer.ProjectId,
BaseHashes = baseRevisions,
TransId = bundleHelper.TransactionId,
StartOfWindow = bundleHelper.StartOfWindow,
ChunkSize = InitialChunkSize
};
int bundleSize = 0;
int loopCtr = 1;
bool retryLoop;
do
{
if (_progress.CancelRequested)
{
throw new Palaso.CommandLineProcessing.UserCancelledException();
}
retryLoop = false;
var response = PullOneChunk(req);
if (response.Status == PullStatus.Unauthorized)
{
throw new UnauthorizedAccessException();
}
if (response.Status == PullStatus.NotAvailable)
{
_progress.ProgressIndicator.Initialize();
_progress.ProgressIndicator.Finish();
return(false);
}
if (response.Status == PullStatus.Timeout)
{
_progress.WriteWarning("Pull operation timed out. Retrying...");
retryLoop = true;
continue;
}
if (response.Status == PullStatus.NoChange)
{
_progress.WriteMessage("No changes");
bundleHelper.Cleanup();
_progress.ProgressIndicator.Initialize();
_progress.ProgressIndicator.Finish();
return(false);
}
if (response.Status == PullStatus.InProgress)
{
_progress.WriteStatus("Preparing data on server");
retryLoop = true;
// advance the progress bar 2% to show that something is happening
_progress.ProgressIndicator.PercentCompleted = _progress.ProgressIndicator.PercentCompleted + 2;
continue;
}
if (response.Status == PullStatus.InvalidHash)
{
// this should not happen...but sometimes it gets into a state where it remembers the wrong basehash of the server (CJH Feb-12)
retryLoop = true;
_progress.WriteVerbose("Invalid basehash response received from server... clearing cache and retrying");
req.BaseHashes = GetHashStringsFromRevisions(GetCommonBaseHashesWithRemoteRepo(false));
continue;
}
if (response.Status == PullStatus.Fail)
{
// Not sure that we need to abort the attempts just because .Net web request says so. See Push also. CP 2012-06
errorMessage = "Pull data chunk failed";
_progress.WriteError(errorMessage);
retryLoop = true;
req.StartOfWindow = bundleHelper.StartOfWindow;
//_progress.ProgressIndicator.Initialize();
//_progress.ProgressIndicator.Finish();
//throw new HgResumeOperationFailed(errorMessage);
continue;
}
if (response.Status == PullStatus.Reset)
{
retryLoop = true;
bundleHelper.Reset();
_progress.WriteVerbose("Server's bundle cache has expired. Restarting pull...");
req.StartOfWindow = bundleHelper.StartOfWindow;
continue;
}
//bundleSizeFromResponse = response.BundleSize;
if (loopCtr == 1)
{
_progress.ProgressIndicator.Initialize();
if (req.StartOfWindow != 0)
{
string message = String.Format("Resuming pull operation at {0} received", GetHumanReadableByteSize(req.StartOfWindow));
_progress.WriteVerbose(message);
}
}
bundleHelper.WriteChunk(req.StartOfWindow, response.Chunk);
req.StartOfWindow = req.StartOfWindow + response.Chunk.Length;
req.ChunkSize = response.ChunkSize;
if (bundleSize == response.BundleSize && bundleSize != 0)
{
_progress.ProgressIndicator.PercentCompleted = (int)((long)req.StartOfWindow * 100 / bundleSize);
string eta = CalculateEstimatedTimeRemaining(bundleSize, req.ChunkSize, req.StartOfWindow);
_progress.WriteStatus(string.Format("Receiving {0} {1}", GetHumanReadableByteSize(bundleSize), eta));
}
else
{
// this is only useful when the bundle size is significantly large (like with a clone operation) such that
// the server takes a long time to create the bundle, and the bundleSize continues to rise as the chunks are received
bundleSize = response.BundleSize;
_progress.WriteStatus(string.Format("Preparing data on server (>{0})", GetHumanReadableByteSize(bundleSize)));
}
loopCtr++;
} while (req.StartOfWindow < bundleSize || retryLoop);
if (_repo.Unbundle(bundleHelper.BundlePath))
{
_progress.WriteMessage("Pull operation completed successfully");
_progress.ProgressIndicator.Finish();
_progress.WriteMessage("Finished Receiving");
bundleHelper.Cleanup();
var response = FinishPull(req.TransId);
if (response == PullStatus.Reset)
{
/* Calling Pull recursively to finish up another pull will mess up the ProgressIndicator. This case is
* // rare enough that it's not worth trying to get the progress indicator working for a recursive Pull()
*/
_progress.WriteMessage("Remote repo has changed. Initiating additional pull operation");
return(Pull());
}
// REVIEW: I'm not sure why this was set to the server tip before, if we just pulled then won't our head
// be the correct common base? Maybe not if a merge needs to happen,
LastKnownCommonBases = new List <Revision>(_repo.BranchingHelper.GetBranches());
return(true);
}
_progress.WriteError("Received all data but local unbundle operation failed or resulted in multiple heads!");
_progress.ProgressIndicator.Finish();
bundleHelper.Cleanup();
errorMessage = "Pull operation failed";
_progress.WriteError(errorMessage);
throw new HgResumeOperationFailed(errorMessage);
}
private PushResponse PushOneChunk(HgResumeApiParameters request, byte[] dataToPush)
{
var pushResponse = new PushResponse(PushStatus.Fail);
try
{
HgResumeApiResponse response = _apiServer.Execute("pushBundleChunk", request, dataToPush, TimeoutInSeconds);
if (response == null)
{
_progress.WriteVerbose("API REQ: {0} Timeout");
pushResponse.Status = PushStatus.Timeout;
return(pushResponse);
}
/* API returns the following HTTP codes:
* 200 OK (SUCCESS)
* 202 Accepted (RECEIVED)
* 412 Precondition Failed (RESEND)
* 400 Bad Request (FAIL, UNKNOWNID, and RESET)
*/
_progress.WriteVerbose("API REQ: {0} RSP: {1} in {2}ms", _apiServer.Url, response.HttpStatus, response.ResponseTimeInMilliseconds);
if (response.HttpStatus == HttpStatusCode.ServiceUnavailable && response.Content.Length > 0)
{
var msg = String.Format("Server temporarily unavailable: {0}",
Encoding.UTF8.GetString(response.Content));
_progress.WriteError(msg);
pushResponse.Status = PushStatus.NotAvailable;
return(pushResponse);
}
if (response.ResumableResponse.HasNote)
{
_progress.WriteWarning(String.Format("Server replied: {0}", response.ResumableResponse.Note));
}
// the chunk was received successfully
if (response.HttpStatus == HttpStatusCode.Accepted)
{
pushResponse.StartOfWindow = response.ResumableResponse.StartOfWindow;
pushResponse.Status = PushStatus.Received;
pushResponse.ChunkSize = CalculateChunkSize(request.ChunkSize, response.ResponseTimeInMilliseconds);
return(pushResponse);
}
// the final chunk was received successfully
if (response.HttpStatus == HttpStatusCode.OK)
{
pushResponse.Status = PushStatus.Complete;
return(pushResponse);
}
if (response.HttpStatus == HttpStatusCode.BadRequest)
{
if (response.ResumableResponse.Status == "UNKNOWNID")
{
_progress.WriteError("The server {0} does not have the project '{1}'", _targetLabel, _apiServer.ProjectId);
return(pushResponse);
}
if (response.ResumableResponse.Status == "RESET")
{
_progress.WriteError("Push failed: All chunks were pushed to the server, but the unbundle operation failed. Try again later.");
pushResponse.Status = PushStatus.Reset;
return(pushResponse);
}
if (response.ResumableResponse.HasError)
{
if (response.ResumableResponse.Error == "invalid baseHash")
{
pushResponse.Status = PushStatus.InvalidHash;
}
else
{
_progress.WriteError("Server Error: {0}", response.ResumableResponse.Error);
}
return(pushResponse);
}
}
_progress.WriteWarning("Invalid Server Response '{0}'", response.HttpStatus);
return(pushResponse);
}
catch (WebException e)
{
_progress.WriteWarning(String.Format("Push data chunk failed: {0}", e.Message));
return(pushResponse);
}
}
public void Push()
{
var baseRevisions = GetCommonBaseHashesWithRemoteRepo();
if (baseRevisions == null)
{
const string errorMessage = "Push failed: A common revision could not be found with the server.";
_progress.WriteError(errorMessage);
throw new HgResumeOperationFailed(errorMessage);
}
// create a bundle to push
string tip = _repo.GetTip().Number.Hash;
string bundleId = "";
foreach (var revision in baseRevisions)
{
bundleId += String.Format("{0}-{1}", revision.Number.Hash, tip);
}
var bundleHelper = new PushStorageManager(PathToLocalStorage, bundleId);
var bundleFileInfo = new FileInfo(bundleHelper.BundlePath);
if (bundleFileInfo.Length == 0)
{
_progress.WriteStatus("Preparing data to send");
bool bundleCreatedSuccessfully = _repo.MakeBundle(GetHashStringsFromRevisions(baseRevisions), bundleHelper.BundlePath);
if (!bundleCreatedSuccessfully)
{
// try again after clearing revision cache
LastKnownCommonBases = new List <Revision>();
baseRevisions = GetCommonBaseHashesWithRemoteRepo();
bundleCreatedSuccessfully = _repo.MakeBundle(GetHashStringsFromRevisions(baseRevisions), bundleHelper.BundlePath);
if (!bundleCreatedSuccessfully)
{
const string errorMessage = "Push failed: Unable to create local bundle.";
_progress.WriteError(errorMessage);
throw new HgResumeOperationFailed(errorMessage);
}
}
bundleFileInfo.Refresh();
if (bundleFileInfo.Length == 0)
{
bundleHelper.Cleanup();
_progress.WriteMessage("No changes to send. Push operation completed");
return;
}
}
var req = new HgResumeApiParameters
{
RepoId = _apiServer.ProjectId,
TransId = bundleHelper.TransactionId,
StartOfWindow = 0,
BundleSize = (int)bundleFileInfo.Length
};
req.ChunkSize = (req.BundleSize < InitialChunkSize) ? req.BundleSize : InitialChunkSize;
// req.BaseHash = baseRevisions; <-- Unless I'm not reading the php right we don't need to set this on push. JLN Aug-12
_progress.ProgressIndicator.Initialize();
_progress.WriteStatus("Sending data");
int loopCtr = 0;
do // loop until finished... or until the user cancels
{
loopCtr++;
if (_progress.CancelRequested)
{
throw new Palaso.CommandLineProcessing.UserCancelledException();
}
int dataRemaining = req.BundleSize - req.StartOfWindow;
if (dataRemaining < req.ChunkSize)
{
req.ChunkSize = dataRemaining;
}
byte[] bundleChunk = bundleHelper.GetChunk(req.StartOfWindow, req.ChunkSize);
/* API parameters
* $repoId, $baseHash, $bundleSize, $offset, $data, $transId
* */
var response = PushOneChunk(req, bundleChunk);
if (response.Status == PushStatus.NotAvailable)
{
_progress.ProgressIndicator.Initialize();
_progress.ProgressIndicator.Finish();
return;
}
if (response.Status == PushStatus.Timeout)
{
_progress.WriteWarning("Push operation timed out. Retrying...");
continue;
}
if (response.Status == PushStatus.InvalidHash)
{
// this should not happen...but sometimes it gets into a state where it remembers the wrong basehash of the server (CJH Feb-12)
_progress.WriteVerbose("Invalid basehash response received from server... clearing cache and retrying");
// req.BaseHash = GetCommonBaseHashesWithRemoteRepo(false); <-- Unless I'm misreading the php we don't need to set this on a push. JLN Aug-12
continue;
}
if (response.Status == PushStatus.Fail)
{
// This 'Fail' intentionally aborts the push attempt. I think we can continue to go around the loop and retry. See Pull also. CP 2012-06
continue;
//var errorMessage = "Push operation failed";
//_progress.WriteError(errorMessage);
//throw new HgResumeOperationFailed(errorMessage);
}
if (response.Status == PushStatus.Reset)
{
FinishPush(req.TransId);
bundleHelper.Cleanup();
const string errorMessage = "Push failed: Server reset.";
_progress.WriteError(errorMessage);
throw new HgResumeOperationFailed(errorMessage);
}
if (response.Status == PushStatus.Complete || req.StartOfWindow >= req.BundleSize)
{
if (response.Status == PushStatus.Complete)
{
_progress.WriteMessage("Finished sending");
}
else
{
_progress.WriteMessage("Finished sending. Server unpacking data");
}
_progress.ProgressIndicator.Finish();
// update our local knowledge of what the server has
LastKnownCommonBases = new List <Revision>(_repo.BranchingHelper.GetBranches()); // This may be a little optimistic, the server may still be unbundling the data.
//FinishPush(req.TransId); // We can't really tell when the server has finished processing our pulled data. The server cleans up after itself. CP 2012-07
bundleHelper.Cleanup();
return;
}
req.ChunkSize = response.ChunkSize;
req.StartOfWindow = response.StartOfWindow;
if (loopCtr == 1 && req.StartOfWindow > req.ChunkSize)
{
string message = String.Format("Resuming push operation at {0} sent", GetHumanReadableByteSize(req.StartOfWindow));
_progress.WriteVerbose(message);
}
string eta = CalculateEstimatedTimeRemaining(req.BundleSize, req.ChunkSize, req.StartOfWindow);
_progress.WriteStatus(string.Format("Sending {0} {1}", GetHumanReadableByteSize(req.BundleSize), eta));
_progress.ProgressIndicator.PercentCompleted = (int)((long)req.StartOfWindow * 100 / req.BundleSize);
} while (req.StartOfWindow < req.BundleSize);
}
public HgResumeApiResponse Execute(string method, HgResumeApiParameters parameters, byte[] contentToSend, int secondsBeforeTimeout)
{
string queryString = parameters.BuildQueryString();
_urlExecuted = String.Format("{0}://{1}/api/v{2}/{3}?{4}", _url.Scheme, _url.Host, APIVERSION, method, queryString);
var req = WebRequest.Create(_urlExecuted) as HttpWebRequest;
req.UserAgent = String.Format("HgResume v{0}", APIVERSION);
req.PreAuthenticate = true;
if (!_url.UserInfo.Contains(":"))
{
throw new HgResumeException("Username or password were not supplied in custom location");
}
req.Credentials = new NetworkCredential(UserName, Password);
req.Timeout = secondsBeforeTimeout * 1000; // timeout is in milliseconds
if (contentToSend.Length == 0)
{
req.Method = WebRequestMethods.Http.Get;
}
else
{
req.Method = WebRequestMethods.Http.Post;
req.ContentLength = contentToSend.Length;
req.ContentType = "text/plain"; // i'm not sure this is really what we want. The other possibility is "application/x-www-form-urlencoded"
using (var reqStream = req.GetRequestStream())
{
reqStream.Write(contentToSend, 0, contentToSend.Length);
}
}
HttpWebResponse res;
HgResumeApiResponse apiResponse;
var stopwatch = new Stopwatch();
stopwatch.Start();
try
{
using (res = (HttpWebResponse)req.GetResponse())
{
apiResponse = HandleResponse(res);
}
}
catch (WebException e)
{
if (e.Status == WebExceptionStatus.ProtocolError)
{
using (res = (HttpWebResponse)e.Response)
{
apiResponse = HandleResponse(res);
}
}
else if (e.Status == WebExceptionStatus.Timeout)
{
apiResponse = null;
}
else
{
throw; // throw for other types of network errors (see WebExceptionStatus for the full list of errors)
}
}
finally
{
stopwatch.Stop();
}
if (apiResponse != null)
{
apiResponse.ResponseTimeInMilliseconds = stopwatch.ElapsedMilliseconds;
}
return(apiResponse);
}
public HgResumeApiResponse Execute(string method, HgResumeApiParameters request, int secondsBeforeTimeout)
{
return(Execute(method, request, new byte[0], secondsBeforeTimeout));
}
