/// <summary>
/// Calls the API that makes the HTTP request to the server. Retries the HTTP request in certain cases. This is a synchronous call.
/// </summary>
/// <param name="opCode">Operation Code</param>
/// <param name="path">Path of the file or directory</param>
/// <param name="requestData">byte array, offset and length of the data of http request</param>
/// <param name="responseData">byte array, offset and length of the data of http response. byte array should be initialized for chunked response</param>
/// <param name="quer">Headers for request</param>
/// <param name="client">ADLS Store CLient</param>
/// <param name="req">Request options containing RetryOption, timout and requestid </param>
/// <param name="resp">Contains the response message </param>
/// <param name="customHeaders">Dictionary containing the custom header that Core wants to pass</param>
/// <returns>Tuple of Byte array containing the bytes returned from the server and number of bytes read from server</returns>
internal static Tuple <byte[], int> MakeCall(string opCode, string path, ByteBuffer requestData, ByteBuffer responseData, QueryParams quer, AdlsClient client, RequestOptions req, OperationResponse resp, IDictionary <string, string> customHeaders = null)
{
if (!VerifyMakeCallArguments(opCode, path, requestData, quer, client, req, resp))
{
return(null);
}
string uuid = req.RequestId;
int numRetries = 0;
Tuple <byte[], int> retVal;
do
{
resp.Reset();
req.RequestId = uuid + "." + numRetries;
resp.Retries = numRetries;
Stopwatch watch = Stopwatch.StartNew();
retVal = MakeSingleCall(opCode, path, requestData, responseData, quer, client, req, resp, customHeaders);
watch.Stop();
resp.LastCallLatency = watch.ElapsedMilliseconds;
HandleMakeSingleCallResponse(opCode, path, resp, retVal?.Item2 ?? 0, requestData.Count, req, client, quer.Serialize(opCode), ref numRetries);
// If dip is used, this request is not ignoring dip, and there is a connection failure, then reset the DIP
if (client.DipIp != null && !req.IgnoreDip && resp.ConnectionFailure)
{
WebTransportLog.Debug("Connection Failure, DIP enabled, Resetting Dip");
client.UpdateDipAsync(default(CancellationToken)).GetAwaiter().GetResult();
}
} while (!resp.IsSuccessful && req.RetryOption.ShouldRetry((int)resp.HttpStatus, resp.Ex));
resp.OpCode = opCode;
return(retVal);
}
/// <summary>
/// Makes a single Http call to the server, sends the request and obtains the response. This is a synchronous call.
/// </summary>
/// <param name="opCode">Operation Code</param>
/// <param name="path">Path of the file or directory</param>
/// <param name="requestData">byte array, offset and length of the data of http request</param>
/// <param name="responseData">byte array, offset and length of the data of http response. byte array should be initialized for chunked response</param>
/// <param name="qp">Headers for request</param>
/// <param name="client">ADLS Store CLient</param>
/// <param name="req">Request options containing RetryOption, timout and requestid </param>
/// <param name="resp">Contains the response message </param>
/// <param name="customHeaders">Dictionary containing the custom header that Core wants to pass</param>
/// <returns>Tuple of Byte array containing the bytes returned from the server and number of bytes read from server</returns>
private static Tuple <byte[], int> MakeSingleCall(string opCode, string path, ByteBuffer requestData, ByteBuffer responseData, QueryParams qp, AdlsClient client, RequestOptions req, OperationResponse resp, IDictionary <string, string> customHeaders)
{
string token = null;
Operation op = Operation.Operations[opCode];
string urlString = CreateHttpRequestUrl(op, path, client, resp, qp.Serialize(opCode), req);
if (string.IsNullOrEmpty(urlString))
{
return(null);
}
try
{
// Create does not throw WebException
HttpWebRequest webReq = (HttpWebRequest)WebRequest.Create(urlString);
// If security certificate is used then no need to pass token
if (req.ClientCert != null)
{
#if NET452
webReq.ClientCertificates.Add(req.ClientCert);
#endif
}
Stopwatch watch = Stopwatch.StartNew();
token = client.GetTokenAsync().GetAwaiter().GetResult();
watch.Stop();
resp.TokenAcquisitionLatency = watch.ElapsedMilliseconds;
if (string.IsNullOrEmpty(token))
{
resp.Ex = new ArgumentException($"Token is null or empty.");
return(null);
}
if (token.Length <= AuthorizationHeaderLengthThreshold)
{
resp.Ex = new ArgumentException($"Token Length is {token.Length}. Token is most probably malformed.");
return(null);
}
resp.AuthorizationHeaderLength = token.Length;
AssignCommonHttpHeaders(webReq, client, req, token, op.Method, customHeaders, requestData.Count);
using (CancellationTokenSource timeoutCancellationTokenSource = GetCancellationTokenSourceForTimeout(req))
{
try
{
//This point onwards if operation is cancelled http request is aborted
timeoutCancellationTokenSource.Token.Register(OnCancel, webReq);
if (!op.Method.Equals("GET"))
{
if (op.RequiresBody && requestData.Data != null)
{
#if NET452
using (Stream ipStream = GetCompressedStream(webReq.GetRequestStream(), client, requestData.Count))
#else
using (Stream ipStream = GetCompressedStream(webReq.GetRequestStreamAsync().GetAwaiter().GetResult(), client, requestData.Count))
#endif
{
ipStream.Write(requestData.Data, requestData.Offset, requestData.Count);
}
}
else
{
SetWebRequestContentLength(webReq, 0);
}
}
#if NET452
using (var webResponse = (HttpWebResponse)webReq.GetResponse())
#else
using (var webResponse = (HttpWebResponse)webReq.GetResponseAsync().GetAwaiter().GetResult())
#endif
{
resp.HttpStatus = webResponse.StatusCode;
resp.HttpMessage = webResponse.StatusDescription;
resp.RequestId = webResponse.Headers["x-ms-request-id"];
PostPowershellLogDetails(webReq, webResponse);
if (op.ReturnsBody)
{
if (!InitializeResponseData(webResponse, ref responseData))
{
return(null);
}
int totalBytes = 0;
using (Stream opStream = webResponse.GetResponseStream())
{
int noBytes;
int totalLengthToRead = responseData.Count;
//Read the required amount of data. In case of chunked it is what users requested, else it is amount of data sent
do
{
noBytes = opStream.Read(responseData.Data, responseData.Offset, totalLengthToRead);
totalBytes += noBytes;
responseData.Offset += noBytes;
totalLengthToRead -= noBytes;
} while (noBytes > 0 && totalLengthToRead > 0);
}
return
(Tuple.Create(responseData.Data,
totalBytes)); //Return the total bytes read also since in case of chunked amount of data returned can be less than data returned
}
}
}
catch (WebException e)
{
HandleWebException(e, resp, path, req.RequestId, token, webReq, timeoutCancellationTokenSource.Token);
}
}
}// Any unhandled exception is caught here
catch (Exception e)
{
resp.Ex = e;
}
return(null);
}
/// <summary>
/// Verifies whether the arguments for MakeCall is correct. Throws exception if any argument is null or out of range.
/// </summary>
/// <param name="opCode">Operation Code</param>
/// <param name="path">Path of the file or directory</param>
/// <param name="requestData">byte array, offset and length of the data of http request</param>
/// <param name="quer">Headers for request</param>
/// <param name="client">ADLS Store CLient</param>
/// <param name="req">Request options containing RetryOption, timout and requestid </param>
/// <param name="resp">Contains the response message </param>
/// <returns>False if there is any errors with arguments else true</returns>
private static bool VerifyMakeCallArguments(string opCode, string path, ByteBuffer requestData, QueryParams quer, AdlsClient client, RequestOptions req, OperationResponse resp)
{
//Check all type of errors and exceptions
if (resp == null)
{
throw new ArgumentNullException(nameof(resp)); //Check if resp is not null
}
if (req == null)
{
throw new ArgumentNullException(nameof(req)); //Check if req is not null
}
if (quer == null)
{
throw new ArgumentNullException(nameof(quer)); //Check if quer is not null
}
if (client == null)
{
throw new ArgumentNullException(nameof(client)); //Check for client
}
if (String.IsNullOrEmpty(client.AccountFQDN) || string.IsNullOrEmpty(client.AccountFQDN)) //Check the client account
{
resp.IsSuccessful = false;
resp.Error = "The client account name is missing.";
return(false);
}
if (!Operation.Operations.ContainsKey(opCode))
{
resp.IsSuccessful = false;
resp.Error = "Operation Code doesnot exist.";
return(false);
}
if (String.IsNullOrEmpty(path) || string.IsNullOrEmpty(path.Trim()))//Check for path
{
resp.IsSuccessful = false;
resp.Error = "The file/directory path for this operation is missing.";
return(false);
}
//Check for request data
if (requestData.Data != null && (requestData.Offset >= requestData.Data.Length || (requestData.Offset < 0) || (requestData.Count + requestData.Offset > requestData.Data.Length)))
{
throw new ArgumentOutOfRangeException(nameof(requestData.Offset));
}
return(true);
}
/// <summary>
/// Calls the API that makes the HTTP request to the server. Retries the HTTP request in certain cases. This is a asynchronous call.
/// </summary>
/// <param name="opCode">Operation Code</param>
/// <param name="path">Path of the file or directory</param>
/// <param name="requestData">byte array, offset and length of the data of http request</param>
/// <param name="responseData">byte array, offset and length of the data of http response. byte array should be initialized for chunked response</param>
/// <param name="quer">Headers for request</param>
/// <param name="client">ADLS Store CLient</param>
/// <param name="req">Request options containing RetryOption, timout and requestid </param>
/// <param name="resp">Contains the response message </param>
/// <param name="cancelToken">CancellationToken to cancel the operation</param>
/// <param name="customHeaders">Dictionary containing the custom header that Core wants to pass</param>
/// <returns>Tuple of Byte array containing the bytes returned from the server and number of bytes read from server</returns>
internal static async Task <Tuple <byte[], int> > MakeCallAsync(string opCode, string path,
ByteBuffer requestData, ByteBuffer responseData, QueryParams quer, AdlsClient client, RequestOptions req, OperationResponse resp, CancellationToken cancelToken, IDictionary <string, string> customHeaders = null)
{
if (!VerifyMakeCallArguments(opCode, path, requestData, quer, client, req, resp))
{
return(null);
}
string uuid = req.RequestId;
int numRetries = 0;
Tuple <byte[], int> retVal;
do
{
resp.Reset();
req.RequestId = uuid + "." + numRetries;
resp.Retries = numRetries;
Stopwatch watch = Stopwatch.StartNew();
retVal = await MakeSingleCallAsync(opCode, path, requestData, responseData, quer, client, req, resp, cancelToken, customHeaders).ConfigureAwait(false);
watch.Stop();
resp.LastCallLatency = watch.ElapsedMilliseconds;
HandleMakeSingleCallResponse(opCode, path, resp, retVal?.Item2 ?? 0, requestData.Count, req, client, quer.Serialize(opCode), ref numRetries);
if (resp.Ex is OperationCanceledException)//Operation is cancelled then no retries
{
break;
}
// If dip is used, this request is not ignoring dip, and there is a connection failure, then reset the DIP
if (client.DipIp != null && !req.IgnoreDip && resp.ConnectionFailure)
{
await client.UpdateDipAsync(cancelToken).ConfigureAwait(false);
}
} while (!resp.IsSuccessful && req.RetryOption.ShouldRetry((int)resp.HttpStatus, resp.Ex));
resp.OpCode = opCode;
return(retVal);
}
/// <summary>
/// Parses RemoteException and populates the remote error fields in OperationResponse
/// </summary>
/// <param name="errorBytes">Error Response bytes</param>
/// <param name="errorBytesLength">Error response bytes length</param>
/// <param name="resp">Response instance</param>
/// <param name="contentType">Content Type</param>
private static void ParseRemoteError(byte[] errorBytes, int errorBytesLength, OperationResponse resp, string contentType)
{
try
{
using (MemoryStream errorStream = new MemoryStream(errorBytes, 0, errorBytesLength))
{
using (StreamReader stReader = new StreamReader(errorStream))
{
using (var jsonReader = new JsonTextReader(stReader))
{
jsonReader.Read(); //StartObject {
jsonReader.Read(); //"RemoteException"
if (jsonReader.Value == null || !((string)jsonReader.Value).Equals("RemoteException"))
{
throw new IOException(
$"Unexpected type of exception in JSON error output. Expected: RemoteException Actual: {jsonReader.Value}");
}
jsonReader.Read(); //StartObject {
do
{
jsonReader.Read();
if (jsonReader.TokenType.Equals(JsonToken.PropertyName))
{
switch ((string)jsonReader.Value)
{
case "exception":
jsonReader.Read();
resp.RemoteExceptionName = (string)jsonReader.Value;
break;
case "message":
jsonReader.Read();
resp.RemoteExceptionMessage = (string)jsonReader.Value;
break;
case "javaClassName":
jsonReader.Read();
resp.RemoteExceptionJavaClassName = (string)jsonReader.Value;
break;
}
}
} while (!jsonReader.TokenType.Equals(JsonToken.EndObject));
}
}
}
}
catch (Exception e)
{
resp.Ex = e;
//Store the actual remote response in a separate variable, since response can have illegal charcaters which will throw exception while setting them to headers
resp.RemoteErrorNonJsonResponse = $" Content-Type of error response: {contentType}. Error: {Encoding.UTF8.GetString(errorBytes, 0, errorBytesLength)}";
}
}
/// <summary>
/// Handles WebException. Determines whether it is due to cancelled operation, remoteexception from server or some other webexception
/// </summary>
/// <param name="e">WebException instance</param>
/// <param name="resp">OperationResponse</param>
/// <param name="path">Path</param>
/// <param name="requestId">Request Id</param>
/// <param name="token">Auth token</param>
/// <param name="webReq">Http Web request</param>
/// <param name="timeoutCancelToken">Cancel Token for timeout</param>
/// <param name="actualCancelToken">Cancel Token sent by user</param>
private static void HandleWebException(WebException e, OperationResponse resp, string path, string requestId, string token, HttpWebRequest webReq, CancellationToken timeoutCancelToken, CancellationToken actualCancelToken = default(CancellationToken))
{
// The status property will be set to RequestCanceled after Abort.
if (timeoutCancelToken.IsCancellationRequested)
{
// Type should not be of operationcancelledexception otherwise this wont be retried
resp.Ex = new Exception("Operation timed out");
}
else if (actualCancelToken.IsCancellationRequested)
{
resp.Ex = new OperationCanceledException(actualCancelToken);
}
//This the case where some exception occured in server but server returned a response
else if (e.Status == WebExceptionStatus.ProtocolError)
{
try
{
using (var errorResponse = (HttpWebResponse)e.Response)
{
PostPowershellLogDetails(webReq, errorResponse);
resp.HttpStatus = errorResponse.StatusCode;
resp.RequestId = errorResponse.Headers["x-ms-request-id"];
if (resp.HttpStatus == HttpStatusCode.Unauthorized && TokenLog.IsDebugEnabled)
{
string tokenLogLine =
$"HTTPRequest,HTTP401,cReqId:{requestId},sReqId:{resp.RequestId},path:{path},token:{token}";
TokenLog.Debug(tokenLogLine);
}
resp.HttpMessage = errorResponse.StatusDescription;
ByteBuffer errorResponseData = default(ByteBuffer);
if (!InitializeResponseData(errorResponse, ref errorResponseData, true))
{
throw new ArgumentException("ContentLength of error response stream is not set");
}
using (Stream errorStream = errorResponse.GetResponseStream())
{
// Reading the data from the error response into a byte array is necessary to show the actual error data as a part of the
// error message in case JSON parsing does not work. We read the bytes and then pass it back to JsonTextReader using a memorystream
int noBytes;
int totalLengthToRead = errorResponseData.Count;
do
{
noBytes = errorStream.Read(errorResponseData.Data, errorResponseData.Offset, totalLengthToRead);
errorResponseData.Offset += noBytes;
totalLengthToRead -= noBytes;
} while (noBytes > 0 && totalLengthToRead > 0);
ParseRemoteError(errorResponseData.Data, errorResponseData.Count, resp, errorResponse.Headers["Content-Type"]);
}
}
}
catch (Exception ex)
{
resp.Ex = ex;
}
}
else//No response stream is returned, Dont know what to do, so just store the exception
{
switch (e.Status)
{
case WebExceptionStatus.NameResolutionFailure:
case WebExceptionStatus.ServerProtocolViolation:
case WebExceptionStatus.ConnectFailure:
case WebExceptionStatus.ConnectionClosed:
case WebExceptionStatus.KeepAliveFailure:
case WebExceptionStatus.ReceiveFailure:
case WebExceptionStatus.SendFailure:
case WebExceptionStatus.Timeout:
case WebExceptionStatus.UnknownError:
resp.ConnectionFailure = true;
break;
}
resp.Ex = e;
}
}
/// <summary>
/// Serializes the client FQDN, queryparams and token into a request URL
/// </summary>
/// <param name="op">Operation</param>
/// <param name="path">Path of directory or file</param>
/// <param name="client">AdlsClient</param>
/// <param name="resp">OperationResponse</param>
/// <param name="queryParams">Serialized queryparams</param>
/// <param name="clientReqOptions">Request options</param>
/// <returns>URL</returns>
private static string CreateHttpRequestUrl(Operation op, string path, AdlsClient client, OperationResponse resp, string queryParams, RequestOptions clientReqOptions)
{
StringBuilder urlString = new StringBuilder(UrlLength);
urlString.Append(client.GetHttpPrefix());
urlString.Append("://");
// If dip ip is set and ignore dip is not specified then request needs to go to dip ip
if (client.DipIp != null && !clientReqOptions.IgnoreDip)
{
urlString.Append(client.DipIp);
}
else
{
urlString.Append(client.AccountFQDN);
}
urlString.Append(op.Namespace);
// This is to prevent badly formed requests for uris not having a preceding /
if (path[0] != '/')
{
urlString.Append(Uri.EscapeDataString("/"));
}
try
{
urlString.Append(Uri.EscapeDataString(path));
}
catch (UriFormatException ex)
{
resp.Ex = ex;
return(null);
}
urlString.Append("?");
urlString.Append(queryParams);
try
{
var uri = new Uri(urlString.ToString());
}
catch (UriFormatException ur)
{
resp.Ex = ur;
return(null);
}
return(urlString.ToString());
}