/// <summary> /// Calls the API that makes the HTTP request to the server. Retries the HTTP request in certain cases. This is a synchronous call. /// </summary> /// <param name="opCode">Operation Code</param> /// <param name="path">Path of the file or directory</param> /// <param name="requestData">byte array, offset and length of the data of http request</param> /// <param name="responseData">byte array, offset and length of the data of http response. byte array should be initialized for chunked response</param> /// <param name="quer">Headers for request</param> /// <param name="client">ADLS Store CLient</param> /// <param name="req">Request options containing RetryOption, timout and requestid </param> /// <param name="resp">Contains the response message </param> /// <param name="customHeaders">Dictionary containing the custom header that Core wants to pass</param> /// <returns>Tuple of Byte array containing the bytes returned from the server and number of bytes read from server</returns> internal static Tuple <byte[], int> MakeCall(string opCode, string path, ByteBuffer requestData, ByteBuffer responseData, QueryParams quer, AdlsClient client, RequestOptions req, OperationResponse resp, IDictionary <string, string> customHeaders = null) { if (!VerifyMakeCallArguments(opCode, path, requestData, quer, client, req, resp)) { return(null); } string uuid = req.RequestId; int numRetries = 0; Tuple <byte[], int> retVal; do { resp.Reset(); req.RequestId = uuid + "." + numRetries; resp.Retries = numRetries; Stopwatch watch = Stopwatch.StartNew(); retVal = MakeSingleCall(opCode, path, requestData, responseData, quer, client, req, resp, customHeaders); watch.Stop(); resp.LastCallLatency = watch.ElapsedMilliseconds; HandleMakeSingleCallResponse(opCode, path, resp, retVal?.Item2 ?? 0, requestData.Count, req, client, quer.Serialize(opCode), ref numRetries); // If dip is used, this request is not ignoring dip, and there is a connection failure, then reset the DIP if (client.DipIp != null && !req.IgnoreDip && resp.ConnectionFailure) { WebTransportLog.Debug("Connection Failure, DIP enabled, Resetting Dip"); client.UpdateDipAsync(default(CancellationToken)).GetAwaiter().GetResult(); } } while (!resp.IsSuccessful && req.RetryOption.ShouldRetry((int)resp.HttpStatus, resp.Ex)); resp.OpCode = opCode; return(retVal); }
/// <summary> /// Makes a single Http call to the server, sends the request and obtains the response. This is a synchronous call. /// </summary> /// <param name="opCode">Operation Code</param> /// <param name="path">Path of the file or directory</param> /// <param name="requestData">byte array, offset and length of the data of http request</param> /// <param name="responseData">byte array, offset and length of the data of http response. byte array should be initialized for chunked response</param> /// <param name="qp">Headers for request</param> /// <param name="client">ADLS Store CLient</param> /// <param name="req">Request options containing RetryOption, timout and requestid </param> /// <param name="resp">Contains the response message </param> /// <param name="customHeaders">Dictionary containing the custom header that Core wants to pass</param> /// <returns>Tuple of Byte array containing the bytes returned from the server and number of bytes read from server</returns> private static Tuple <byte[], int> MakeSingleCall(string opCode, string path, ByteBuffer requestData, ByteBuffer responseData, QueryParams qp, AdlsClient client, RequestOptions req, OperationResponse resp, IDictionary <string, string> customHeaders) { string token = null; Operation op = Operation.Operations[opCode]; string urlString = CreateHttpRequestUrl(op, path, client, resp, qp.Serialize(opCode), req); if (string.IsNullOrEmpty(urlString)) { return(null); } try { // Create does not throw WebException HttpWebRequest webReq = (HttpWebRequest)WebRequest.Create(urlString); // If security certificate is used then no need to pass token if (req.ClientCert != null) { #if NET452 webReq.ClientCertificates.Add(req.ClientCert); #endif } Stopwatch watch = Stopwatch.StartNew(); token = client.GetTokenAsync().GetAwaiter().GetResult(); watch.Stop(); resp.TokenAcquisitionLatency = watch.ElapsedMilliseconds; if (string.IsNullOrEmpty(token)) { resp.Ex = new ArgumentException($"Token is null or empty."); return(null); } if (token.Length <= AuthorizationHeaderLengthThreshold) { resp.Ex = new ArgumentException($"Token Length is {token.Length}. Token is most probably malformed."); return(null); } resp.AuthorizationHeaderLength = token.Length; AssignCommonHttpHeaders(webReq, client, req, token, op.Method, customHeaders, requestData.Count); using (CancellationTokenSource timeoutCancellationTokenSource = GetCancellationTokenSourceForTimeout(req)) { try { //This point onwards if operation is cancelled http request is aborted timeoutCancellationTokenSource.Token.Register(OnCancel, webReq); if (!op.Method.Equals("GET")) { if (op.RequiresBody && requestData.Data != null) { #if NET452 using (Stream ipStream = GetCompressedStream(webReq.GetRequestStream(), client, requestData.Count)) #else using (Stream ipStream = GetCompressedStream(webReq.GetRequestStreamAsync().GetAwaiter().GetResult(), client, requestData.Count)) #endif { ipStream.Write(requestData.Data, requestData.Offset, requestData.Count); } } else { SetWebRequestContentLength(webReq, 0); } } #if NET452 using (var webResponse = (HttpWebResponse)webReq.GetResponse()) #else using (var webResponse = (HttpWebResponse)webReq.GetResponseAsync().GetAwaiter().GetResult()) #endif { resp.HttpStatus = webResponse.StatusCode; resp.HttpMessage = webResponse.StatusDescription; resp.RequestId = webResponse.Headers["x-ms-request-id"]; PostPowershellLogDetails(webReq, webResponse); if (op.ReturnsBody) { if (!InitializeResponseData(webResponse, ref responseData)) { return(null); } int totalBytes = 0; using (Stream opStream = webResponse.GetResponseStream()) { int noBytes; int totalLengthToRead = responseData.Count; //Read the required amount of data. In case of chunked it is what users requested, else it is amount of data sent do { noBytes = opStream.Read(responseData.Data, responseData.Offset, totalLengthToRead); totalBytes += noBytes; responseData.Offset += noBytes; totalLengthToRead -= noBytes; } while (noBytes > 0 && totalLengthToRead > 0); } return (Tuple.Create(responseData.Data, totalBytes)); //Return the total bytes read also since in case of chunked amount of data returned can be less than data returned } } } catch (WebException e) { HandleWebException(e, resp, path, req.RequestId, token, webReq, timeoutCancellationTokenSource.Token); } } }// Any unhandled exception is caught here catch (Exception e) { resp.Ex = e; } return(null); }
/// <summary> /// Verifies whether the arguments for MakeCall is correct. Throws exception if any argument is null or out of range. /// </summary> /// <param name="opCode">Operation Code</param> /// <param name="path">Path of the file or directory</param> /// <param name="requestData">byte array, offset and length of the data of http request</param> /// <param name="quer">Headers for request</param> /// <param name="client">ADLS Store CLient</param> /// <param name="req">Request options containing RetryOption, timout and requestid </param> /// <param name="resp">Contains the response message </param> /// <returns>False if there is any errors with arguments else true</returns> private static bool VerifyMakeCallArguments(string opCode, string path, ByteBuffer requestData, QueryParams quer, AdlsClient client, RequestOptions req, OperationResponse resp) { //Check all type of errors and exceptions if (resp == null) { throw new ArgumentNullException(nameof(resp)); //Check if resp is not null } if (req == null) { throw new ArgumentNullException(nameof(req)); //Check if req is not null } if (quer == null) { throw new ArgumentNullException(nameof(quer)); //Check if quer is not null } if (client == null) { throw new ArgumentNullException(nameof(client)); //Check for client } if (String.IsNullOrEmpty(client.AccountFQDN) || string.IsNullOrEmpty(client.AccountFQDN)) //Check the client account { resp.IsSuccessful = false; resp.Error = "The client account name is missing."; return(false); } if (!Operation.Operations.ContainsKey(opCode)) { resp.IsSuccessful = false; resp.Error = "Operation Code doesnot exist."; return(false); } if (String.IsNullOrEmpty(path) || string.IsNullOrEmpty(path.Trim()))//Check for path { resp.IsSuccessful = false; resp.Error = "The file/directory path for this operation is missing."; return(false); } //Check for request data if (requestData.Data != null && (requestData.Offset >= requestData.Data.Length || (requestData.Offset < 0) || (requestData.Count + requestData.Offset > requestData.Data.Length))) { throw new ArgumentOutOfRangeException(nameof(requestData.Offset)); } return(true); }
/// <summary> /// Calls the API that makes the HTTP request to the server. Retries the HTTP request in certain cases. This is a asynchronous call. /// </summary> /// <param name="opCode">Operation Code</param> /// <param name="path">Path of the file or directory</param> /// <param name="requestData">byte array, offset and length of the data of http request</param> /// <param name="responseData">byte array, offset and length of the data of http response. byte array should be initialized for chunked response</param> /// <param name="quer">Headers for request</param> /// <param name="client">ADLS Store CLient</param> /// <param name="req">Request options containing RetryOption, timout and requestid </param> /// <param name="resp">Contains the response message </param> /// <param name="cancelToken">CancellationToken to cancel the operation</param> /// <param name="customHeaders">Dictionary containing the custom header that Core wants to pass</param> /// <returns>Tuple of Byte array containing the bytes returned from the server and number of bytes read from server</returns> internal static async Task <Tuple <byte[], int> > MakeCallAsync(string opCode, string path, ByteBuffer requestData, ByteBuffer responseData, QueryParams quer, AdlsClient client, RequestOptions req, OperationResponse resp, CancellationToken cancelToken, IDictionary <string, string> customHeaders = null) { if (!VerifyMakeCallArguments(opCode, path, requestData, quer, client, req, resp)) { return(null); } string uuid = req.RequestId; int numRetries = 0; Tuple <byte[], int> retVal; do { resp.Reset(); req.RequestId = uuid + "." + numRetries; resp.Retries = numRetries; Stopwatch watch = Stopwatch.StartNew(); retVal = await MakeSingleCallAsync(opCode, path, requestData, responseData, quer, client, req, resp, cancelToken, customHeaders).ConfigureAwait(false); watch.Stop(); resp.LastCallLatency = watch.ElapsedMilliseconds; HandleMakeSingleCallResponse(opCode, path, resp, retVal?.Item2 ?? 0, requestData.Count, req, client, quer.Serialize(opCode), ref numRetries); if (resp.Ex is OperationCanceledException)//Operation is cancelled then no retries { break; } // If dip is used, this request is not ignoring dip, and there is a connection failure, then reset the DIP if (client.DipIp != null && !req.IgnoreDip && resp.ConnectionFailure) { await client.UpdateDipAsync(cancelToken).ConfigureAwait(false); } } while (!resp.IsSuccessful && req.RetryOption.ShouldRetry((int)resp.HttpStatus, resp.Ex)); resp.OpCode = opCode; return(retVal); }
/// <summary> /// Parses RemoteException and populates the remote error fields in OperationResponse /// </summary> /// <param name="errorBytes">Error Response bytes</param> /// <param name="errorBytesLength">Error response bytes length</param> /// <param name="resp">Response instance</param> /// <param name="contentType">Content Type</param> private static void ParseRemoteError(byte[] errorBytes, int errorBytesLength, OperationResponse resp, string contentType) { try { using (MemoryStream errorStream = new MemoryStream(errorBytes, 0, errorBytesLength)) { using (StreamReader stReader = new StreamReader(errorStream)) { using (var jsonReader = new JsonTextReader(stReader)) { jsonReader.Read(); //StartObject { jsonReader.Read(); //"RemoteException" if (jsonReader.Value == null || !((string)jsonReader.Value).Equals("RemoteException")) { throw new IOException( $"Unexpected type of exception in JSON error output. Expected: RemoteException Actual: {jsonReader.Value}"); } jsonReader.Read(); //StartObject { do { jsonReader.Read(); if (jsonReader.TokenType.Equals(JsonToken.PropertyName)) { switch ((string)jsonReader.Value) { case "exception": jsonReader.Read(); resp.RemoteExceptionName = (string)jsonReader.Value; break; case "message": jsonReader.Read(); resp.RemoteExceptionMessage = (string)jsonReader.Value; break; case "javaClassName": jsonReader.Read(); resp.RemoteExceptionJavaClassName = (string)jsonReader.Value; break; } } } while (!jsonReader.TokenType.Equals(JsonToken.EndObject)); } } } } catch (Exception e) { resp.Ex = e; //Store the actual remote response in a separate variable, since response can have illegal charcaters which will throw exception while setting them to headers resp.RemoteErrorNonJsonResponse = $" Content-Type of error response: {contentType}. Error: {Encoding.UTF8.GetString(errorBytes, 0, errorBytesLength)}"; } }
/// <summary> /// Handles WebException. Determines whether it is due to cancelled operation, remoteexception from server or some other webexception /// </summary> /// <param name="e">WebException instance</param> /// <param name="resp">OperationResponse</param> /// <param name="path">Path</param> /// <param name="requestId">Request Id</param> /// <param name="token">Auth token</param> /// <param name="webReq">Http Web request</param> /// <param name="timeoutCancelToken">Cancel Token for timeout</param> /// <param name="actualCancelToken">Cancel Token sent by user</param> private static void HandleWebException(WebException e, OperationResponse resp, string path, string requestId, string token, HttpWebRequest webReq, CancellationToken timeoutCancelToken, CancellationToken actualCancelToken = default(CancellationToken)) { // The status property will be set to RequestCanceled after Abort. if (timeoutCancelToken.IsCancellationRequested) { // Type should not be of operationcancelledexception otherwise this wont be retried resp.Ex = new Exception("Operation timed out"); } else if (actualCancelToken.IsCancellationRequested) { resp.Ex = new OperationCanceledException(actualCancelToken); } //This the case where some exception occured in server but server returned a response else if (e.Status == WebExceptionStatus.ProtocolError) { try { using (var errorResponse = (HttpWebResponse)e.Response) { PostPowershellLogDetails(webReq, errorResponse); resp.HttpStatus = errorResponse.StatusCode; resp.RequestId = errorResponse.Headers["x-ms-request-id"]; if (resp.HttpStatus == HttpStatusCode.Unauthorized && TokenLog.IsDebugEnabled) { string tokenLogLine = $"HTTPRequest,HTTP401,cReqId:{requestId},sReqId:{resp.RequestId},path:{path},token:{token}"; TokenLog.Debug(tokenLogLine); } resp.HttpMessage = errorResponse.StatusDescription; ByteBuffer errorResponseData = default(ByteBuffer); if (!InitializeResponseData(errorResponse, ref errorResponseData, true)) { throw new ArgumentException("ContentLength of error response stream is not set"); } using (Stream errorStream = errorResponse.GetResponseStream()) { // Reading the data from the error response into a byte array is necessary to show the actual error data as a part of the // error message in case JSON parsing does not work. We read the bytes and then pass it back to JsonTextReader using a memorystream int noBytes; int totalLengthToRead = errorResponseData.Count; do { noBytes = errorStream.Read(errorResponseData.Data, errorResponseData.Offset, totalLengthToRead); errorResponseData.Offset += noBytes; totalLengthToRead -= noBytes; } while (noBytes > 0 && totalLengthToRead > 0); ParseRemoteError(errorResponseData.Data, errorResponseData.Count, resp, errorResponse.Headers["Content-Type"]); } } } catch (Exception ex) { resp.Ex = ex; } } else//No response stream is returned, Dont know what to do, so just store the exception { switch (e.Status) { case WebExceptionStatus.NameResolutionFailure: case WebExceptionStatus.ServerProtocolViolation: case WebExceptionStatus.ConnectFailure: case WebExceptionStatus.ConnectionClosed: case WebExceptionStatus.KeepAliveFailure: case WebExceptionStatus.ReceiveFailure: case WebExceptionStatus.SendFailure: case WebExceptionStatus.Timeout: case WebExceptionStatus.UnknownError: resp.ConnectionFailure = true; break; } resp.Ex = e; } }
/// <summary> /// Serializes the client FQDN, queryparams and token into a request URL /// </summary> /// <param name="op">Operation</param> /// <param name="path">Path of directory or file</param> /// <param name="client">AdlsClient</param> /// <param name="resp">OperationResponse</param> /// <param name="queryParams">Serialized queryparams</param> /// <param name="clientReqOptions">Request options</param> /// <returns>URL</returns> private static string CreateHttpRequestUrl(Operation op, string path, AdlsClient client, OperationResponse resp, string queryParams, RequestOptions clientReqOptions) { StringBuilder urlString = new StringBuilder(UrlLength); urlString.Append(client.GetHttpPrefix()); urlString.Append("://"); // If dip ip is set and ignore dip is not specified then request needs to go to dip ip if (client.DipIp != null && !clientReqOptions.IgnoreDip) { urlString.Append(client.DipIp); } else { urlString.Append(client.AccountFQDN); } urlString.Append(op.Namespace); // This is to prevent badly formed requests for uris not having a preceding / if (path[0] != '/') { urlString.Append(Uri.EscapeDataString("/")); } try { urlString.Append(Uri.EscapeDataString(path)); } catch (UriFormatException ex) { resp.Ex = ex; return(null); } urlString.Append("?"); urlString.Append(queryParams); try { var uri = new Uri(urlString.ToString()); } catch (UriFormatException ur) { resp.Ex = ur; return(null); } return(urlString.ToString()); }