internal override async Task <bool> NextAsync()
{
if (isClosed)
{
throw new SnowflakeDbException(SFError.DATA_READER_ALREADY_CLOSED);
}
_currentChunkRowIdx++;
if (_currentChunkRowIdx < _currentChunkRowCount)
{
return(true);
}
if (_chunkDownloader != null)
{
// GetNextChunk could be blocked if download result is not done yet.
// So put this piece of code in a seperate task
Logger.Info("Get next chunk from chunk downloader");
IResultChunk nextChunk = await _chunkDownloader.GetNextChunkAsync();
if (nextChunk != null)
{
resetChunkInfo(nextChunk);
return(true);
}
else
{
return(false);
}
}
return(false);
}
/*public Task<IResultChunk> GetNextChunkAsync()
* {
* return _downloadTasks.IsCompleted ? Task.FromResult<SFResultChunk>(null) : _downloadTasks.Take();
* }*/
public Task <IResultChunk> GetNextChunkAsync()
{
logger.Info($"NextChunkToConsume: {nextChunkToConsumeIndex}, NextChunkToDownload: {nextChunkToDownloadIndex}");
if (nextChunkToConsumeIndex < chunkInfos.Count)
{
Task <IResultChunk> chunk = taskQueues[nextChunkToConsumeIndex % prefetchSlot];
if (nextChunkToDownloadIndex < chunkInfos.Count && nextChunkToConsumeIndex > 0)
{
SFReusableChunk reusableChunk = chunkDatas[nextChunkToDownloadIndex % prefetchSlot];
reusableChunk.Reset(chunkInfos[nextChunkToDownloadIndex], nextChunkToDownloadIndex);
taskQueues[nextChunkToDownloadIndex % prefetchSlot] = DownloadChunkAsync(new DownloadContextV3()
{
chunk = reusableChunk,
qrmk = this.qrmk,
chunkHeaders = this.chunkHeaders,
cancellationToken = externalCancellationToken
});
nextChunkToDownloadIndex++;
}
nextChunkToConsumeIndex++;
return(chunk);
}
else
{
return(Task.FromResult <IResultChunk>(null));
}
}
internal override Task <bool> NextAsync()
{
if (isClosed)
{
throw new SnowflakeDbException(SFError.DATA_READER_ALREADY_CLOSED);
}
_currentChunkRowIdx++;
if (_currentChunkRowIdx < _currentChunkRowCount)
{
return(Task.FromResult(true));
}
if (_chunkDownloader != null)
{
// GetNextChunk could be blocked if download result is not done yet.
// So put this piece of code in a seperate task
return(Task.Run(() =>
{
Logger.Info("Get next chunk from chunk downloader");
SFResultChunk nextChunk;
if ((nextChunk = _chunkDownloader.GetNextChunk()) != null)
{
resetChunkInfo(nextChunk);
return true;
}
else
{
return false;
}
}));
}
return(Task.FromResult(false));
}
/// <see cref="IAuthenticator"/>
async Task IAuthenticator.AuthenticateAsync(CancellationToken cancellationToken)
{
logger.Info("External Browser Authentication");
int localPort = GetRandomUnusedPort();
string proofKey;
string samlResponseToken;
using (var httpListener = GetHttpListener(localPort))
{
httpListener.Start();
logger.Debug("Get IdpUrl and ProofKey");
var authenticatorRestRequest = BuildAuthenticatorRestRequest(localPort);
var authenticatorRestResponse =
await session.restRequester.PostAsync <AuthenticatorResponse>(
authenticatorRestRequest,
cancellationToken
).ConfigureAwait(false);
authenticatorRestResponse.FilterFailedResponse();
var idpUrl = authenticatorRestResponse.data.ssoUrl;
proofKey = authenticatorRestResponse.data.proofKey;
logger.Debug("Open browser");
StartBrowser(idpUrl);
logger.Debug("Get the redirect SAML request");
var context = await httpListener.GetContextAsync().ConfigureAwait(false);
var request = context.Request;
samlResponseToken = ValidateAndExtractToken(request);
HttpListenerResponse response = context.Response;
try
{
using (var output = response.OutputStream)
{
await output.WriteAsync(SUCCESS_RESPONSE, 0, SUCCESS_RESPONSE.Length).ConfigureAwait(false);
}
}
catch (Exception e)
{
// Ignore the exception as it does not affect the overall authentication flow
logger.Warn("External browser response not sent out");
}
httpListener.Stop();
}
logger.Debug("Send login request");
var loginResponse = await session.restRequester.PostAsync <LoginResponse>(
BuildExternalBrowserLoginRequest(samlResponseToken, proofKey),
cancellationToken
).ConfigureAwait(false);
session.ProcessLoginResponse(loginResponse);
}
internal static SFSessionProperties parseConnectionString(String connectionString, SecureString password)
{
logger.Info("Start parsing connection string.");
SFSessionProperties properties = new SFSessionProperties();
string[] propertyEntry = connectionString.Split(';');
foreach (string keyVal in propertyEntry)
{
if (keyVal.Length > 0)
{
string[] token = keyVal.Split(new string[] { "=" }, StringSplitOptions.None);
if (token.Length == 2)
{
try
{
SFSessionProperty p = (SFSessionProperty)Enum.Parse(
typeof(SFSessionProperty), token[0].ToUpper());
properties.Add(p, token[1]);
logger.Info($"Connection property: {p}, value: {(p == SFSessionProperty.PASSWORD ? "XXXXXXXX" : token[1])}");
}
catch (ArgumentException e)
{
logger.Warn($"Property {token[0]} not found ignored.");
}
}
else
{
string invalidStringDetail = String.Format("Invalid kay value pair {0}", keyVal);
SnowflakeDbException e = new SnowflakeDbException(SFError.INVALID_CONNECTION_STRING,
new object[] { invalidStringDetail });
logger.Error("Invalid string.", e);
throw e;
}
}
}
if (password != null)
{
properties[SFSessionProperty.PASSWORD] = new NetworkCredential(string.Empty, password).Password;
}
checkSessionProperties(properties);
// compose host value if not specified
if (!properties.ContainsKey(SFSessionProperty.HOST))
{
string hostName = String.Format("%s.snowflakecomputing.com", properties[SFSessionProperty.ACCOUNT]);
properties.Add(SFSessionProperty.HOST, hostName);
logger.Info($"Compose host name: {hostName}");
}
return(properties);
}
private void AssignQueryRequestId()
{
lock (_requestIdLock)
{
if (_requestId != null)
{
logger.Info("Another query is running.");
throw new SnowflakeDbException(SFError.STATEMENT_ALREADY_RUNNING_QUERY);
}
_requestId = Guid.NewGuid().ToString();
}
}
internal void renewSession()
{
RenewSessionRequest postBody = new RenewSessionRequest()
{
oldSessionToken = this.sessionToken,
requestType = "RENEW"
};
SFRestRequest renewSessionRequest = new SFRestRequest
{
jsonBody = postBody,
Url = BuildUri(RestPath.SF_TOKEN_REQUEST_PATH,
new Dictionary <string, string> {
{ SF_QUERY_REQUEST_ID, Guid.NewGuid().ToString() }
}),
authorizationToken = string.Format(SF_AUTHORIZATION_SNOWFLAKE_FMT, masterToken),
RestTimeout = Timeout.InfiniteTimeSpan
};
logger.Info("Renew the session.");
var response = restRequester.Post <RenewSessionResponse>(renewSessionRequest);
if (!response.success)
{
SnowflakeDbException e = new SnowflakeDbException("",
response.code, response.message, "");
logger.Error("Renew session failed", e);
throw e;
}
else
{
sessionToken = response.data.sessionToken;
masterToken = response.data.masterToken;
}
}
public SFChunkDownloaderV2(int colCount, List <ExecResponseChunk> chunkInfos, string qrmk,
Dictionary <string, string> chunkHeaders, CancellationToken cancellationToken)
{
this.qrmk = qrmk;
this.chunkHeaders = chunkHeaders;
this.chunks = new List <SFResultChunk>();
externalCancellationToken = cancellationToken;
var idx = 0;
foreach (ExecResponseChunk chunkInfo in chunkInfos)
{
this.chunks.Add(new SFResultChunk(chunkInfo.url, chunkInfo.rowCount, colCount, idx++));
}
logger.Info($"Total chunk number: {chunks.Count}");
FillDownloads();
}
/// <see cref="IAuthenticator"/>
async Task IAuthenticator.AuthenticateAsync(CancellationToken cancellationToken)
{
logger.Info("Okta Authentication");
// Clear cookies before authenticating because when a cookie is present in the request,
// Okta will assume it is coming from a browser and perform a CSRF check.
// This will ensure that we are NOT including the ‘sid’ cookie with the request.
HttpUtil.ClearCookies(oktaUrl);
logger.Debug("step 1: get sso and token url");
var authenticatorRestRequest = BuildAuthenticatorRestRequest();
var authenticatorResponse = await session.restRequester.PostAsync <AuthenticatorResponse>(authenticatorRestRequest, cancellationToken).ConfigureAwait(false);
authenticatorResponse.FilterFailedResponse();
Uri ssoUrl = new Uri(authenticatorResponse.data.ssoUrl);
Uri tokenUrl = new Uri(authenticatorResponse.data.tokenUrl);
logger.Debug("step 2: verify urls fetched from step 1");
logger.Debug("Checking sso url");
VerifyUrls(ssoUrl, oktaUrl);
logger.Debug("Checking token url");
VerifyUrls(tokenUrl, oktaUrl);
logger.Debug("step 3: get idp onetime token");
IdpTokenRestRequest idpTokenRestRequest = BuildIdpTokenRestRequest(tokenUrl);
var idpResponse = await session.restRequester.PostAsync <IdpTokenResponse>(idpTokenRestRequest, cancellationToken).ConfigureAwait(false);
string onetimeToken = idpResponse.CookieToken;
logger.Debug("step 4: get SAML reponse from sso");
var samlRestRequest = BuildSAMLRestRequest(ssoUrl, onetimeToken);
var samlRawResponse = await session.restRequester.GetAsync(samlRestRequest, cancellationToken).ConfigureAwait(false);
samlRawHtmlString = await samlRawResponse.Content.ReadAsStringAsync().ConfigureAwait(false);
logger.Debug("step 5: verify postback url in SAML reponse");
VerifyPostbackUrl();
logger.Debug("step 6: send SAML reponse to snowflake to login");
await base.LoginAsync(cancellationToken);
}
/// <see cref="IAuthenticator"/>
async Task IAuthenticator.AuthenticateAsync(CancellationToken cancellationToken)
{
logger.Info("Okta Authentication");
logger.Debug("step 1: get sso and token url");
var authenticatorRestRequest = BuildAuthenticatorRestRequest();
var authenticatorResponse = await session.restRequester.PostAsync <AuthenticatorResponse>(authenticatorRestRequest, cancellationToken);
authenticatorResponse.FilterFailedResponse();
Uri ssoUrl = new Uri(authenticatorResponse.data.ssoUrl);
Uri tokenUrl = new Uri(authenticatorResponse.data.tokenUrl);
logger.Debug("step 2: verify urls fetched from step 1");
logger.Debug("Checking sso url");
VerifyUrls(ssoUrl, oktaUrl);
logger.Debug("Checking token url");
VerifyUrls(tokenUrl, oktaUrl);
logger.Debug("step 3: get idp onetime token");
IdpTokenRestRequest idpTokenRestRequest = BuildIdpTokenRestRequest(tokenUrl);
var idpResponse = await session.restRequester.PostAsync <IdpTokenResponse>(idpTokenRestRequest, cancellationToken);
string onetimeToken = idpResponse.CookieToken;
logger.Debug("step 4: get SAML reponse from sso");
var samlRestRequest = BuildSAMLRestRequest(ssoUrl, onetimeToken);
var samlRawResponse = await session.restRequester.GetAsync(samlRestRequest, cancellationToken);
var samlRawHtmlString = await samlRawResponse.Content.ReadAsStringAsync();
logger.Debug("step 5: verify postback url in SAML reponse");
VerifyPostbackUrl(samlRawHtmlString);
logger.Debug("step 6: send SAML reponse to snowflake to login");
var loginRestRequest = BuildOktaLoginRestRequest(samlRawHtmlString);
var authnResponse = await session.restRequester.PostAsync <LoginResponse>(loginRestRequest, cancellationToken);
session.ProcessLoginResponse(authnResponse);
}
public SFBlockingChunkDownloader(int colCount,
List <ExecResponseChunk> chunkInfos, string qrmk,
Dictionary <string, string> chunkHeaders,
CancellationToken cancellationToken,
SFBaseResultSet ResultSet)
{
this.qrmk = qrmk;
this.chunkHeaders = chunkHeaders;
this.chunks = new List <SFResultChunk>();
this.nextChunkToDownloadIndex = 0;
this.ResultSet = ResultSet;
this.prefetchThreads = GetPrefetchThreads(ResultSet);
var idx = 0;
foreach (ExecResponseChunk chunkInfo in chunkInfos)
{
this.chunks.Add(new SFResultChunk(chunkInfo.url, chunkInfo.rowCount, colCount, idx++));
}
logger.Info($"Total chunk number: {chunks.Count}");
FillDownloads();
}
/// <summary>
/// </summary>
/// <returns>index of column given a name, -1 if no column names are found</returns>
internal int getColumnIndexByName(string targetColumnName)
{
int resultIndex;
if (columnNameToIndexCache.TryGetValue(targetColumnName, out resultIndex))
{
return(resultIndex);
}
else
{
int indexCounter = 0;
foreach (ExecResponseRowType rowType in rowTypes)
{
if (String.Compare(rowType.name, targetColumnName, false) == 0)
{
logger.Info($"Found colun name {targetColumnName} under index {indexCounter}");
columnNameToIndexCache[targetColumnName] = indexCounter;
return(indexCounter);
}
indexCounter++;
}
}
return(-1);
}
internal static SFSessionProperties parseConnectionString(String connectionString, SecureString password)
{
logger.Info("Start parsing connection string.");
SFSessionProperties properties = new SFSessionProperties();
string[] propertyEntry = connectionString.Split(';');
foreach (string keyVal in propertyEntry)
{
if (keyVal.Length > 0)
{
string[] tokens = keyVal.Split(new string[] { "=" }, StringSplitOptions.None);
if (tokens.Length != 2)
{
// https://docs.microsoft.com/en-us/dotnet/api/system.data.oledb.oledbconnection.connectionstring
// To include an equal sign (=) in a keyword or value, it must be preceded
// by another equal sign. For example, in the hypothetical connection
// string "key==word=value" :
// the keyword is "key=word" and the value is "value".
int currentIndex = 0;
int singleEqualIndex = -1;
while (currentIndex <= keyVal.Length)
{
currentIndex = keyVal.IndexOf("=", currentIndex);
if (-1 == currentIndex)
{
// No '=' found
break;
}
if ((currentIndex < (keyVal.Length - 1)) &&
('=' != keyVal[currentIndex + 1]))
{
if (0 > singleEqualIndex)
{
// First single '=' encountered
singleEqualIndex = currentIndex;
currentIndex++;
}
else
{
// Found another single '=' which is not allowed
singleEqualIndex = -1;
break;
}
}
else
{
// skip the doubled one
currentIndex += 2;
}
}
if ((singleEqualIndex > 0) && (singleEqualIndex < keyVal.Length - 1))
{
// Split the key/value at the right index and deduplicate '=='
tokens = new string[2];
tokens[0] = keyVal.Substring(0, singleEqualIndex).Replace("==", "=");
tokens[1] = keyVal.Substring(
singleEqualIndex + 1,
keyVal.Length - (singleEqualIndex + 1)).Replace("==", "=");;
}
else
{
// An equal sign was not doubled or something else happened
// making the connection invalid
string invalidStringDetail =
String.Format("Invalid key value pair {0}", keyVal);
SnowflakeDbException e =
new SnowflakeDbException(SFError.INVALID_CONNECTION_STRING,
new object[] { invalidStringDetail });
logger.Error("Invalid string.", e);
throw e;
}
}
try
{
SFSessionProperty p = (SFSessionProperty)Enum.Parse(
typeof(SFSessionProperty), tokens[0].ToUpper());
properties.Add(p, tokens[1]);
logger.Info($"Connection property: {p}, value: {(secretProps.Contains(p) ? "XXXXXXXX" : tokens[1])}");
}
catch (ArgumentException e)
{
logger.Warn($"Property {tokens[0]} not found ignored.", e);
}
}
}
if (password != null)
{
properties[SFSessionProperty.PASSWORD] = new NetworkCredential(string.Empty, password).Password;
}
checkSessionProperties(properties);
// compose host value if not specified
if (!properties.ContainsKey(SFSessionProperty.HOST) ||
(0 == properties[SFSessionProperty.HOST].Length))
{
string hostName = String.Format("{0}.snowflakecomputing.com", properties[SFSessionProperty.ACCOUNT]);
// Remove in case it's here but empty
properties.Remove(SFSessionProperty.HOST);
properties.Add(SFSessionProperty.HOST, hostName);
logger.Info($"Compose host name: {hostName}");
}
return(properties);
}
protected override async Task <HttpResponseMessage> SendAsync(HttpRequestMessage requestMessage,
CancellationToken cancellationToken)
{
HttpResponseMessage response = null;
int backOffInSec = 1;
TimeSpan httpTimeout = (TimeSpan)requestMessage.Properties["TIMEOUT_PER_HTTP_REQUEST"];
CancellationTokenSource childCts = null;
while (true)
{
try
{
childCts = null;
if (!httpTimeout.Equals(Timeout.InfiniteTimeSpan))
{
childCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
childCts.CancelAfter(httpTimeout);
}
response = await base.SendAsync(requestMessage, childCts == null?
cancellationToken : childCts.Token);
}
catch (Exception e)
{
if (cancellationToken.IsCancellationRequested)
{
logger.Debug("SF rest request timeout.");
cancellationToken.ThrowIfCancellationRequested();
}
else if (childCts != null && childCts.Token.IsCancellationRequested)
{
logger.Warn("Http request timeout. Retry the request");
}
else
{
//TODO: Should probably check to see if the error is recoverable or transient.
logger.Warn("Error occurred during request, retrying...", e);
}
}
if (response != null)
{
if (response.IsSuccessStatusCode)
{
logger.Debug($"Success Response: {response.ToString()}");
return(response);
}
logger.Debug($"Failed Response: {response.ToString()}");
}
else
{
logger.Info("Response returned was null.");
}
logger.Debug($"Sleep {backOffInSec} seconds and then retry the request");
Thread.Sleep(backOffInSec * 1000);
backOffInSec = backOffInSec >= 16 ? 16 : backOffInSec * 2;
}
}
protected override async Task <HttpResponseMessage> SendAsync(HttpRequestMessage requestMessage,
CancellationToken cancellationToken)
{
HttpResponseMessage response = null;
int backOffInSec = 1;
int totalRetryTime = 0;
int maxDefaultBackoff = 16;
ServicePoint p = ServicePointManager.FindServicePoint(requestMessage.RequestUri);
p.Expect100Continue = false; // Saves about 100 ms per request
p.UseNagleAlgorithm = false; // Saves about 200 ms per request
p.ConnectionLimit = 20; // Default value is 2, we need more connections for performing multiple parallel queries
TimeSpan httpTimeout = (TimeSpan)requestMessage.Properties[SFRestRequest.HTTP_REQUEST_TIMEOUT_KEY];
TimeSpan restTimeout = (TimeSpan)requestMessage.Properties[SFRestRequest.REST_REQUEST_TIMEOUT_KEY];
if (logger.IsDebugEnabled())
{
logger.Debug("Http request timeout : " + httpTimeout);
logger.Debug("Rest request timeout : " + restTimeout);
}
CancellationTokenSource childCts = null;
UriUpdater updater = new UriUpdater(requestMessage.RequestUri);
while (true)
{
try
{
childCts = null;
if (!httpTimeout.Equals(Timeout.InfiniteTimeSpan))
{
childCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
childCts.CancelAfter(httpTimeout);
}
response = await base.SendAsync(requestMessage, childCts == null?
cancellationToken : childCts.Token).ConfigureAwait(false);
}
catch (Exception e)
{
if (cancellationToken.IsCancellationRequested)
{
logger.Debug("SF rest request timeout or explicit cancel called.");
cancellationToken.ThrowIfCancellationRequested();
}
else if (childCts != null && childCts.Token.IsCancellationRequested)
{
logger.Warn("Http request timeout. Retry the request");
totalRetryTime += (int)httpTimeout.TotalSeconds;
}
else
{
//TODO: Should probably check to see if the error is recoverable or transient.
logger.Warn("Error occurred during request, retrying...", e);
}
}
if (response != null)
{
if (response.IsSuccessStatusCode)
{
return(response);
}
else
{
logger.Debug($"Failed Response: {response.ToString()}");
bool isRetryable = isRetryableHTTPCode((int)response.StatusCode);
if (!isRetryable)
{
// No need to keep retrying, stop here
return(response);
}
}
}
else
{
logger.Info("Response returned was null.");
}
requestMessage.RequestUri = updater.Update();
logger.Debug($"Sleep {backOffInSec} seconds and then retry the request");
Thread.Sleep(backOffInSec * 1000);
totalRetryTime += backOffInSec;
backOffInSec = backOffInSec >= maxDefaultBackoff ?
maxDefaultBackoff : backOffInSec * 2;
if (totalRetryTime + backOffInSec > restTimeout.TotalSeconds)
{
// No need to wait more than necessary if it can be avoided.
// If the rest timeout will be reached before the next back-off,
// use a smaller one to give the Rest request a chance to timeout early
backOffInSec = Math.Max(1, (int)restTimeout.TotalSeconds - totalRetryTime - 1);
}
}
}
protected override async Task <HttpResponseMessage> SendAsync(HttpRequestMessage requestMessage,
CancellationToken cancellationToken)
{
HttpResponseMessage response = null;
int backOffInSec = 1;
ServicePoint p = ServicePointManager.FindServicePoint(requestMessage.RequestUri);
p.Expect100Continue = false; // Saves about 100 ms per request
p.UseNagleAlgorithm = false; // Saves about 200 ms per request
p.ConnectionLimit = 20; // Default value is 2, we need more connections for performing multiple parallel queries
TimeSpan httpTimeout = (TimeSpan)requestMessage.Properties["TIMEOUT_PER_HTTP_REQUEST"];
CancellationTokenSource childCts = null;
UriUpdater updater = new UriUpdater(requestMessage.RequestUri);
while (true)
{
try
{
childCts = null;
if (!httpTimeout.Equals(Timeout.InfiniteTimeSpan))
{
childCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
childCts.CancelAfter(httpTimeout);
}
response = await base.SendAsync(requestMessage, childCts == null?
cancellationToken : childCts.Token).ConfigureAwait(false);
}
catch (Exception e)
{
if (cancellationToken.IsCancellationRequested)
{
logger.Debug("SF rest request timeout.");
cancellationToken.ThrowIfCancellationRequested();
}
else if (childCts != null && childCts.Token.IsCancellationRequested)
{
logger.Warn("Http request timeout. Retry the request");
}
else
{
//TODO: Should probably check to see if the error is recoverable or transient.
logger.Warn("Error occurred during request, retrying...", e);
}
}
if (response != null)
{
if (response.IsSuccessStatusCode)
{
return(response);
}
logger.Debug($"Failed Response: {response.ToString()}");
}
else
{
logger.Info("Response returned was null.");
}
requestMessage.RequestUri = updater.Update();
logger.Debug($"Sleep {backOffInSec} seconds and then retry the request");
Thread.Sleep(backOffInSec * 1000);
backOffInSec = backOffInSec >= 16 ? 16 : backOffInSec * 2;
}
}