public virtual WfResult Run(CancellationToken token) { WfResult result = WfResult.Unknown; //_logger.Write(String.Format("SqlServer: {0} query: {1}", _attributes[CONNECTION_STRING], _attributes[QUERY_STRING])); int runId = 0; Int32.TryParse(_attributes[ETL_RUNID], out runId); int batchId = 0; Int32.TryParse(_attributes[ETL_BATCHID], out batchId); int stepId = 0; Int32.TryParse(_attributes[ETL_STEPID], out stepId); ControllerCounter counter = new ControllerCounter(_attributes[CONTROLLER_CONNECTION_STRING], _logger) { BatchId = batchId, StepId = stepId, RunId = runId }; Dictionary <string, string> controlset = new Dictionary <string, string>(); controlset.Add(_attributes[COUNTER_NAME], GetControlValue(token)); counter.SetCounters(controlset); return(WfResult.Succeeded); }
public virtual WfResult Run(CancellationToken token) { WfResult result = WfResult.Unknown; //_logger.Write(String.Format("SqlServer: {0} query: {1}", _attributes[CONNECTION_STRING], _attributes[QUERY_STRING])); int runId = 0; Int32.TryParse(_attributes[ETL_RUNID], out runId); int batchId = 0; Int32.TryParse(_attributes[ETL_BATCHID], out batchId); int stepId = 0; Int32.TryParse(_attributes[ETL_STEPID], out stepId); string input = _attributes[FILE_PATH]; ControllerCounter counter = new ControllerCounter(_attributes[CONNECTION_STRING], _logger) { BatchId = batchId, StepId = stepId, RunId = runId }; int id = 0; Dictionary <string, string> files = Directory.GetFiles(Path.GetDirectoryName(input), Path.GetFileName(input), SearchOption.TopDirectoryOnly) .ToDictionary(k => String.Format("file_{0}", id++), v => v); counter.SetCounters(files); return(WfResult.Succeeded); }
public WfResult Run(CancellationToken token) { WfResult result = WfResult.Unknown; //_logger.Write(String.Format("SqlServer: {0} query: {1}", _attributes[CONNECTION_STRING], _attributes[QUERY_STRING])); int runId = 0; Int32.TryParse(_attributes[ETL_RUNID], out runId); int batchId = 0; Int32.TryParse(_attributes[ETL_BATCHID], out batchId); int stepId = 0; Int32.TryParse(_attributes[ETL_STEPID], out stepId); bool setCounterInd = !String.IsNullOrEmpty(_attributes[COUNTER_NAME]); int count = 0; Int32.TryParse(_attributes[COUNT], out count); if (count <= 0) { count = MAX_COUNT; } ControllerCounter counter = new ControllerCounter(_attributes[CONNECTION_STRING], _logger) { BatchId = batchId, StepId = stepId, RunId = runId }; try { CloudStorageAccount account; if (Boolean.Parse(_attributes[IS_SAS_TOKEN])) { StorageCredentials credentials = new StorageCredentials(_attributes[ACCOUNT_KEY]); account = new CloudStorageAccount(credentials, _attributes[ACCOUNT_NAME], endpointSuffix: null, useHttps: true); } else { StorageCredentials credentials = new StorageCredentials(_attributes[ACCOUNT_NAME], _attributes[ACCOUNT_KEY]); account = new CloudStorageAccount(credentials, useHttps: true); } CloudFileClient fileClient = account.CreateCloudFileClient(); var fileShare = fileClient.GetShareReference(_attributes[FILE_SHARE_NAME]); if (!fileShare.Exists()) { throw new ArgumentException(String.Format("File share {0} is not found", fileShare.Name)); } string prefix = Path.GetFileName(_attributes[INPUT_PREFIX]); string dir = Path.GetDirectoryName(_attributes[INPUT_PREFIX] + ".filler"); CloudFileDirectory rootDir = fileShare.GetRootDirectoryReference(); if (!fileShare.Exists()) { throw new ArgumentException(String.Format("File share {0} is not found", fileShare.Name)); } bool bModified = !String.IsNullOrEmpty(_attributes[INPUT_MODIFIED]); DateTimeOffset modified = DateTimeOffset.MinValue; if (bModified && !DateTimeOffset.TryParse(_attributes[INPUT_MODIFIED], out modified)) { throw new ArgumentException(String.Format("Invalid Modified filter {0}", _attributes[INPUT_MODIFIED])); } CloudFileDirectory leafDir = TraverseDirectoryTree(dir, rootDir); var list = leafDir.ListFilesAndDirectories(); if (!String.IsNullOrEmpty(prefix)) { list = list.OfType <CloudFile>().Where(l => l.Name.StartsWith(prefix, StringComparison.InvariantCultureIgnoreCase)); } if (bModified) { list = list.OfType <CloudFile>().Where(l => l.Properties.LastModified <= modified); } if (_attributes[SORT_ORDER].Equals(sortList[0], StringComparison.InvariantCultureIgnoreCase)) { list = list.OfType <CloudFile>().OrderBy(l => l.Name); } else if (_attributes[SORT_ORDER].Equals(sortList[1], StringComparison.InvariantCultureIgnoreCase)) { list = list.OfType <CloudFile>().OrderByDescending(l => l.Name); } if (count > 0) { list = list.OfType <CloudFile>().Take(count); } Dictionary <string, string> files = new Dictionary <string, string>(); Directory.CreateDirectory(_attributes[OUTPUT_FOLDER]); int i = 0; foreach (var fileItem in list.OfType <CloudFile>()) { token.ThrowIfCancellationRequested(); string outputFile = Path.Combine(_attributes[OUTPUT_FOLDER], fileItem.Name); if (!bModified && File.Exists(outputFile)) { continue; } //CloudFile cloudFile = leafDir.GetFileReference(fileItem); ; fileItem.DownloadToFile(outputFile, FileMode.OpenOrCreate); if (setCounterInd) { files.Add(String.Format("{0}_{1}", _attributes[COUNTER_NAME], i++), outputFile); } } if (setCounterInd) { counter.SetCounters(files); } } catch (Exception ex) { throw ex; } result = WfResult.Succeeded; return(result); }
public WfResult Run(CancellationToken token) { WfResult result = WfResult.Unknown; //_logger.Write(String.Format("SqlServer: {0} query: {1}", _attributes[CONNECTION_STRING], _attributes[QUERY_STRING])); int runId = 0; Int32.TryParse(_attributes[ETL_RUNID], out runId); int batchId = 0; Int32.TryParse(_attributes[ETL_BATCHID], out batchId); int stepId = 0; Int32.TryParse(_attributes[ETL_STEPID], out stepId); bool setCounterInd = !String.IsNullOrEmpty(_attributes[COUNTER_NAME]); int count = 0; Int32.TryParse(_attributes[COUNT], out count); if (count <= 0) { count = MAX_COUNT; } ControllerCounter counter = new ControllerCounter(_attributes[CONNECTION_STRING], _logger) { BatchId = batchId, StepId = stepId, RunId = runId }; try { CloudStorageAccount account; if (Boolean.Parse(_attributes[IS_SAS_TOKEN])) { StorageCredentials credentials = new StorageCredentials(_attributes[ACCOUNT_KEY]); account = new CloudStorageAccount(credentials, _attributes[ACCOUNT_NAME], endpointSuffix: null, useHttps: true); } else { StorageCredentials credentials = new StorageCredentials(_attributes[ACCOUNT_NAME], _attributes[ACCOUNT_KEY]); account = new CloudStorageAccount(credentials, useHttps: true); } CloudBlobClient blobClient = account.CreateCloudBlobClient(); //var container_list = blobClient.ListContainers(); //List<string> containerItems = container_list.OfType<CloudBlobContainer>().Select(b => b.Name).ToList(); CloudBlobContainer container = blobClient.GetContainerReference(_attributes[CONTAINER_NAME]); var list = container.ListBlobs(_attributes[INPUT_PREFIX], true); //List<string> blobParents = list.OfType<CloudBlob>().Select(b => b.Parent.Prefix).Distinct().ToList(); if (_attributes[SORT_ORDER].Equals(sortList[0], StringComparison.InvariantCultureIgnoreCase)) { list = list.OfType <CloudBlob>().OrderBy(b => b.Name); } if (_attributes[SORT_ORDER].Equals(sortList[1], StringComparison.InvariantCultureIgnoreCase)) { list = list.OfType <CloudBlob>().OrderByDescending(b => b.Name); } if (count > 0) { list = list.OfType <CloudBlob>().Take(count); } Dictionary <string, string> files = new Dictionary <string, string>(); Directory.CreateDirectory(_attributes[OUTPUT_FOLDER]); int i = 0; foreach (var blobItem in list.OfType <CloudBlob>()) { token.ThrowIfCancellationRequested(); string outputFile = Path.Combine(_attributes[OUTPUT_FOLDER], blobItem.Name.Replace('/', '-')); if (File.Exists(outputFile)) { continue; } //CloudBlob blob = container.GetBlobReference(blobItem); blobItem.DownloadToFile(outputFile, FileMode.OpenOrCreate); if (setCounterInd) { files.Add(String.Format("{0}_{1}", _attributes[COUNTER_NAME], i++), outputFile); } } if (setCounterInd) { counter.SetCounters(files); } } catch (Exception ex) { throw ex; } result = WfResult.Succeeded; return(result); }
public virtual WfResult Run(CancellationToken token) { WfResult result = WfResult.Unknown; //_logger.Write(String.Format("SqlServer: {0} query: {1}", _attributes[CONNECTION_STRING], _attributes[QUERY_STRING])); int runId = 0; Int32.TryParse(_attributes[ETL_RUNID], out runId); int batchId = 0; Int32.TryParse(_attributes[ETL_BATCHID], out batchId); int stepId = 0; Int32.TryParse(_attributes[ETL_STEPID], out stepId); Dictionary <string, string> files = new Dictionary <string, string>(); ControllerCounter counter = new ControllerCounter(_attributes[CONNECTION_STRING], _logger) { BatchId = batchId, StepId = stepId, RunId = runId }; SqlConnection cn = new SqlConnection(_attributes[CONNECTION_STRING_REGISTER]); try { cn.Open(); using (SqlCommand cmd = new SqlCommand(GET_LIST_QUERY, cn)) { cmd.CommandType = CommandType.StoredProcedure; cmd.CommandTimeout = Int32.Parse(_attributes[TIMEOUT]); cmd.Parameters.AddWithValue("@processId", runId); cmd.Parameters.AddWithValue("@sourceName", _attributes[FILE_SOURCE]); cmd.Parameters.AddWithValue("@count", 1); using (token.Register(cmd.Cancel)) { var reader = cmd.ExecuteReader(); if (reader.HasRows) { while (reader.Read()) { //_logger.WriteDebug(String.Format("File to process id: {0}, name: {1}", reader["fileId"], reader["fullName"])); files.Add("fileId", reader["fileId"].ToString()); files.Add(String.Format("fileName_{0}", reader["fileId"]), reader["fullName"].ToString()); } } else { _logger.Debug("No files to process"); files.Add("fileId", "0"); } result = WfResult.Succeeded; } } } catch (SqlException ex) { throw ex; //_logger.Write(String.Format("SqlServer exception: {0}", ex.Message)); //result = WfResult.Create(WfStatus.Failed, ex.Message, ex.ErrorCode); } finally { if (cn.State != ConnectionState.Closed) { cn.Close(); } } counter.SetCounters(files); return(result); }
public WfResult Run(CancellationToken token) { WfResult result = WfResult.Unknown; //_logger.Write(String.Format("SqlServer: {0} query: {1}", _attributes[CONNECTION_STRING], _attributes[QUERY_STRING])); int runId = 0; Int32.TryParse(_attributes[ETL_RUNID], out runId); int batchId = 0; Int32.TryParse(_attributes[ETL_BATCHID], out batchId); int stepId = 0; Int32.TryParse(_attributes[ETL_STEPID], out stepId); bool setCounterInd = !String.IsNullOrEmpty(_attributes[COUNTER_NAME]); int count = 0; Int32.TryParse(_attributes[COUNT], out count); if (count <= 0) { count = MAX_COUNT; } ControllerCounter counter = new ControllerCounter(_attributes[CONNECTION_STRING], _logger) { BatchId = batchId, StepId = stepId, RunId = runId }; try { //if(!Amazon.Util.ProfileManager.IsProfileKnown(_attributes[PROFILE_NAME])) //{ // Amazon.Util.ProfileManager.RegisterProfile(_attributes[PROFILE_NAME], _attributes[ACCOUNT_NAME], _attributes[ACCOUNT_KEY]); //} //var credentials = new StoredProfileAWSCredentials(_attributes[PROFILE_NAME]); RegionEndpoint endpoint = RegionEndpoint.GetBySystemName(_attributes[REGION_NAME]); AWSCredentials credentials = null; if (!String.IsNullOrEmpty(_attributes[ACCOUNT_NAME])) { credentials = new BasicAWSCredentials(_attributes[ACCOUNT_NAME], _attributes[ACCOUNT_KEY]); } using (var client = new AmazonS3Client(credentials, endpoint)) { //ListBucketsRequest bucketRequest = new ListBucketsRequest(); //ListBucketsResponse backetResponse; //backetResponse = client.ListBuckets(bucketRequest); //var buckets = backetResponse.Buckets; //GetBucketLocationResponse bucketLocationResponse = client.GetBucketLocation(_attributes[CONTAINER_NAME]); ListObjectsV2Request objectRequest = new ListObjectsV2Request { BucketName = _attributes[CONTAINER_NAME], MaxKeys = 100, Prefix = String.IsNullOrEmpty(_attributes[INPUT_PREFIX]) ? null : _attributes[INPUT_PREFIX] }; List <S3Object> objectlist = new List <S3Object>(); ListObjectsV2Response objectResponse; do { objectResponse = client.ListObjectsV2(objectRequest); objectlist.AddRange(objectResponse.S3Objects); objectRequest.ContinuationToken = objectResponse.NextContinuationToken; } while (objectResponse.IsTruncated == true); IEnumerable <S3Object> list = objectlist; if (_attributes[SORT_ORDER].Equals(sortList[0], StringComparison.InvariantCultureIgnoreCase)) { list = list.OfType <S3Object>().OrderBy(b => b.Key); } if (_attributes[SORT_ORDER].Equals(sortList[1], StringComparison.InvariantCultureIgnoreCase)) { list = list.OfType <S3Object>().OrderByDescending(b => b.Key); } if (count > 0) { list = list.OfType <S3Object>().Take(count); } Dictionary <string, string> files = new Dictionary <string, string>(); Directory.CreateDirectory(_attributes[OUTPUT_FOLDER]); int i = 0; foreach (var blobItem in list.OfType <S3Object>()) { token.ThrowIfCancellationRequested(); string outputFile = Path.Combine(_attributes[OUTPUT_FOLDER], Path.GetFileName(blobItem.Key)); if (File.Exists(outputFile)) { continue; } GetObjectRequest blobRequest = new GetObjectRequest { BucketName = _attributes[CONTAINER_NAME], Key = blobItem.Key }; using (FileStream outputFileStream = File.Create(outputFile)) using (GetObjectResponse blobResponse = client.GetObject(blobRequest)) using (Stream responseStream = blobResponse.ResponseStream) //using (StreamReader reader = new StreamReader(responseStream)) { responseStream.CopyTo(outputFileStream); //responseBody = reader.ReadToEnd(); } _logger.Debug("downloaded: {File}", outputFile); if (setCounterInd) { files.Add(String.Format("{0}_{1}", _attributes[COUNTER_NAME], i++), outputFile); } } if (setCounterInd) { counter.SetCounters(files); } } } catch (Exception ex) { throw ex; } result = WfResult.Succeeded; return(result); }