Esempio n. 1
0
 public ShardMap(
     Executor executor,
     KPLNETConfiguration config,
     AwsHttpClient http_client,
     AwsKinesisClient kinesis_client,
     AwsCredentialsProvider creds,
     string region,
     string stream,
     MetricsManager metrics_manager,
     int min_backoff = kMinBackoff,
     int max_backoff = kMaxBackoff)
 {
     this.executor        = executor;
     this.http_client     = http_client;
     this.kinesis_client  = kinesis_client;
     this.creds_provider  = creds;
     this.config          = config;
     this.region          = region;
     this.stream          = stream;
     this.metrics_manager = metrics_manager;
     this.state           = State.INVALID;
     this.min_backoff     = min_backoff;
     this.max_backoff     = max_backoff;
     this.backoff         = min_backoff;
     update();
 }
Esempio n. 2
0
        static KeyValuePair <AwsCredentialsProvider, AwsCredentialsProvider> get_creds_providers(Executor executor, Ec2Metadata ec2_md, KPLNETConfiguration config)
        {
            AwsCredentialsProvider creds_provider         = null;
            AwsCredentialsProvider metrics_creds_provider = null;

            if (null != config.AWSCredentials)
            {
                creds_provider = new BasicAwsCredentialsProvider(config.AWSCredentials.Akid, config.AWSCredentials.SecretKey, config.AWSCredentials.SessionToken);
                if (null != config.AWSMetricsCredentials)
                {
                    metrics_creds_provider = new BasicAwsCredentialsProvider(config.AWSMetricsCredentials.Akid, config.AWSMetricsCredentials.SecretKey, config.AWSMetricsCredentials.SessionToken);
                }
                else
                {
                    metrics_creds_provider = creds_provider;
                }
            }
            else
            {
                creds_provider = new DefaultAwsCredentialsProviderChain(executor, ec2_md);
                Thread.Sleep(250);
            }

            if (null == creds_provider.get_credentials())
            {
                StdErrorOut.Instance.StdError("Could not retrieve credentials from anywhere.");
                throw new Exception("Could not retrieve credentials from anywhere.");
            }

            return(new KeyValuePair <AwsCredentialsProvider, AwsCredentialsProvider>(creds_provider, metrics_creds_provider));
        }
Esempio n. 3
0
        static KeyValuePair <AwsCredentialsProvider, AwsCredentialsProvider> get_creds_providers(Executor executor, Ec2Metadata ec2_md, List <SetCredentials> set_creds)
        {
            AwsCredentialsProvider creds_provider         = null;
            AwsCredentialsProvider metrics_creds_provider = null;

            foreach (var sc in set_creds)
            {
                var cp = new BasicAwsCredentialsProvider(sc.Credentials.Akid, sc.Credentials.SecretKey, sc.Credentials.Token);
                if (sc.ForMetrics)
                {
                    metrics_creds_provider = cp;
                }
                else
                {
                    creds_provider = cp;
                }
            }

            if (null == creds_provider)
            {
                creds_provider = new DefaultAwsCredentialsProviderChain(executor, ec2_md);
                Thread.Sleep(250);
            }

            if (null == creds_provider.get_credentials())
            {
                StdErrorOut.Instance.StdError("Could not retrieve credentials from anywhere.");
                throw new Exception("Could not retrieve credentials from anywhere.");
            }

            if (null == metrics_creds_provider)
            {
                metrics_creds_provider = creds_provider;
            }

            return(new KeyValuePair <AwsCredentialsProvider, AwsCredentialsProvider>(creds_provider, metrics_creds_provider));
        }
Esempio n. 4
0
        public KinesisProducer(IMessageManager ipc_manager, ISocketFactory socket_factory, string region, KPLNETConfiguration config, AwsCredentialsProvider creds_provider, AwsCredentialsProvider metrics_creds_provider, Executor executor)
        {
            region_         = region;
            config_         = config;
            socket_factory_ = socket_factory;
            executor_       = executor;
            ipc_manager_    = ipc_manager;
            creds_chain_    = AwsCredentialsProviderChain.create(new List <AwsCredentialsProvider> {
                creds_provider
            });
            metrics_creds_chain_ = metrics_creds_provider == creds_provider ? creds_chain_ : AwsCredentialsProviderChain.create(new List <AwsCredentialsProvider> {
                metrics_creds_provider
            });
            pipelines_ = new ConcurrentDictionary <string, Pipeline>();
            shutdown_  = false;

            create_metrics_manager();

            if (config.clientType == KPLNETInterface.ClientType.SocketClient)
            {
                create_http_client();
            }
            else
            {
                create_kinesis_client();
            }

            report_outstanding();

            message_drainer_ = new Thread(() => { drain_messages(); });
            message_drainer_.Start();
        }
Esempio n. 5
0
        public MetricsManager(Executor executor, ISocketFactory socket_factory, AwsCredentialsProvider creds, KPLNETConfiguration config, string region, string cw_namespace = "KinesisProducerLib",
                              Level level                      = Level.Detailed,
                              Granularity granularity          = Granularity.Shard,
                              ExtraDimensions extra_dimensions = null,
                              string custom_endpoint           = "",
                              ulong port = 443,
                              double upload_frequency = 1 * 60 * 1000,
                              double retry_frequency  = 10 * 1000)
        {
            if (extra_dimensions == null)
            {
                extra_dimensions = new ExtraDimensions();
            }
            this.creds            = creds;
            this.config_          = config;
            this.region           = region;
            this.metrics_index    = new MetricsIndex();
            region_               = Core.Clients.KinesisClient.AwsKinesisClient.MapRegion(config.region);
            this.cw_namespace     = cw_namespace;
            this.socket_factory_  = socket_factory;
            this.level            = level;
            this.granularity      = granularity;
            this.upload_frequency = upload_frequency;
            this.retry_frequency  = retry_frequency;
            this.executor         = executor;
            this.endpoint         = string.IsNullOrEmpty(custom_endpoint) ? "monitoring." + region + ".amazonaws.com" : custom_endpoint;
            this.http_client      = new AwsHttpClient(executor, socket_factory_, endpoint, (int)port, true, string.IsNullOrEmpty(custom_endpoint));
            retryable_requests    = new List <string>();
            if (level != Level.None)
            {
                StdErrorOut.Instance.StdOut(LogLevel.warn, string.Format("Uploading metrics to {0}:{1}", endpoint, port));
            }

            extra_dimensionsMap[Granularity.Global] = new Dimensions();
            extra_dimensionsMap[Granularity.Stream] = new Dimensions();
            extra_dimensionsMap[Granularity.Shard]  = new Dimensions();

            foreach (var t in extra_dimensions)
            {
                extra_dimensionsMap[t.Item3].Add(new KeyValuePair <string, string>(t.Item1, t.Item2));
            }

            scheduled_upload = executor.Schedule(
                () =>
            {
                try
                {
                    scheduled_upload.reschedule(upload_frequency);
                    upload();
                }
                catch (Exception ex)
                {
                    StdErrorOut.Instance.StdError("Scheduled Upload failed.", ex);
                }
            },
                upload_frequency);

            scheduled_retry = executor.Schedule(
                () =>
            {
                try
                {
                    scheduled_retry.reschedule(retry_frequency);
                    retry_uploads();
                }
                catch (Exception ex)
                {
                    StdErrorOut.Instance.StdError("Scheduled Upload failed.", ex);
                }
            },
                retry_frequency);
        }
Esempio n. 6
0
 public Pipeline(string region, string stream, KPLNETConfiguration config, Executor executor, AwsHttpClient http_client, AwsKinesisClient kinesis_client, AwsCredentialsProvider creds_provider, MetricsManager metrics_manager, Action <UserRecord> finish_user_record_cb)
 {
     StdErrorOut.Instance.StdOut(LogLevel.debug, "started pipeline creation.");
     try
     {
         this.region_                = region;
         this.config_                = config;
         this.executor_              = executor;
         this.http_client_           = http_client;
         this.kinesis_client_        = kinesis_client;
         this.metrics_manager_       = metrics_manager;
         this.finish_user_record_cb_ = finish_user_record_cb;
         this.sig_v4_ctx_            = new SigV4Context(this.region_, "kinesis", creds_provider);
         this.shard_map_             = new ShardMap(executor, config, http_client, kinesis_client, creds_provider, this.region_, stream, this.metrics_manager_, 1000, 30000);
         StdErrorOut.Instance.StdOut(LogLevel.debug, "after shard_map_ pipeline creation.");
         this.aggregator_ = new Aggregator(this.executor_, this.shard_map_, (Action <KinesisRecord>)(kr => this.limiter_put(kr)), this.config_, this.metrics_manager_);
         this.limiter_    = new Limiter(this.executor_, (Action <KinesisRecord>)(kr => this.collector_put(kr)), (Action <KinesisRecord>)(kr => this.retrier_put_kr(kr)), this.config_);
         this.collector_  = new Collector(this.executor_, (Action <PutRecordsRequest>)(prr => this.send_put_records_request(prr)), this.config_, this.metrics_manager_);
         this.retrier_    = new Retrier(this.config_, (Action <UserRecord>)(ur => this.finish_user_record(ur)), (Action <UserRecord>)(ur => this.aggregator_put(ur)), (Action <DateTime>)(tp => this.shard_map_.invalidate(tp)), (Action <string, string>)((code, msg) => this.limiter_.add_error(code, msg)), this.metrics_manager_);
         this.outstanding_user_records_ = 0;
         StdErrorOut.Instance.StdOut(LogLevel.debug, "done pipeline creation.");
     }
     catch (Exception ex)
     {
         StdErrorOut.Instance.StdOut(LogLevel.error, ex.ToString());
     }
 }
Esempio n. 7
0
 static KinesisProducer get_kp(IMessageManager ipc_manager, ISocketFactory socket_factory, string region, KPLNETConfiguration config, AwsCredentialsProvider creds_provider, AwsCredentialsProvider metrics_creds_provider, Executor executor)
 {
     try
     {
         return(new KinesisProducer(ipc_manager, socket_factory, region, config, creds_provider, metrics_creds_provider, executor));
     }
     catch (Exception ex)
     {
         StdErrorOut.Instance.StdError("KPLNET Service get_kp failed.", ex);
         throw new Exception("get_kp failed.", ex);
     }
 }