static KPLNETConfiguration Get_config(string hex)
{
Message msg = null;
try
{
msg = hex.DeserializeAsMessage();
}
catch (Exception e)
{
StdErrorOut.Instance.StdError(string.Format("Could not deserialize config: {0}", e.ToString()));
}
if (msg.Configuration == null)
{
StdErrorOut.Instance.StdError("Protobuf message did not contain a Configuration message\n");
}
var config = new KPLNETConfiguration();
try
{
config.transfer_from_protobuf_msg(msg);
}
catch (Exception e)
{
StdErrorOut.Instance.StdError(string.Format("Error in config: {0}\n", e.ToString()));
throw e;
}
return(config);
}
public ShardMap(
Executor executor,
KPLNETConfiguration config,
AwsHttpClient http_client,
AwsKinesisClient kinesis_client,
AwsCredentialsProvider creds,
string region,
string stream,
MetricsManager metrics_manager,
int min_backoff = kMinBackoff,
int max_backoff = kMaxBackoff)
{
this.executor = executor;
this.http_client = http_client;
this.kinesis_client = kinesis_client;
this.creds_provider = creds;
this.config = config;
this.region = region;
this.stream = stream;
this.metrics_manager = metrics_manager;
this.state = State.INVALID;
this.min_backoff = min_backoff;
this.max_backoff = max_backoff;
this.backoff = min_backoff;
update();
}
public Pipeline(string region, string stream, KPLNETConfiguration config, Executor executor, AwsHttpClient http_client, AwsKinesisClient kinesis_client, AwsCredentialsProvider creds_provider, MetricsManager metrics_manager, Action <UserRecord> finish_user_record_cb)
{
StdErrorOut.Instance.StdOut(LogLevel.debug, "started pipeline creation.");
try
{
this.region_ = region;
this.config_ = config;
this.executor_ = executor;
this.http_client_ = http_client;
this.kinesis_client_ = kinesis_client;
this.metrics_manager_ = metrics_manager;
this.finish_user_record_cb_ = finish_user_record_cb;
this.sig_v4_ctx_ = new SigV4Context(this.region_, "kinesis", creds_provider);
this.shard_map_ = new ShardMap(executor, config, http_client, kinesis_client, creds_provider, this.region_, stream, this.metrics_manager_, 1000, 30000);
StdErrorOut.Instance.StdOut(LogLevel.debug, "after shard_map_ pipeline creation.");
this.aggregator_ = new Aggregator(this.executor_, this.shard_map_, (Action <KinesisRecord>)(kr => this.limiter_put(kr)), this.config_, this.metrics_manager_);
this.limiter_ = new Limiter(this.executor_, (Action <KinesisRecord>)(kr => this.collector_put(kr)), (Action <KinesisRecord>)(kr => this.retrier_put_kr(kr)), this.config_);
this.collector_ = new Collector(this.executor_, (Action <PutRecordsRequest>)(prr => this.send_put_records_request(prr)), this.config_, this.metrics_manager_);
this.retrier_ = new Retrier(this.config_, (Action <UserRecord>)(ur => this.finish_user_record(ur)), (Action <UserRecord>)(ur => this.aggregator_put(ur)), (Action <DateTime>)(tp => this.shard_map_.invalidate(tp)), (Action <string, string>)((code, msg) => this.limiter_.add_error(code, msg)), this.metrics_manager_);
this.outstanding_user_records_ = 0;
StdErrorOut.Instance.StdOut(LogLevel.debug, "done pipeline creation.");
}
catch (Exception ex)
{
StdErrorOut.Instance.StdOut(LogLevel.error, ex.ToString());
}
}
/**
* Start up a KinesisProducer instance.
*
* <p>
* Since this creates a child process, it is fairly expensive. Avoid
* creating more than one per application, unless putting to multiple
* regions at the same time. All streams in the same region can share the
* same instance.
*
* <p>
* All methods in KinesisProducer are thread-safe.
*
* @param config
* Configuration for the KinesisProducer. See the docs for that
* class for details.
*
* @see KinesisProducerConfiguration
*/
public KinesisDotNetProducer(ILogger log, KPLNETConfiguration config)
{
this.config = config;
this.log = log;
if (config.runAsDaemon)
{
extractBinaries();
}
if (config.AWSCredentials == null)
{
string akid = Environment.GetEnvironmentVariable(AWS_ACCESS_KEY_ID);
string sKey = Environment.GetEnvironmentVariable(AWS_SECRET_ACCESS_KEY);
if (akid == null || sKey == null)
{
throw new ArgumentNullException("AWSCredentials is null. AWS_ACCESS_KEY_ID or AWS_SECRET_KEY is not available in Environment variable. Either set the environment variables or pass AWSCredentials in the config object.");
}
config.AWSCredentials = new AWSCredentials()
{
Akid = akid, SecretKey = sKey
};
}
env = new Dictionary <string, string>()
{
{ "LD_LIBRARY_PATH", pathToLibDir },
{ "DYLD_LIBRARY_PATH", pathToLibDir },
{ "CA_DIR", pathToTmpDir }
};
child = KPLRepBuilder.GetKPLRep(pathToExecutable, OnMessage, OnError, pathToTmpDir, config, env, log);
}
public Collector(Executor executor, FlushCallback flush_callback, KPLNETConfiguration config, MetricsManager metrics_manager = null)
{
flush_callback_ = flush_callback;
reducer_ = new Reducer <KinesisRecord, PutRecordsRequest>(executor, (prr) => handle_flush(prr), config.collectionMaxSize, config.collectionMaxCount, (kr) => { return(should_flush(kr)); });
buffered_data_ = new ConcurrentDictionary <long, ulong>();
metrics_manager_ = metrics_manager ?? new NullMetricsManager();
}
public KinesisProducer(IMessageManager ipc_manager, ISocketFactory socket_factory, string region, KPLNETConfiguration config, AwsCredentialsProvider creds_provider, AwsCredentialsProvider metrics_creds_provider, Executor executor)
{
region_ = region;
config_ = config;
socket_factory_ = socket_factory;
executor_ = executor;
ipc_manager_ = ipc_manager;
creds_chain_ = AwsCredentialsProviderChain.create(new List <AwsCredentialsProvider> {
creds_provider
});
metrics_creds_chain_ = metrics_creds_provider == creds_provider ? creds_chain_ : AwsCredentialsProviderChain.create(new List <AwsCredentialsProvider> {
metrics_creds_provider
});
pipelines_ = new ConcurrentDictionary <string, Pipeline>();
shutdown_ = false;
create_metrics_manager();
if (config.clientType == KPLNETInterface.ClientType.SocketClient)
{
create_http_client();
}
else
{
create_kinesis_client();
}
report_outstanding();
message_drainer_ = new Thread(() => { drain_messages(); });
message_drainer_.Start();
}
public Limiter(Executor executor,
Callback callback,
Callback expired_callback,
KPLNETConfiguration config)
{
executor_ = executor;
callback_ = callback;
config_ = config;
expired_callback_ = expired_callback;
limiters_ = new ConcurrentDictionary <long, ShardLimiter>();
poll();
}
public Aggregator
(
Executor executor,
ShardMap shard_map,
Action <KinesisRecord> deadline_callback,
KPLNETConfiguration config,
MetricsManager metrics_manager = null
)
{
executor_ = executor;
shard_map_ = shard_map;
deadline_callback_ = deadline_callback;
config_ = config;
metrics_manager_ = metrics_manager ?? new NullMetricsManager();
reducers_ = new ReducerMap();
}
static string get_region(KPLNETConfiguration config, Ec2Metadata ec2_md)
{
if (AWSRegions.Unknown != config.region)
{
return(config.region.ToString());
}
var ec2_region = ec2_md.get_region();
if (null != ec2_region)
{
StdErrorOut.Instance.StdError("Could not configure the region. It was not given in the config and we were unable to retrieve it from EC2 metadata.");
throw new Exception("Could not configure the region. It was not given in the config and we were unable to retrieve it from EC2 metadata.");
}
return(ec2_region);
}
public Retrier(KPLNETConfiguration config, UserRecordCallback finish_cb, UserRecordCallback retry_cb, ShardMapInvalidateCallback shard_map_invalidate_cb, ErrorCallback error_cb = null, MetricsManager metrics_manager = null)
{
config_ = config;
finish_cb_ = finish_cb;
retry_cb_ = retry_cb;
shard_map_invalidate_cb_ = shard_map_invalidate_cb;
error_cb_ = error_cb;
metrics_manager_ = metrics_manager;
if (metrics_manager == null)
{
metrics_manager = new NullMetricsManager();
}
//if (error_cb == null)
// error_cb = new ErrorCallback();
}
static void UpdateConfigWithCredentials(KPLNETConfiguration config, KeyValuePair <AwsCredentialsProvider, AwsCredentialsProvider> creds_providers)
{
var cred1 = creds_providers.Key.get_credentials();
var cred2 = creds_providers.Value.get_credentials();
config.AWSCredentials = new AWSCredentials()
{
Akid = cred1.Access_key_id(),
SecretKey = cred1.Secret_key(),
SessionToken = cred1.Session_token()
};
config.AWSMetricsCredentials = new AWSCredentials()
{
Akid = cred2.Access_key_id(),
SecretKey = cred2.Secret_key(),
SessionToken = cred2.Session_token()
};
}
public KPLRep(Action <Message> messageHandler, Action <Exception> exceptionHandler, KPLNETConfiguration config, ILogger log)
{
this.exceptionHandler = exceptionHandler;
this.messageHandler = messageHandler;
this.log = log;
this.config = config;
this.messageManager = DirectMessageService.Instance.GetClientInterfaceMessageManager();
Task.Run(() =>
{
try
{
startChildProcess();
}
catch (Exception e)
{
HandleError("Error running child process", e, true);
}
});
}
public static void StartChildProcess(KPLNETConfiguration config, ILogger logger)
{
Ec2Metadata ec2_md = null;
try
{
SetDirectLogging(config.logLevel, logger);
var executor = get_executor();
var socket_factory = get_socket_factory();
var region = get_region(config, ec2_md);
var creds_providers = get_creds_providers(executor, ec2_md, config);
var ipc_manager = DirectMessageService.Instance.GetChildProcessMessageManager();
var kp = get_kp(ipc_manager, socket_factory, region, config, creds_providers.Key, creds_providers.Value, executor);
}
catch (Exception e)
{
StdErrorOut.Instance.StdError(e.ToString());
throw new Exception("Service Creation failed.", e);
}
}
/**
* Starts up the child process, connects to it, and beings sending and
* receiving messages.
*
* @param pathToExecutable
* Path to the binary that we will use to start up the child.
* @param handler
* Message handler that handles messages received from the child.
* @param workingDir
* Working directory. The KPL creates FIFO files; it will do so
* in this directory.
* @param config
* KPL configuration.
*/
public Daemon(String pathToExecutable, Action <Message> messageHandler, Action <Exception> exceptionHandler, String workingDir, KPLNETConfiguration config, Dictionary <String, String> environmentVariables, ILogger log)
{
this.environmentVariables = environmentVariables;
this.pathToExecutable = pathToExecutable;
this.exceptionHandler = exceptionHandler;
this.messageHandler = messageHandler;
this.workingDir = workingDir;
this.config = config;
this.log = log;
Task.Run(() =>
{
try
{
startChildProcess();
}
catch (Exception e)
{
fatalError("Error running child process", e);
}
});
}
void AppStart()
{
KPLNETConfiguration config = new KPLNETConfiguration()
{
runAsDaemon = false,
nativeExecutable = Properties.Settings.Default.DeamonAppPath,
region = AWSRegions.USWest2,
aggregationMaxCount = 100,
collectionMaxCount = 100,
logLevel = log.LogLevel.ToString().ToLower()
};
KinesisDotNetProducer kinesisDotNetProducer = new KinesisDotNetProducer(log, config);
string streamName = "KPLNETTest";
while (true)
{
Console.WriteLine("Enter Number Of messages to add. Enter X to exit.");
string uEntry = Console.ReadLine();
if (uEntry.ToLower() == "x")
{
break;
}
int batchesCount = 0;
if (int.TryParse(uEntry, out batchesCount))
{
RunForNumberOfBatches(batchesCount, streamName, kinesisDotNetProducer);
}
else
{
Console.WriteLine("Invalid entry");
}
}
}
public static IKPLRep GetKPLRep(string pathToExecutable, Action <Message> messageHandler, Action <Exception> exceptionHandler, String workingDir, KPLNETConfiguration config, Dictionary <String, String> environmentVariables, ILogger log)
{
if (config.runAsDaemon)
{
return(new Daemon(pathToExecutable, messageHandler, exceptionHandler, workingDir, config, environmentVariables, log));
}
else
{
return(new KPLRep(messageHandler, exceptionHandler, config, log));
}
}
public MetricsManager(Executor executor, ISocketFactory socket_factory, AwsCredentialsProvider creds, KPLNETConfiguration config, string region, string cw_namespace = "KinesisProducerLib",
Level level = Level.Detailed,
Granularity granularity = Granularity.Shard,
ExtraDimensions extra_dimensions = null,
string custom_endpoint = "",
ulong port = 443,
double upload_frequency = 1 * 60 * 1000,
double retry_frequency = 10 * 1000)
{
if (extra_dimensions == null)
{
extra_dimensions = new ExtraDimensions();
}
this.creds = creds;
this.config_ = config;
this.region = region;
this.metrics_index = new MetricsIndex();
region_ = Core.Clients.KinesisClient.AwsKinesisClient.MapRegion(config.region);
this.cw_namespace = cw_namespace;
this.socket_factory_ = socket_factory;
this.level = level;
this.granularity = granularity;
this.upload_frequency = upload_frequency;
this.retry_frequency = retry_frequency;
this.executor = executor;
this.endpoint = string.IsNullOrEmpty(custom_endpoint) ? "monitoring." + region + ".amazonaws.com" : custom_endpoint;
this.http_client = new AwsHttpClient(executor, socket_factory_, endpoint, (int)port, true, string.IsNullOrEmpty(custom_endpoint));
retryable_requests = new List <string>();
if (level != Level.None)
{
StdErrorOut.Instance.StdOut(LogLevel.warn, string.Format("Uploading metrics to {0}:{1}", endpoint, port));
}
extra_dimensionsMap[Granularity.Global] = new Dimensions();
extra_dimensionsMap[Granularity.Stream] = new Dimensions();
extra_dimensionsMap[Granularity.Shard] = new Dimensions();
foreach (var t in extra_dimensions)
{
extra_dimensionsMap[t.Item3].Add(new KeyValuePair <string, string>(t.Item1, t.Item2));
}
scheduled_upload = executor.Schedule(
() =>
{
try
{
scheduled_upload.reschedule(upload_frequency);
upload();
}
catch (Exception ex)
{
StdErrorOut.Instance.StdError("Scheduled Upload failed.", ex);
}
},
upload_frequency);
scheduled_retry = executor.Schedule(
() =>
{
try
{
scheduled_retry.reschedule(retry_frequency);
retry_uploads();
}
catch (Exception ex)
{
StdErrorOut.Instance.StdError("Scheduled Upload failed.", ex);
}
},
retry_frequency);
}
static KinesisProducer get_kp(IMessageManager ipc_manager, ISocketFactory socket_factory, string region, KPLNETConfiguration config, AwsCredentialsProvider creds_provider, AwsCredentialsProvider metrics_creds_provider, Executor executor)
{
try
{
return(new KinesisProducer(ipc_manager, socket_factory, region, config, creds_provider, metrics_creds_provider, executor));
}
catch (Exception ex)
{
StdErrorOut.Instance.StdError("KPLNET Service get_kp failed.", ex);
throw new Exception("get_kp failed.", ex);
}
}
static KeyValuePair <AwsCredentialsProvider, AwsCredentialsProvider> get_creds_providers(Executor executor, Ec2Metadata ec2_md, KPLNETConfiguration config)
{
AwsCredentialsProvider creds_provider = null;
AwsCredentialsProvider metrics_creds_provider = null;
if (null != config.AWSCredentials)
{
creds_provider = new BasicAwsCredentialsProvider(config.AWSCredentials.Akid, config.AWSCredentials.SecretKey, config.AWSCredentials.SessionToken);
if (null != config.AWSMetricsCredentials)
{
metrics_creds_provider = new BasicAwsCredentialsProvider(config.AWSMetricsCredentials.Akid, config.AWSMetricsCredentials.SecretKey, config.AWSMetricsCredentials.SessionToken);
}
else
{
metrics_creds_provider = creds_provider;
}
}
else
{
creds_provider = new DefaultAwsCredentialsProviderChain(executor, ec2_md);
Thread.Sleep(250);
}
if (null == creds_provider.get_credentials())
{
StdErrorOut.Instance.StdError("Could not retrieve credentials from anywhere.");
throw new Exception("Could not retrieve credentials from anywhere.");
}
return(new KeyValuePair <AwsCredentialsProvider, AwsCredentialsProvider>(creds_provider, metrics_creds_provider));
}
