public void UploadBatch()
{
if (_currentBatch != null)
{
throw new Exception("Upload must not be called before the batch currently being uploaded is complete");
}
_currentBatch = _nextBatch;
_nextBatch = new List <T>();
var data = new EnumerableDataReader <T>(_currentBatch, _fields);
try
{
_ingestClient.IngestFromDataReader(data, _ingestionProperties);
int recordsUploaded = _currentBatch.Count;
_currentBatch = null;
_lastUploadTime = DateTime.Now;
Console.Write("{0} ", recordsUploaded);
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
public void UploadBatch(bool lastBatch)
{
lock (uploadLock)
{
if (_currentBatch != null)
{
throw new Exception("Upload must not be called before the batch currently being uploaded is complete");
}
_currentBatch = _nextBatch;
_nextBatch = new List <IDictionary <string, object> >();
try
{
if (_currentBatch.Count > 0)
{
if (_ingestClient != null)
{
var data = new DictionaryDataReader(_currentBatch);
_ingestClient.IngestFromDataReader(data, _ingestionProperties);
Console.Write("{0} ", _currentBatch.Count);
}
else if (outputFile != null)
{
string content = lastBatch ?
$"{JsonConvert.SerializeObject(_currentBatch, Formatting.Indented).Trim(new char[] { '[', ']' })}" :
$"{JsonConvert.SerializeObject(_currentBatch, Formatting.Indented).Trim(new char[] { '[', ']' })},";
try
{
outputFile.Write(content);
}
catch
{
}
Console.Write("{0} ", _currentBatch.Count);
}
else
{
foreach (var item in _currentBatch)
{
Console.WriteLine(string.Join("\t", item.Values));
}
}
}
_currentBatch = null;
_lastUploadTime = DateTime.UtcNow;
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
}
public void UploadBatch(bool lastBatch)
{
lock (uploadLock)
{
if (_currentBatch != null)
{
throw new Exception("Upload must not be called before the batch currently being uploaded is complete");
}
_currentBatch = _nextBatch;
_nextBatch = new List <IDictionary <string, object> >();
try
{
if (_currentBatch.Count > 0)
{
if (_ingestClient != null)
{
var data = new DictionaryDataReader(_currentBatch);
_ingestClient.IngestFromDataReader(data, _ingestionProperties);
Console.Write("{0} ", _currentBatch.Count);
}
else if (blobContainerClient != null)
{
//Create a blob with unique names and upload
string blobName = $"{Guid.NewGuid()}_1_{Guid.NewGuid():N}.json";
var blobClient = blobContainerClient.GetBlobClient(blobName);
UploadToContainerAsync(blobClient, _currentBatch).Wait();
}
}
_currentBatch = null;
_lastUploadTime = DateTime.UtcNow;
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
}
private void UploadBatch()
{
lock (_uploadLock)
{
if (_currentBatch != null)
{
// Something went wrong with ingesting previous batch
// Drop events in previous batch in an attempt to stay alive
_logger.Log(LogLevel.ERROR, "Error ingesting current batch of events. Dropping events in current batch and moving onto next one.");
_currentBatch = null;
}
_currentBatch = _nextBatch;
_nextBatch = new List <IDictionary <string, object> >();
try
{
if (_currentBatch.Count > 0)
{
if (_ingestClient != null)
{
var data = new DictionaryDataReader(_currentBatch);
_ingestClient.IngestFromDataReader(data, _ingestionProperties);
_logger.Log(LogLevel.VERBOSE, $"Current Batch Count: {_currentBatch.Count}");
}
}
_currentBatch = null;
}
catch (Exception ex)
{
OutputError(ex);
}
finally
{
// Resetting flush timer
_lastUploadTime = DateTime.UtcNow;
}
}
}
static void IngestData(KustoConnectionStringBuilder engineKustoConnectionStringBuilder, KustoConnectionStringBuilder dmKustoConnectionStringBuilderDM,
string databaseName, string tableName)
{
// 1. Ingest by connecting directly to the Kustolab cluster and sending a command
using (IKustoIngestClient directClient = KustoIngestFactory.CreateDirectIngestClient(engineKustoConnectionStringBuilder))
{
var kustoIngestionProperties = new KustoIngestionProperties(databaseName, tableName);
directClient.IngestFromDataReader(GetDataAsIDataReader(), kustoIngestionProperties);
}
// 2. Ingest by submitting the data to the Kustolab ingestion cluster
// Note that this is an async operation, so data might not appear immediately
using (IKustoIngestClient queuedClient = KustoIngestFactory.CreateQueuedIngestClient(dmKustoConnectionStringBuilderDM))
{
var kustoIngestionProperties = new KustoIngestionProperties(databaseName, tableName);
queuedClient.IngestFromDataReader(GetDataAsIDataReader(), kustoIngestionProperties);
}
// 3. Ingest by submitting the data to the Kustolab ingestion cluster -
// This time, update the report method and level so you can track the status of your ingestion
// using the IKustoIngestionResult returned from the ingest operation.
IKustoIngestionResult ingestionResult;
using (IKustoIngestClient queuedClient = KustoIngestFactory.CreateQueuedIngestClient(dmKustoConnectionStringBuilderDM))
{
var kustoIngestionProperties = new KustoQueuedIngestionProperties(databaseName, tableName)
{
// The default ReportLevel is set to FailuresOnly.
// In this case we want to check the status of successful ingestions as well.
ReportLevel = IngestionReportLevel.FailuresAndSuccesses,
// You can use either a queue or a table to track the status of your ingestion.
ReportMethod = IngestionReportMethod.Table
};
ingestionResult = queuedClient.IngestFromDataReader(GetDataAsIDataReader(), kustoIngestionProperties);
}
// Obtain the status of our ingestion
var ingestionStatus = ingestionResult.GetIngestionStatusCollection().First();
var watch = System.Diagnostics.Stopwatch.StartNew();
var shouldContinue = true;
while ((ingestionStatus.Status == Status.Pending) && (shouldContinue))
{
// Wait a minute...
Thread.Sleep(TimeSpan.FromMinutes(1));
// Try again
ingestionStatus = ingestionResult.GetIngestionStatusBySourceId(ingestionStatus.IngestionSourceId);
shouldContinue = watch.ElapsedMilliseconds < TimeOutInMilliSeconds;
}
watch.Stop();
if (ingestionStatus.Status == Status.Pending)
{
// The status of the ingestion did not change.
Console.WriteLine(
"Ingestion with ID:'{0}' did not complete. Timed out after :'{1}' Milliseconds"
.FormatWithInvariantCulture(
ingestionStatus.IngestionSourceId, watch.ElapsedMilliseconds));
}
else
{
// The status of the ingestion has changed
Console.WriteLine(
"Ingestion with ID:'{0}' is complete. Ingestion Status:'{1}'".FormatWithInvariantCulture(
ingestionStatus.IngestionSourceId, ingestionStatus.Status));
}
// 4. Show the contents of the table
using (var client = Kusto.Data.Net.Client.KustoClientFactory.CreateCslQueryProvider(engineKustoConnectionStringBuilder))
{
while (true)
{
var query = string.Format("{0}", tableName);
var reader = client.ExecuteQuery(query);
Kusto.Cloud.Platform.Data.ExtendedDataReader.WriteAsText(reader, "Data ingested into the table:", tabify: true, firstOnly: true);
Console.WriteLine("Press 'r' to retry retrieving data from the table, any other key to quit");
var key = Console.ReadKey();
if (key.KeyChar != 'r' || key.KeyChar != 'R')
{
break;
}
}
}
}
public static void Main(string[] args)
{
KustoConnectionStringBuilder readStringBuilder;
KustoConnectionStringBuilder ingestStringBuilder;
readStringBuilder = new KustoConnectionStringBuilder(@"https://kuskusops.kustomfa.windows.net")
{
FederatedSecurity = true,
InitialCatalog = "testdb",
Authority = "72f988bf-86f1-41af-91ab-2d7cd011db47"
};
ingestStringBuilder = new KustoConnectionStringBuilder($"https://ingest-kuskusops.kustomfa.windows.net")
{
FederatedSecurity = true,
InitialCatalog = "testdb",
Authority = "72f988bf-86f1-41af-91ab-2d7cd011db47"
};
using (IKustoIngestClient kustoIngestClient = KustoIngestFactory.CreateQueuedIngestClient(ingestStringBuilder))
{
var kustoIngestionProperties = new KustoQueuedIngestionProperties("testdb", "test_table");
var dataReader = GetDataAsIDataReader();
IKustoIngestionResult ingestionResult =
kustoIngestClient.IngestFromDataReader(dataReader, kustoIngestionProperties);
var ingestionStatus = ingestionResult.GetIngestionStatusCollection().First();
var watch = System.Diagnostics.Stopwatch.StartNew();
var shouldContinue = true;
while ((ingestionStatus.Status == Status.Pending) && (shouldContinue))
{
// Wait a minute...
Thread.Sleep(TimeSpan.FromMinutes(1));
// Try again
ingestionStatus = ingestionResult.GetIngestionStatusBySourceId(ingestionStatus.IngestionSourceId);
shouldContinue = watch.ElapsedMilliseconds < 360000;
}
watch.Stop();
if (ingestionStatus.Status == Status.Pending)
{
// The status of the ingestion did not change.
Console.WriteLine(
"Ingestion with ID:'{0}' did not complete. Timed out after :'{1}' Milliseconds"
.FormatWithInvariantCulture(
ingestionStatus.IngestionSourceId, watch.ElapsedMilliseconds));
}
else
{
// The status of the ingestion has changed
Console.WriteLine(
"Ingestion with ID:'{0}' is complete. Ingestion Status:'{1}'".FormatWithInvariantCulture(
ingestionStatus.IngestionSourceId, ingestionStatus.Status));
}
}
// 4. Show the contents of the table
using (var client = Kusto.Data.Net.Client.KustoClientFactory.CreateCslQueryProvider(readStringBuilder))
{
while (true)
{
var query = "test_table | count";
var reader = client.ExecuteQuery(query);
Kusto.Cloud.Platform.Data.ExtendedDataReader.WriteAsText(reader, "Data ingested into the table:", tabify: true, firstOnly: true);
Console.WriteLine("Press 'r' to retry retrieving data from the table, any other key to quit");
var key = Console.ReadKey();
if (key.KeyChar != 'r' || key.KeyChar != 'R')
{
break;
}
}
}
}