public KustoEndpointInfo()
{
if (!Config.IsKustoConfigured())
{
string errMessage = "kusto not configured";
Log.Error(errMessage);
throw new ArgumentNullException(errMessage);
}
DeleteSourceOnSuccess = !Config.KustoUseBlobAsSource;
if (Regex.IsMatch(Config.KustoCluster, KustoUrlPattern))
{
Match matches = Regex.Match(Config.KustoCluster, KustoUrlPattern);
string domainName = matches.Groups["domainName"].Value;
DatabaseName = matches.Groups["databaseName"].Value;
TableName = Config.KustoTable;
string ingestPrefix = matches.Groups["ingest"].Value;
ClusterName = matches.Groups["clusterName"].Value;
string location = matches.Groups["location"].Value;
HostName = $"{ClusterName}.{location}.{domainName}";
ManagementUrl = $"https://{HostName}";
ClusterIngestUrl = $"https://{ingestPrefix}{ClusterName}.{location}.{domainName}";
RestMgmtUri = $"{ClusterIngestUrl}/v1/rest/mgmt";
RestQueryUri = $"{ManagementUrl}/v1/rest/query";
}
else
{
string errMessage = $"invalid kusto url.";
Log.Error(errMessage);
throw new ArgumentException(errMessage);
}
}
public FileObjectCollection PopulateCollection <T>(FileObject fileObject) where T : IRecord
{
FileObjectCollection collection = new FileObjectCollection()
{
fileObject
};
_instance.TotalFilesFormatted++;
_instance.TotalRecords += fileObject.RecordCount;
if (_config.IsKustoConfigured())
{
// kusto native format is Csv
// kusto json ingest is 2 to 3 times slower and does *not* use standard json format. uses json document per line no comma
// using csv and compression for best performance
collection = SerializeCsv <T>(fileObject);
if (_config.KustoCompressed)
{
collection.ForEach(x => x.Stream.Compress());
}
}
else if (_config.IsLogAnalyticsConfigured())
{
// la is kusto based but only accepts non compressed json format ingest
collection = SerializeJson <T>(fileObject);
}
collection.ForEach(x => SaveToCache(x));
return(collection);
}
public void Authenticate(bool throwOnError = false)
{
_arm.Scopes = new List <string>()
{
$"{ClusterIngestUrl}/kusto.read", $"{ClusterIngestUrl}/kusto.write"
};
if (_config.IsClientIdConfigured())
{
_arm.Scopes = new List <string>()
{
$"{ClusterIngestUrl}/.default"
};
}
if (_config.IsKustoConfigured() && _arm.Authenticate(throwOnError, ClusterIngestUrl))
{
DatabaseConnection = new KustoConnectionStringBuilder(ClusterIngestUrl)
{
FederatedSecurity = true, InitialCatalog = DatabaseName, UserToken = _arm.BearerToken
};
ManagementConnection = new KustoConnectionStringBuilder(ManagementUrl)
{
FederatedSecurity = true, InitialCatalog = DatabaseName, UserToken = _arm.BearerToken
};
}
else
{
DatabaseConnection = new KustoConnectionStringBuilder(ClusterIngestUrl)
{
FederatedSecurity = true, InitialCatalog = DatabaseName
};
ManagementConnection = new KustoConnectionStringBuilder(ManagementUrl)
{
FederatedSecurity = true, InitialCatalog = DatabaseName
};
}
IdentityToken = RetrieveKustoIdentityToken();
IngestionResources = RetrieveIngestionResources();
}
public void Authenticate(bool throwOnError = false)
{
_arm.Scopes = new List <string>()
{
$"{ClusterIngestUrl}/kusto.read", $"{ClusterIngestUrl}/kusto.write"
};
if (_config.IsClientIdConfigured())
{
_arm.Scopes = new List <string>()
{
$"{ClusterIngestUrl}/.default"
};
}
if (_config.IsKustoConfigured() && _arm.Authenticate(throwOnError, ClusterIngestUrl))
{
if (_arm.ClientIdentity.IsAppRegistration)
{
Log.Info($"connecting to kusto with app registration {_config.AzureClientId}");
DatabaseConnection = new KustoConnectionStringBuilder(ClusterIngestUrl)
{
FederatedSecurity = true,
InitialCatalog = DatabaseName,
ApplicationClientId = _config.AzureClientId,
ApplicationCertificateBlob = _config.ClientCertificate,
Authority = _config.AzureTenantId,
ApplicationCertificateSendX5c = true
};
ManagementConnection = new KustoConnectionStringBuilder(ManagementUrl)
{
FederatedSecurity = true,
InitialCatalog = DatabaseName,
ApplicationClientId = _config.AzureClientId,
ApplicationCertificateBlob = _config.ClientCertificate,
Authority = _config.AzureTenantId,
ApplicationCertificateSendX5c = true
};
}
else
{
DatabaseConnection = new KustoConnectionStringBuilder(ClusterIngestUrl)
{
FederatedSecurity = true,
InitialCatalog = DatabaseName,
UserToken = _arm.BearerToken
};
ManagementConnection = new KustoConnectionStringBuilder(ManagementUrl)
{
FederatedSecurity = true,
InitialCatalog = DatabaseName,
UserToken = _arm.BearerToken
};
}
}
else
{
DatabaseConnection = new KustoConnectionStringBuilder(ClusterIngestUrl)
{
FederatedSecurity = true,
InitialCatalog = DatabaseName
};
ManagementConnection = new KustoConnectionStringBuilder(ManagementUrl)
{
FederatedSecurity = true,
InitialCatalog = DatabaseName
};
}
IdentityToken = RetrieveKustoIdentityToken();
IngestionResources = RetrieveIngestionResources();
}