public override async Task RefreshDataSetAsync(ConnectedSystemDataSet dataSet, CancellationToken cancellationToken)
{
using var connection = new SqlConnection(_connectedSystem.Credentials.ConnectionString);
Logger.LogDebug($"Opening MS SQL connection for {_connectedSystem.Name}...");
await connection
.OpenAsync(cancellationToken)
.ConfigureAwait(false);
Logger.LogDebug($"Refreshing DataSet {dataSet.Name}");
// Process any ncalc in the query
var inputText = dataSet.QueryConfig.Query ?? throw new ConfigurationException($"Missing Query in QueryConfig for dataSet '{dataSet.Name}'");
var query = new SubstitutionString(inputText);
var substitutedQuery = query.ToString();
// Send the query off to MS SQL Server
var results = (await connection.QueryAsync <object>(substitutedQuery).ConfigureAwait(false)).ToList();
Logger.LogDebug($"Got {results.Count} results for {dataSet.Name}.");
// Convert to JObjects for easier generic manipulation
var connectedSystemItems = results
.ConvertAll(entity => JObject.FromObject(entity))
;
await ProcessConnectedSystemItemsAsync(
dataSet,
connectedSystemItems,
ConnectedSystem,
cancellationToken)
.ConfigureAwait(false);
}
public override async System.Threading.Tasks.Task RefreshDataSetAsync(ConnectedSystemDataSet dataSet, CancellationToken cancellationToken)
{
Logger.LogDebug($"Refreshing DataSet {dataSet.Name}");
var inputText = dataSet.QueryConfig.Query ?? throw new ConfigurationException($"Missing Query in QueryConfig for dataSet '{dataSet.Name}'");
var query = new SubstitutionString(inputText);
var substitutedQuery = query.ToString();
// Send the query off to AutoTask
var autoTaskResult = await _autoTaskClient
.GetAllAsync(substitutedQuery, cancellationToken)
.ConfigureAwait(false);
Logger.LogDebug($"Got {autoTaskResult.Count():N0} results for {dataSet.Name}.");
// Convert to JObjects for easier generic manipulation
var connectedSystemItems = autoTaskResult
.Select(entity => JObject.FromObject(entity))
.ToList();
await ProcessConnectedSystemItemsAsync(
dataSet,
connectedSystemItems,
ConnectedSystem,
cancellationToken
).ConfigureAwait(false);
}
public override async Task RefreshDataSetAsync(ConnectedSystemDataSet dataSet, CancellationToken cancellationToken)
{
Logger.LogDebug($"Refreshing Jira DataSet {dataSet.Name}");
var inputText = dataSet.QueryConfig.Query ?? throw new ConfigurationException($"Missing Query in QueryConfig for dataSet '{dataSet.Name}'");
var query = new SubstitutionString(inputText);
var substitutedQuery = query.ToString();
List <JObject> connectedSystemItems;
switch (dataSet.QueryConfig.Type)
{
case "Issue":
var issues = await _jiraClient
.Issues
.GetIssuesFromJqlAsync(dataSet.QueryConfig.Query, token : cancellationToken)
.ConfigureAwait(false);
connectedSystemItems = issues
.Select(issue => JObject.FromObject(issue))
.ToList();
break;
default:
throw new NotSupportedException($"Jira type '{dataSet.QueryConfig.Type}' not supported.");
}
Logger.LogDebug($"Got {connectedSystemItems.Count} results for Jira dataset {dataSet.Name}.");
await ProcessConnectedSystemItemsAsync(
dataSet,
connectedSystemItems,
ConnectedSystem,
cancellationToken
).ConfigureAwait(false);
}
public override async Task RefreshDataSetAsync(ConnectedSystemDataSet dataSet, CancellationToken cancellationToken)
{
Logger.LogDebug($"Refreshing DataSet {dataSet.Name}");
var inputText = dataSet.QueryConfig.Query ?? throw new ConfigurationException($"Missing Query in QueryConfig for dataSet '{dataSet.Name}'");
var query = new SubstitutionString(inputText);
var substitutedQuery = query.ToString();
// Send the query off to ServiceNow
var connectedSystemItems = await _serviceNowClient
.GetAllByQueryAsync(
dataSet.QueryConfig.Type,
substitutedQuery,
extraQueryString : dataSet.QueryConfig.Options,
cancellationToken : cancellationToken)
.ConfigureAwait(false);
Logger.LogDebug($"Got {connectedSystemItems.Count} results for {dataSet.Name}.");
await ProcessConnectedSystemItemsAsync(
dataSet,
connectedSystemItems,
ConnectedSystem,
cancellationToken
).ConfigureAwait(false);
}
public Task <List <SyncAction>?> TestProcessConnectedSystemItemsAsync(ConnectedSystemDataSet dataSet, List <JObject> connectedSystemItems)
=> ProcessConnectedSystemItemsAsync(
dataSet,
connectedSystemItems,
new ConnectedSystem {
Name = "Fake"
},
default);
/// <inheritdoc />
internal override async Task DeleteOutwardsAsync(
ConnectedSystemDataSet dataSet,
JObject connectedSystemItem,
CancellationToken cancellationToken)
{
var endpoint = new SubstitutionString(
dataSet.QueryConfig.DeleteQuery
?? dataSet.QueryConfig.Query
?? throw new ConfigurationException($"In {nameof(ConnectedSystemDataSet)} {dataSet.Name}, one of {nameof(dataSet.QueryConfig.DeleteQuery)} or {nameof(dataSet.QueryConfig.Query)} must be set.")
).ToString();
await _logicMonitorClient.DeleteAsync(endpoint, cancellationToken).ConfigureAwait(false);
}
/// <inheritdoc />
internal override async Task <JObject> CreateOutwardsAsync(
ConnectedSystemDataSet dataSet,
JObject connectedSystemItem,
CancellationToken cancellationToken
)
{
// TODO - Handle functions
var itemToCreate = MakeAutoTaskObject(dataSet, connectedSystemItem);
return(JObject.FromObject(await _autoTaskClient
.CreateAsync(itemToCreate, cancellationToken)
.ConfigureAwait(false)));
}
/// <summary>
/// Strategy: create a Patch containing all of the fields in the connectedSystemItem
/// </summary>
/// <param name="dataSet"></param>
/// <param name="connectedSystemItem"></param>
/// <returns></returns>
/// <exception cref="NotSupportedException"></exception>
internal override Task UpdateOutwardsAsync(
ConnectedSystemDataSet dataSet,
SyncAction syncAction,
CancellationToken cancellationToken)
{
var endpoint = new SubstitutionString(
dataSet.QueryConfig.UpdateQuery
?? dataSet.QueryConfig.Query
?? throw new ConfigurationException($"In {nameof(ConnectedSystemDataSet)} {dataSet.Name}, one of {nameof(dataSet.QueryConfig.UpdateQuery)} or {nameof(dataSet.QueryConfig.Query)} must be set.")
).ToString();
return(_logicMonitorClient.PutAsync(endpoint, syncAction.ConnectedSystemItem, cancellationToken));
}
/// <inheritdoc />
internal override async Task <JObject> CreateOutwardsAsync(
ConnectedSystemDataSet dataSet,
JObject connectedSystemItem,
CancellationToken cancellationToken
)
{
Logger.LogDebug($"Creating Jira {dataSet.QueryConfig.Type}");
var newIssueId = await _jiraClient
.Issues
.CreateIssueAsync(connectedSystemItem.ToObject <Issue>(), cancellationToken)
.ConfigureAwait(false);
Logger.LogDebug($"Created Jira {dataSet.QueryConfig.Type} with id={newIssueId}");
return(JObject.FromObject(new { id = newIssueId }));
}
/// <inheritdoc />
internal override async Task <JObject> CreateOutwardsAsync(
ConnectedSystemDataSet dataSet,
JObject connectedSystemItem,
CancellationToken cancellationToken
)
{
Logger.LogDebug("Creating ServiceNow item");
var newConnectedSystemItem = await _serviceNowClient
.CreateAsync(
dataSet.QueryConfig.Type,
connectedSystemItem, cancellationToken
)
.ConfigureAwait(false);
Logger.LogDebug($"Created ServiceNow item with sys_id={newConnectedSystemItem["sys_id"]}");
return(newConnectedSystemItem);
}
/// <inheritdoc />
internal override async Task DeleteOutwardsAsync(
ConnectedSystemDataSet dataSet,
JObject connectedSystemItem,
CancellationToken cancellationToken
)
{
Logger.LogDebug($"Creating Jira {dataSet.QueryConfig.Type} with id {connectedSystemItem["id"]}");
var id = connectedSystemItem["id"]?.ToString();
if (string.IsNullOrWhiteSpace(id))
{
throw new ConfigurationException($"Cannot delete ServiceNow item with sysId: '{id}'");
}
await _jiraClient
.Issues.DeleteIssueAsync(id, cancellationToken)
.ConfigureAwait(false);
}
public override async Task RefreshDataSetAsync(ConnectedSystemDataSet dataSet, CancellationToken cancellationToken)
{
Logger.LogDebug($"Refreshing DataSet {dataSet.Name}");
var inputText = dataSet.QueryConfig.Query ?? throw new ConfigurationException($"Missing Query in QueryConfig for dataSet '{dataSet.Name}'");
var query = new SubstitutionString(inputText);
var substitutedQuery = query.ToString();
var connectedSystemItems = await _salesforceClient.GetAllJObjectsAsync(substitutedQuery).ConfigureAwait(false);
Logger.LogDebug($"Got {connectedSystemItems.Count} results for {dataSet.Name}.");
await ProcessConnectedSystemItemsAsync(
dataSet,
connectedSystemItems,
ConnectedSystem,
cancellationToken)
.ConfigureAwait(false);
}
/// <inheritdoc />
internal async override Task UpdateOutwardsAsync(
ConnectedSystemDataSet dataSet,
SyncAction syncAction,
CancellationToken cancellationToken
)
{
if (syncAction.ConnectedSystemItem == null)
{
throw new InvalidOperationException($"{nameof(syncAction.ConnectedSystemItem)} must not be null when Updating Outwards.");
}
// Handle simple update
await _serviceNowClient
.UpdateAsync(
dataSet.QueryConfig?.Type ?? throw new ConfigurationException($"DataSet {dataSet.Name} is missing a type"),
syncAction.ConnectedSystemItem,
cancellationToken)
.ConfigureAwait(false);
}
/// <inheritdoc />
internal async override Task UpdateOutwardsAsync(
ConnectedSystemDataSet dataSet,
SyncAction syncAction,
CancellationToken cancellationToken
)
{
if (syncAction.ConnectedSystemItem == null)
{
throw new InvalidOperationException($"{nameof(syncAction.ConnectedSystemItem)} must not be null when Updating Outwards.");
}
throw new NotImplementedException();
// Handle simple update
//await _jiraClient
// .Issues.UpdateIssueAsync(
// new Issue() { Key = syncAction["id"].ToString()},
// cancellationToken)
// .ConfigureAwait(false);
}
/// <inheritdoc />
internal override async Task DeleteOutwardsAsync(
ConnectedSystemDataSet dataSet,
JObject connectedSystemItem,
CancellationToken cancellationToken
)
{
Logger.LogDebug($"Deleting item with id {connectedSystemItem["sys_id"]}");
var sysId = connectedSystemItem["sys_id"]?.ToString();
if (string.IsNullOrWhiteSpace(sysId))
{
throw new ConfigurationException($"Cannot delete ServiceNow item with sysId: '{sysId}'");
}
await _serviceNowClient
.DeleteAsync(
dataSet.QueryConfig.Type,
sysId,
cancellationToken)
.ConfigureAwait(false);
}
private static Entity MakeAutoTaskObject(ConnectedSystemDataSet dataSet, JObject connectedSystemItem)
{
var type = Type.GetType($"AutoTask.Api.{dataSet.QueryConfig.Type}, {typeof(Entity).Assembly.FullName}")
?? throw new ConfigurationException($"AutoTask type {dataSet.QueryConfig.Type} not supported.");
var instance = Activator.CreateInstance(type)
?? throw new ConfigurationException($"AutoTask type {dataSet.QueryConfig.Type} could not be created.");
var connectedSystemItemPropertyNames = connectedSystemItem.Properties().Select(p => p.Name);
var typePropertyInfos = type.GetProperties();
foreach (var propertyInfo in typePropertyInfos.Where(pi => connectedSystemItemPropertyNames.Contains(pi.Name)))
{
propertyInfo.SetValue(instance, connectedSystemItem[propertyInfo.Name] !.ToObject(propertyInfo.PropertyType));
}
var entity = (Entity)instance;
const string UserDefinedFieldPrefix = "UserDefinedFields.";
// Set the UserDefinedFields
var udfNamesToSet = connectedSystemItemPropertyNames
.Where(n => n.StartsWith(UserDefinedFieldPrefix))
.ToList();
// Do we have UDFs to update?
if (udfNamesToSet.Count > 0)
{
// Yes
// It is possible that the entity does not have a UserDefinedFields property set if it is either:
// - being created from scratch
// - or had all UDFs set to null
// Is this the case?
if (entity.UserDefinedFields == null)
{
// Yes - they are not present. Create a new array.
entity.UserDefinedFields = udfNamesToSet.Select(udfName => new UserDefinedField
{
Name = udfName[UserDefinedFieldPrefix.Length..],
private void AssertCorrectPermissions(
Permissions connectedSystemPermissions,
Permissions connectedSystemDataSetPermissions,
Dictionary <SyncActionType, DirectionPermissions> expectedActionTypePermissions)
{
var connectedSystem = new ConnectedSystem(SystemType.Test, "Test")
{
Permissions = connectedSystemPermissions
};
var connectedSystemDataSet = new ConnectedSystemDataSet {
Permissions = connectedSystemDataSetPermissions
};
foreach (var item in expectedActionTypePermissions)
{
ConnectedSystemManagerBase
.DeterminePermission(connectedSystem, connectedSystemDataSet, item.Key)
.Should()
.BeEquivalentTo(item.Value);
}
}
public override Task RefreshDataSetAsync(ConnectedSystemDataSet dataSet, CancellationToken cancellationToken) => throw new NotSupportedException();
public async System.Threading.Tasks.Task TestAsync()
{
var connectedSystem = new ConnectedSystem(SystemType.Test, "Test")
{
Permissions = new Permissions
{
CanWrite = true,
CanCreateIn = true,
CanUpdateIn = true,
CanDeleteIn = true,
CanCreateOut = true,
CanUpdateOut = true,
CanDeleteOut = true
}
};
var state = new State();
state.ItemLists["TestDataSet"] = new StateItemList
{
new StateItem(
new JObject(
new JProperty("Id", "ExistingKey1"),
new JProperty("FullName", "Sarah Jane"), new
JProperty("Description", "Is lovely")
)
),
new StateItem(
new JObject(
new JProperty("Id", "Key1"),
new JProperty("FullName", "Bob1 Smith1"),
new JProperty("Description", "OldDescription1")
)
)
};
var dataSet = new ConnectedSystemDataSet
{
CreateDeleteDirection = CreateDeleteDirection.In,
Name = "DataSet1",
StateDataSetName = "TestDataSet",
Mappings = new List <Mapping>
{
new Mapping
{
Direction = MappingType.Join,
SystemExpression = "ConnectedSystemKey",
StateExpression = "Id"
},
new Mapping
{
Direction = MappingType.In,
SystemExpression = "ConnectedSystemKey",
StateExpression = "Id"
},
new Mapping
{
Direction = MappingType.In,
//SystemExpression = "FirstName",
SystemExpression = "'' + FirstName + ' ' + LastName",
StateExpression = "FullName"
},
new Mapping
{
Direction = MappingType.In,
SystemExpression = "Description",
StateExpression = "Description"
}
},
Permissions = new Permissions {
CanWrite = true, CanCreateIn = true, CanUpdateIn = true, CanDeleteIn = true, CanCreateOut = true, CanUpdateOut = true, CanDeleteOut = true
}
};
using var testConnectedSystemManger = new TestConnectedSystemManager(
connectedSystem,
state,
TimeSpan.FromHours(1),
_outputHelper.BuildLoggerFor <TestConnectedSystemManager>()
);
var actionList = await testConnectedSystemManger
.TestProcessConnectedSystemItemsAsync(dataSet, testConnectedSystemManger.Items[dataSet.Name])
.ConfigureAwait(false);
actionList.Should().NotBeNullOrEmpty();
actionList.Should().HaveCount(6);
var creates = actionList.Where(a => a.Type == SyncActionType.CreateState);
creates.Should().HaveCount(4);
creates.All(a => a.Permission.In == DataSetPermission.Allowed).Should().BeTrue();
creates.All(a => a.Permission.Out == DataSetPermission.InvalidOperation).Should().BeTrue();
var updates = actionList.Where(a => a.Type == SyncActionType.UpdateBoth);
updates.Should().HaveCount(1);
updates.All(a => a.Permission.In == DataSetPermission.Allowed).Should().BeTrue();
updates.All(a => a.Permission.Out == DataSetPermission.Allowed).Should().BeTrue();
var deletes = actionList.Where(a => a.Type == SyncActionType.DeleteState);
deletes.Should().HaveCount(1);
deletes.All(a => a.Permission.In == DataSetPermission.Allowed).Should().BeTrue();
deletes.All(a => a.Permission.Out == DataSetPermission.InvalidOperation).Should().BeTrue();
// Process a second time - should be in stable state
actionList = await testConnectedSystemManger
.TestProcessConnectedSystemItemsAsync(dataSet, testConnectedSystemManger.Items[dataSet.Name])
.ConfigureAwait(false);
actionList.Should().NotBeNullOrEmpty();
actionList.Should().HaveCount(5);
actionList.All(a => a.Type == SyncActionType.AlreadyInSync).Should().BeTrue();
actionList.All(a => a.Permission.In == DataSetPermission.Allowed).Should().BeTrue();
actionList.All(a => a.Permission.Out == DataSetPermission.Allowed).Should().BeTrue();
}
/// <inheritdoc /> internal override Task UpdateOutwardsAsync(ConnectedSystemDataSet dataSet, SyncAction syncAction, CancellationToken cancellationToken) => throw new NotSupportedException();
/// <inheritdoc /> internal override Task DeleteOutwardsAsync(ConnectedSystemDataSet dataSet, JObject connectedSystemItem, CancellationToken cancellationToken) => throw new NotSupportedException();
public DataSetDuplicateItemException(ConnectedSystemDataSet dataSet, Mapping mapping, JObject item)
{
DataSet = dataSet;
Mapping = mapping;
Item = item;
}
public override async Task RefreshDataSetAsync(ConnectedSystemDataSet dataSet, CancellationToken cancellationToken)
{
List <JObject> connectedSystemItems;
Logger.LogDebug($"Refreshing DataSet {dataSet.Name}");
if (string.IsNullOrWhiteSpace(dataSet.QueryConfig.Type))
{
throw new ConfigurationException($"In {dataSet.Name}, {nameof(dataSet.QueryConfig)} {nameof(dataSet.QueryConfig.Type)} must be set.");
}
if (string.IsNullOrWhiteSpace(dataSet.QueryConfig.Query))
{
throw new ConfigurationException($"In {dataSet.Name}, {nameof(dataSet.QueryConfig)} {nameof(dataSet.QueryConfig.Query)} must be set.");
}
var type = dataSet.QueryConfig.Type;
var query = new SubstitutionString(dataSet.QueryConfig.Query).ToString();
var configItems = query.Split('|');
try
{
var configItemsExceptFirst = configItems
.ToList();
switch (type.ToLowerInvariant())
{
case "exprptglds":
var exprptgldsConfig = new ExprptgldsConfig(configItemsExceptFirst);
// We have the index
var expenseReportGlds = await _certifyClient
.ExpenseReportGlds
.GetAllAsync(exprptgldsConfig.Index, active : 1)
.ConfigureAwait(false);
connectedSystemItems = expenseReportGlds
.ConvertAll(entity => JObject.FromObject(entity))
;
break;
case "expensereports":
var expenseReports = await _certifyClient
.ExpenseReports
.GetAllAsync()
.ConfigureAwait(false);
connectedSystemItems = expenseReports
.ConvertAll(entity => JObject.FromObject(entity))
;
break;
case "expenses":
var propertyFilters = new List <Filter>();
string?startDate = null;
string?endDate = null;
string?batchId = null;
uint? processed = null;
uint? includeDisapproved = null;
var allFilters = configItemsExceptFirst.Select(ci => new Filter(ci));
foreach (var filter in allFilters)
{
var name = filter.Name;
var value = filter.Value;
var isProperty = false;
switch (name.ToLowerInvariant())
{
case "startdate":
startDate = value;
break;
case "enddate":
endDate = value;
break;
case "batchid":
batchId = value;
break;
case "processed":
processed = GetBoolUint(name, value);
break;
case "includedisapproved":
includeDisapproved = GetBoolUint(name, value);
break;
default:
// Perhaps we will filter on this later?
propertyFilters.Add(filter);
isProperty = true;
break;
}
if (!isProperty && filter.Operator != Operator.Equals)
{
throw new ConfigurationException($"Expense configItem {filter.Name} must in the form 'a==b'");
}
}
// Fetch using the query filters
var expenses = (await _certifyClient
.Expenses
.GetAllAsync(
startDate,
endDate,
batchId,
processed,
includeDisapproved
)
.ConfigureAwait(false))
.AsQueryable();
// Apply property filters
foreach (var propertyFilter in propertyFilters)
{
// Does this refer to a valid property?
var matchingPropertyInfo = ExpensePropertyInfos.SingleOrDefault(pi => string.Equals(pi.Name, propertyFilter.Name, StringComparison.InvariantCultureIgnoreCase));
if (matchingPropertyInfo == null)
{
// No
throw new ConfigurationException($"Expenses do not have property '{propertyFilter.Name}'");
}
// Yes
// Filter on this criteria
expenses = propertyFilter.Operator switch
{
Operator.Equals => expenses.Where(e => matchingPropertyInfo.GetValue(e).EmptyStringIfNull() == propertyFilter.Value),
Operator.NotEquals => expenses.Where(e => matchingPropertyInfo.GetValue(e).EmptyStringIfNull() != propertyFilter.Value),
Operator.LessThanOrEquals => expenses.Where(e => string.Compare(matchingPropertyInfo.GetValue(e).EmptyStringIfNull(), propertyFilter.Value) <= 0),
Operator.LessThan => expenses.Where(e => string.Compare(matchingPropertyInfo.GetValue(e).EmptyStringIfNull(), propertyFilter.Value) < 0),
Operator.GreaterThanOrEquals => expenses.Where(e => string.Compare(matchingPropertyInfo.GetValue(e).EmptyStringIfNull(), propertyFilter.Value) >= 0),
Operator.GreaterThan => expenses.Where(e => string.Compare(matchingPropertyInfo.GetValue(e).EmptyStringIfNull(), propertyFilter.Value) > 0),
_ => throw new NotSupportedException($"Operator '{propertyFilter.Operator}' not supported.")
};
}
var expensesList = expenses.ToList();
// Only one currency supported - convert all to GBP using Fixer
var badExpenses = expensesList
.Where(e => e.Currency != "GBP")
.ToList();
foreach (var badExpense in badExpenses)
{
if (!DateTime.TryParseExact(badExpense.ExpenseDate, "yyyy-MM-dd", DateTimeFormatInfo.InvariantInfo, DateTimeStyles.AssumeUniversal, out var date))
{
throw new FormatException($"Certify date not in expected format: {badExpense.ExpenseDate}");
}
var currency = badExpense.Currency;
var exchangeRateOnDate = await GetExchangeRateOnDateAsync(currency, date).ConfigureAwait(false);
badExpense.Amount *= (float)exchangeRateOnDate;
badExpense.Currency = "GBP";
}
connectedSystemItems = expensesList
.ConvertAll(entity => JObject.FromObject(entity))
;
break;
default:
throw new NotSupportedException();
}
}
catch (Exception e)
{
Logger.LogError($"Could not fetch {type} due to {e.Message}");
throw;
}
await ProcessConnectedSystemItemsAsync(
dataSet,
connectedSystemItems,
ConnectedSystem,
cancellationToken)
.ConfigureAwait(false);
}
