public void GetDeltaImport()
{
using (SQLManagementAgent ma = new SQLManagementAgent())
{
Configuration.Schema = ma.DefaultSchemaXml.XmlDeserializeFromString <SchemaConfiguration>();
Schema schema = ma.GetSchemaDetached();
// first get full data to get deltawatermark
ma.Schema = schema;
ma.ImportType = OperationType.Delta;
ma.CustomData = "140180";
ma.PageSize = 1;
System.Collections.ObjectModel.KeyedCollection <string, ConfigParameter> configParams = null;
ma.OpenImportConnectionDetached(configParams, schema, null);
GetImportEntriesRunStep rs = new GetImportEntriesRunStep();
GetImportEntriesResults rest = new GetImportEntriesResults();
rest.MoreToImport = true;
while (rest.MoreToImport)
{
rest = ma.GetImportEntriesDetached(rs);
}
CloseImportConnectionRunStep dummyCloseImportRunStep = null;
ma.CloseImportConnectionDetached(dummyCloseImportRunStep);
}
}
public GetImportEntriesResults GetImportEntries(GetImportEntriesRunStep importRunStep)
{
GetImportEntriesResults results = new GetImportEntriesResults();
if (this.importRunStepParameters.ImportType == OperationType.Delta)
{
int count = 0;
// Add items to the result collection until the page size has been exceeded or
// The queue is empty
while (CSEntryChangeQueue.Count > 0 && (count < this.importRunStepParameters.PageSize))
{
results.CSEntries.Add(CSEntryChangeQueue.Take());
count++;
}
// If the queue is not yet empty, tell the sync engine that we have more to import
results.MoreToImport = CSEntryChangeQueue.Count > 0;
}
else
{
// Perform normal full import
}
return(results);
}
/// <summary>
/// Gets a batch of entries from the database and returns them to the synchronization service
/// </summary>
/// <param name="importRunStep">The current run step</param>
/// <returns>The results of the import batch</returns>
GetImportEntriesResults IMAExtensible2CallImport.GetImportEntries(GetImportEntriesRunStep importRunStep)
{
List <CSEntryChange> csentries = new List <CSEntryChange>();
int count = 0;
bool mayHaveMore = false;
while (this.importEnumerator.MoveNext())
{
CSEntryChange csentry = this.importEnumerator.Current;
csentries.Add(csentry);
if (csentry.ErrorCodeImport == MAImportError.ImportErrorCustomStopRun)
{
break;
}
count++;
if (count >= this.importPageSize)
{
mayHaveMore = true;
break;
}
}
GetImportEntriesResults importReturnInfo = new GetImportEntriesResults();
importReturnInfo.MoreToImport = mayHaveMore;
importReturnInfo.CSEntries = csentries;
return(importReturnInfo);
}
/// <summary>
///
/// </summary>
/// <param name="importRunStep"></param>
/// <returns></returns>
public GetImportEntriesResults GetImportEntries(GetImportEntriesRunStep importRunStep)
{
HttpWebRequest request = WebRequest.Create(this.imUrl) as HttpWebRequest;
// Get response
//string URLToSend = LiteralURLTrackerAddress.Text.ToString();
//HttpWebRequest request = (HttpWebRequest)WebRequest.Create(URLToSend.ToString()); request.Method = "GET"; request.KeepAlive = true;
using (HttpWebResponse response = request.GetResponse() as HttpWebResponse)
{
List <CSEntryChange> csentries = new List <CSEntryChange>();
GetImportEntriesResults importReturnInfo;
StreamReader responseStream = new StreamReader(response.GetResponseStream());
string webResponseStream = responseStream.ReadToEnd();
XmlDocument xmlDoc = new XmlDocument();
xmlDoc.LoadXml(webResponseStream);
//load data into dataset
DataSet da = new DataSet();
da.ReadXml(response.GetResponseStream());
XmlNodeList nodeList = xmlDoc.DocumentElement.SelectNodes("/Table/Product");
foreach (XmlNode node in nodeList)
{
USERNAME = node.SelectSingleNode("USERNAME").InnerText;
PASSWORD = node.SelectSingleNode("PASSWORD").InnerText;
EMPLOYEE_ID = node.SelectSingleNode("EMPLOYEE_ID").InnerText;
FIRST_NAME = node.SelectSingleNode("FIRST_NAME").InnerText;
LAST_NAME = node.SelectSingleNode("LAST_NAME").InnerText;
EMAIL = node.SelectSingleNode("EMAIL").InnerText;
EMPLOYEE_STATUS = node.SelectSingleNode("EMPLOYEE_STATUS").InnerText;
ROLE_ASSIGNMENT = node.SelectSingleNode("ROLE_ASSIGNMENT").InnerText;
WORKPHONE = node.SelectSingleNode("WORKPHONE").InnerText;
CSEntryChange csentry1 = CSEntryChange.Create();
csentry1.ObjectModificationType = ObjectModificationType.Add;
csentry1.ObjectType = "Person";
csentry1.AttributeChanges.Add(AttributeChange.CreateAttributeAdd("USERNAME", USERNAME));
csentry1.AttributeChanges.Add(AttributeChange.CreateAttributeAdd("PASSWORD", PASSWORD));
csentry1.AttributeChanges.Add(AttributeChange.CreateAttributeAdd("EMPLOYEE_ID", EMPLOYEE_ID));
csentry1.AttributeChanges.Add(AttributeChange.CreateAttributeAdd("FIRST_NAME", FIRST_NAME));
csentry1.AttributeChanges.Add(AttributeChange.CreateAttributeAdd("LAST_NAME", LAST_NAME));
csentry1.AttributeChanges.Add(AttributeChange.CreateAttributeAdd("EMAIL", EMAIL));
csentry1.AttributeChanges.Add(AttributeChange.CreateAttributeAdd("EMPLOYEE_STATUS", EMPLOYEE_STATUS));
csentry1.AttributeChanges.Add(AttributeChange.CreateAttributeAdd("ROLE_ASSIGNMENT", ROLE_ASSIGNMENT));
csentry1.AttributeChanges.Add(AttributeChange.CreateAttributeAdd("WORKPHONE", WORKPHONE));
csentries.Add(csentry1);
}
importReturnInfo = new GetImportEntriesResults();
importReturnInfo.MoreToImport = false;
importReturnInfo.CSEntries = csentries;
return(importReturnInfo);
}
}
private GetImportEntriesResults GetImportEntriesFull()
{
GetImportEntriesResults results = new GetImportEntriesResults {
CSEntries = new List <CSEntryChange>()
};
try
{
for (int i = 0; i < this.importRunStep.PageSize; i++)
{
if (this.importTask.IsFaulted || this.importTask.IsCanceled || this.cancellationToken.IsCancellationRequested)
{
break;
}
if (this.importCollection.IsCompleted && this.importCollection.Count == 0)
{
break;
}
if (!this.importCollection.TryTake(out object item, 50, this.cancellationToken.Token))
{
continue;
}
Interlocked.Increment(ref this.opCount);
if (item is CSEntryChange csentry)
{
results.CSEntries.Add(csentry);
continue;
}
throw new NotSupportedException("The object enumeration returned an unsupported type: " + item.GetType().Name);
}
results.MoreToImport = !(this.importCollection.IsCompleted && this.importCollection.Count == 0);
if (results.MoreToImport && results.CSEntries.Count == 0)
{
Thread.Sleep(1000);
}
}
catch (OperationCanceledException)
{
}
if (this.importTask.IsFaulted)
{
throw this.importTask.Exception ?? new Exception("The task was faulted, but an exception was not provided");
}
return(results);
}
private GetImportEntriesResults GetImportEntriesDelta()
{
GetImportEntriesResults results = new GetImportEntriesResults {
CSEntries = new List <CSEntryChange>()
};
int count = 0;
while (CSEntryChangeQueue.Count > 0 && (count < this.importRunStep.PageSize))
{
Interlocked.Increment(ref this.opCount);
results.CSEntries.Add(CSEntryChangeQueue.Take());
count++;
}
results.MoreToImport = CSEntryChangeQueue.Count > 0;
return(results);
}
/// <summary>
/// Gets a batch of entries from the database and returns them to the synchronization service
/// </summary>
/// <param name="importRunStep">The current run step</param>
/// <returns>The results of the import batch</returns>
public GetImportEntriesResults GetImportEntries(GetImportEntriesRunStep importRunStep)
{
try
{
List <CSEntryChange> csentries = new List <CSEntryChange>();
PageRequest request = new PageRequest();
request.PageSize = this.importPageSize;
request.Context = this.importResponse.Context;
Logger.WriteLine("Requesting page of {0} results", LogLevel.Debug, this.importPageSize);
ImportResponse page = this.client.ImportPage(request);
Logger.WriteLine("Got page of {0} results", LogLevel.Debug, this.importPageSize);
foreach (CSEntryChange csentry in page.Objects)
{
csentries.Add(csentry);
if (csentry.ErrorCodeImport == MAImportError.ImportErrorCustomStopRun)
{
break;
}
if (string.IsNullOrWhiteSpace(csentry.ObjectType) || csentry.ErrorCodeImport != MAImportError.Success)
{
Logger.WriteLine("An error was detected in the following CSEntryChange object");
Logger.WriteLine(csentry.ToDetailString());
}
}
Logger.WriteLine("Returning page of {0} results. More to import: {1}", LogLevel.Debug, page.Objects.Count, page.HasMoreItems);
GetImportEntriesResults importReturnInfo = new GetImportEntriesResults();
importReturnInfo.MoreToImport = page.HasMoreItems;
importReturnInfo.CSEntries = csentries;
return(importReturnInfo);
}
catch (Exception ex)
{
Logger.WriteException(ex);
throw;
}
}
// GetImportEntries
public GetImportEntriesResults GetImportEntries(GetImportEntriesRunStep _importRunStep)
{
utils.Logger(TraceEventType.Information,
ConstDefinition.ID0500_START_GETIMPORTENTRIES,
ConstDefinition.MSG0500_START_GETIMPORTENTRIES);
try
{
var _csentries = new List <CSEntryChange>();
string _importEntriesJSON = utils.GetContentsWithAccessToken(resource_uri, auth_token, null);
var _getImportEntriesByObjectTypeResult = JObject.Parse(_importEntriesJSON).SelectToken("value").ToString();
var _importObjectJSONArray = JArray.Parse(_getImportEntriesByObjectTypeResult);
foreach (var _importObjectJSON in _importObjectJSONArray)
{
var _csentryChange = CSEntryChange.Create();
_csentryChange.ObjectModificationType = ObjectModificationType.Add;
_csentryChange.ObjectType = "Person";
_csentryChange.AttributeChanges.Add(AttributeChange.CreateAttributeAdd("Alias", _importObjectJSON["Alias"].ToString()));
_csentryChange.AttributeChanges.Add(AttributeChange.CreateAttributeAdd("Email", _importObjectJSON["Email"].ToString()));
_csentryChange.AttributeChanges.Add(AttributeChange.CreateAttributeAdd("Username", _importObjectJSON["Username"].ToString()));
_csentryChange.AttributeChanges.Add(AttributeChange.CreateAttributeAdd("LastName", _importObjectJSON["LastName"].ToString()));
_csentryChange.AttributeChanges.Add(AttributeChange.CreateAttributeAdd("EmailEncodingKey", _importObjectJSON["EmailEncodingKey"].ToString()));
_csentryChange.AttributeChanges.Add(AttributeChange.CreateAttributeAdd("LanguageLocaleKey", _importObjectJSON["LanguageLocaleKey"].ToString()));
_csentryChange.AttributeChanges.Add(AttributeChange.CreateAttributeAdd("LocaleSidKey", _importObjectJSON["LocaleSidKey"].ToString()));
_csentryChange.AttributeChanges.Add(AttributeChange.CreateAttributeAdd("ProfileId", _importObjectJSON["ProfileId"].ToString()));
_csentryChange.AttributeChanges.Add(AttributeChange.CreateAttributeAdd("TimeZoneSidKey", _importObjectJSON["TimeZoneSidKey"].ToString()));
_csentryChange.AttributeChanges.Add(AttributeChange.CreateAttributeAdd("UserPermissionsOfflineUser", false));
_csentryChange.AttributeChanges.Add(AttributeChange.CreateAttributeAdd("UserPermissionsMarketingUser", false));
_csentries.Add(_csentryChange);
}
var _importReturnInfo = new GetImportEntriesResults();
_importReturnInfo.MoreToImport = false;
_importReturnInfo.CSEntries = _csentries;
return(_importReturnInfo);
}
catch (Exception ex)
{
utils.Logger(TraceEventType.Error,
ConstDefinition.ID0599_ERROR_GETIMPORTENTRIES,
ConstDefinition.MSG0599_ERROR_GETIMPORTENTRIES + ex.Message);
throw new ExtensibleExtensionException(ConstDefinition.MSG0599_ERROR_GETIMPORTENTRIES + ex.Message);
}
}
public GetImportEntriesResults GetImportEntries(GetImportEntriesRunStep importRunStep)
{
IObjectSource <IExternalObject> objectSource = _objectSources[_currentRepoIndex];
List <IExternalObject> importedObjects = _objectImportsDictionary[objectSource];
List <IExternalObject> currentImportBatchEntities = importedObjects
.Skip(_entitesImportedCount)
.Take(ImportPageSize)
.ToList();
_entitesImportedCount += currentImportBatchEntities.Count;
List <CSEntryChange> csentries = objectSource.CSentryConverter.ConvertToCSentries(currentImportBatchEntities);
var importInfo = new GetImportEntriesResults {
MoreToImport = true
};
if (_entitesImportedCount >= importedObjects.Count)
{
importInfo.MoreToImport = false;
}
//switching to next repository
if (importInfo.MoreToImport == false)
{
if (_objectSources.Count > _currentRepoIndex + 1)
{
_currentRepoIndex++;
importInfo.MoreToImport = true;
_entitesImportedCount = 0;
}
}
importInfo.CSEntries = csentries;
return(importInfo);
}
public GetImportEntriesResults GetImportEntries(GetImportEntriesRunStep importRunStep)
{
try
{
GetImportEntriesResults importReturnInfo;
List <CSEntryChange> csentries = new List <CSEntryChange>();
while (userToRead < rsaResult.Count && csentries.Count < m_importPageSize)
{
RSAAccount rsaAccount = rsaResult[userToRead];
PrincipalDTO principal = rsaAccount.Principal;
firstName = principal.firstName;
lastName = principal.lastName;
middleName = principal.middleName;
userID = principal.userID;
managerEmailAddress = principal.email;
identitySource = principal.identitySourceGuid;
securityDomain = principal.securityDomainGuid;
lockoutStatus = principal.lockoutStatus.ToString();
tokenSerialNumber = new List <string>();
tokenGuid = new List <string>();
foreach (ListTokenDTO token in rsaAccount.Tokens)
{
tokenSerialNumber.Add(token.serialNumber);
tokenGuid.Add(token.guid);
}
CSEntryChange csentry1 = CSEntryChange.Create();
csentry1.ObjectModificationType = ObjectModificationType.Add;
csentry1.ObjectType = "Person";
if (firstName != null)
{
csentry1.AttributeChanges.Add(AttributeChange.CreateAttributeAdd("First Name", firstName));
}
if (lastName != null)
{
csentry1.AttributeChanges.Add(AttributeChange.CreateAttributeAdd("Last Name", lastName));
}
if (middleName != null)
{
csentry1.AttributeChanges.Add(AttributeChange.CreateAttributeAdd("Middle Name", middleName));
}
csentry1.AttributeChanges.Add(AttributeChange.CreateAttributeAdd("User ID", userID));
if (managerEmailAddress != null)
{
csentry1.AttributeChanges.Add(AttributeChange.CreateAttributeAdd("Manager Email Address", managerEmailAddress));
}
csentry1.AttributeChanges.Add(AttributeChange.CreateAttributeAdd("Identity Source", identitySource));
csentry1.AttributeChanges.Add(AttributeChange.CreateAttributeAdd("Security Domain", securityDomain));
csentry1.AttributeChanges.Add(AttributeChange.CreateAttributeAdd("Lockout Status", lockoutStatus));
IList <object> serials = (IList <object>)tokenSerialNumber.Select(x => (object)x).ToList();
IList <object> guids = (IList <object>)tokenGuid.Select(x => (object)x).ToList();
csentry1.AttributeChanges.Add(AttributeChange.CreateAttributeAdd("Token Serial Number", serials));
csentry1.AttributeChanges.Add(AttributeChange.CreateAttributeAdd("Token GUID", guids));
csentries.Add(csentry1);
userToRead++;
}
importReturnInfo = new GetImportEntriesResults();
importReturnInfo.MoreToImport = (userToRead < rsaResult.Count);
importReturnInfo.CSEntries = csentries;
return(importReturnInfo);
}
catch (Exception ex)
{
EmailError(ex);
throw ex;
}
}
public GetImportEntriesResults GetImportEntries(GetImportEntriesRunStep importRunStep)
{
Tracer.Enter("getimportentries");
try
{
#region call import script
// if results is null, then this is the first time that we're called,
// so call script and get pipeline object and custom data
Tracer.TraceInformation("more-to-import '{0}'", MoreToImport);
if (MoreToImport)
{
MoreToImport = false; // make sure we set more-to-import to false; could be overwritten further down if pagedimports is true, though
// on first call, we set customdata to value from last successful run
returnedCustomData = importRunStep.CustomData;
Command cmd = new Command(Path.GetFullPath(ImportScript));
cmd.Parameters.Add(new CommandParameter("Username", Username));
cmd.Parameters.Add(new CommandParameter("Password", Password));
cmd.Parameters.Add(new CommandParameter("Credentials", GetSecureCredentials(Username, SecureStringPassword)));
cmd.Parameters.Add(new CommandParameter("AuxUsername", UsernameAux));
cmd.Parameters.Add(new CommandParameter("AuxPassword", PasswordAux));
cmd.Parameters.Add(new CommandParameter("AuxCredentials", GetSecureCredentials(UsernameAux, SecureStringPasswordAux)));
cmd.Parameters.Add(new CommandParameter("ConfigurationParameter", ConfigurationParameter));
cmd.Parameters.Add(new CommandParameter("OperationType", importOperationType.ToString()));
cmd.Parameters.Add(new CommandParameter("UsePagedImport", UsePagedImport));
cmd.Parameters.Add(new CommandParameter("PageSize", ImportRunStepPageSize));
cmd.Parameters.Add(new CommandParameter("Schema", schemaPSObject));
Tracer.TraceInformation("setting-custom-data '{0}'", importRunStep.CustomData);
powershell.Runspace.SessionStateProxy.SetVariable("RunStepCustomData", importRunStep.CustomData);
Tracer.TraceInformation("setting-page-token '{0}'", pageToken);
powershell.Runspace.SessionStateProxy.SetVariable("PageToken", pageToken);
importResults = InvokePowerShellScript(cmd, null).ToList <PSObject>();
returnedCustomData = powershell.Runspace.SessionStateProxy.GetVariable("RunStepCustomData");
pageToken = powershell.Runspace.SessionStateProxy.GetVariable("PageToken");
Tracer.TraceInformation("page-token-returned '{0}'", pageToken == null ? "(null)" : pageToken);
Tracer.TraceInformation("custom-data returned '{0}'", returnedCustomData);
Tracer.TraceInformation("number-of-object(s)-in-pipeline {0:n0}", importResults.Count);
if (UsePagedImport)
{
// Tracer.TraceError("paged-import-not-supported-currently");
object moreToImportObject = powershell.Runspace.SessionStateProxy.GetVariable("MoreToImport");
if (moreToImportObject == null)
{
Tracer.TraceError("For paged imports, the global variable 'MoreToImport' must be set to 'true' or 'false'");
}
else
{
Tracer.TraceInformation("MoreToImport-value-returned '{0}'", moreToImportObject);
if (bool.TryParse(moreToImportObject == null ? bool.FalseString : moreToImportObject.ToString(), out MoreToImport))
{
Tracer.TraceInformation("paged-import-setting-MoreToImport-to '{0}'", MoreToImport);
}
else
{
Tracer.TraceError("Value returned in MoreToImport must be a boolean with value of 'true' or 'false'");
}
}
}
else
{
MoreToImport = false;
Tracer.TraceInformation("non-paged-import-setting-MoreToImport-to '{0}'", MoreToImport);
}
}
#endregion
#region parse returned objects
if (importResults != null && importResults.Count > 0)
{
List <PSObject> importResultsBatch = importResults.Take(ImportRunStepPageSize).ToList();
if (importResults.Count > ImportRunStepPageSize)
{
importResults.RemoveRange(0, importResultsBatch.Count);
}
else
{
importResults.Clear();
}
Tracer.TraceInformation("converting-objects-to-csentrychange {0:n0}", importResultsBatch.Count);
foreach (PSObject obj in importResultsBatch)
{
HashSet <AttributeDefinition> attrs = new HashSet <AttributeDefinition>();
Tracer.TraceInformation("start-connector-space-object");
try
{
CSEntryChange csobject = CSEntryChange.Create();
if (obj.BaseObject.GetType() != typeof(System.Collections.Hashtable))
{
Tracer.TraceWarning("invalid-object-in-pipeline '{0}'", 1, obj.BaseObject.GetType());
continue;
}
object AnchorValue = null;
string AnchorAttributeName = null;
string objectDN = null;
string objectClass = ""; // should be string to prevent null exceptions
string changeType = null;
string ErrorName = null;
string ErrorDetail = null;
MAImportError ImportErrorType = MAImportError.Success; // assume no error
Hashtable hashTable = (Hashtable)obj.BaseObject;
#region get control values
Tracer.TraceInformation("start-getting-control-values");
foreach (string key in hashTable.Keys)
{
if (key.Equals(Constants.ControlValues.ObjectClass, StringComparison.OrdinalIgnoreCase) || key.Equals(Constants.ControlValues.ObjectClassEx, StringComparison.OrdinalIgnoreCase))
{
objectClass = (string)hashTable[key];
Tracer.TraceInformation("got-objectclass {0}, {1}", objectClass, key);
continue;
}
if (key.Equals(Constants.ControlValues.DN, StringComparison.OrdinalIgnoreCase))
{
objectDN = (string)hashTable[key];
Tracer.TraceInformation("got-dn {0}, {1}", objectDN, key);
continue;
}
if (key.Equals(Constants.ControlValues.ChangeType, StringComparison.OrdinalIgnoreCase) || key.Equals(Constants.ControlValues.ChangeTypeEx, StringComparison.OrdinalIgnoreCase))
{
changeType = (string)hashTable[key];
Tracer.TraceInformation("got-changetype {0}, {1}", changeType, key);
continue;
}
if (key.Equals(Constants.ControlValues.ErrorName, StringComparison.OrdinalIgnoreCase))
{
ErrorName = (string)hashTable[key];
Tracer.TraceInformation("got-errorname {0}, {1}", ErrorName, key);
continue;
}
if (key.Equals(Constants.ControlValues.ErrorDetail, StringComparison.OrdinalIgnoreCase))
{
ErrorDetail = (string)hashTable[key];
Tracer.TraceInformation("got-errordetail {0}, {1}", ErrorDetail, key);
continue;
}
}
if (string.IsNullOrEmpty(objectClass))
{
Tracer.TraceError("missing-objectclass");
ImportErrorType = MAImportError.ImportErrorCustomContinueRun;
ErrorName = "missing-objectclass-value";
ErrorDetail = "No value provided for objectclass attribute";
}
else
{
AnchorAttributeName = objectTypeAnchorAttributeNames[objectClass] == null ? "" : (string)objectTypeAnchorAttributeNames[objectClass];
if (string.IsNullOrEmpty(AnchorAttributeName))
{
ImportErrorType = MAImportError.ImportErrorInvalidAttributeValue;
ErrorName = "invalid-objecttype";
ErrorDetail = "Objecttype not defined in schema";
}
foreach (string key in hashTable.Keys)
{
if (key.Equals(AnchorAttributeName, StringComparison.OrdinalIgnoreCase))
{
AnchorValue = hashTable[key];
Tracer.TraceInformation("got-anchor {0}, {1}", AnchorValue, key);
break;
}
}
}
Tracer.TraceInformation("end-getting-control-values");
if (AnchorValue == null)
{
Tracer.TraceError("missing-anchor");
ImportErrorType = MAImportError.ImportErrorCustomContinueRun;
ErrorName = "missing-anchor-value";
ErrorDetail = "No value provided for anchor attribute";
}
if (AnchorValue != null && string.IsNullOrEmpty(objectDN))
{
Tracer.TraceInformation("setting-anchor-as-dn {0}", AnchorValue);
objectDN = AnchorValue.ToString();
}
if (!string.IsNullOrEmpty(ErrorName))
{
ImportErrorType = MAImportError.ImportErrorCustomContinueRun;
if (string.IsNullOrEmpty(ErrorDetail))
{
ErrorDetail = "No error details provided";
}
}
#endregion control values
#region return invalid object
if (ImportErrorType != MAImportError.Success)
{
Tracer.TraceInformation("returning-invalid-object");
if (AnchorValue != null)
{
csobject.AnchorAttributes.Add(AnchorAttribute.Create(AnchorAttributeName, AnchorValue));
}
csobject.ObjectModificationType = ObjectModificationType.Add;
if (!string.IsNullOrEmpty(objectClass))
{
try
{
csobject.ObjectType = objectClass;
}
catch (NoSuchObjectTypeException otEx)
{
Tracer.TraceError("no-such-object '{0}'", otEx);
}
}
if (!string.IsNullOrEmpty(objectClass))
{
csobject.DN = objectDN;
}
Tracer.TraceError("invalid-object dn: {0}, type: {1}, name: {2}, details: {3} ", objectDN, ImportErrorType, ErrorName, ErrorDetail);
csobject.ErrorCodeImport = ImportErrorType;
csobject.ErrorName = ErrorName;
csobject.ErrorDetail = ErrorDetail;
csentryqueue.Add(csobject);
continue;
}
#endregion
#region return deleted object
// we must set ObjectModificationType before any other attributes; otherwise it will default to 'Add'
if (!string.IsNullOrEmpty(changeType) && changeType.Equals("delete", StringComparison.OrdinalIgnoreCase))
{
Tracer.TraceInformation("returning-deleted-object");
Tracer.TraceInformation("change-type {0}", changeType);
csobject.ObjectModificationType = ObjectModificationType.Delete;
csobject.ObjectType = objectClass;
csobject.DN = objectDN;
// we need to get the object anchor value for the deletion
csobject.AnchorAttributes.Add(AnchorAttribute.Create(AnchorAttributeName, AnchorValue));
csentryqueue.Add(csobject);
continue;
}
#endregion
#region returned live object
Tracer.TraceInformation("returning-valid-object");
csobject.ObjectModificationType = ObjectModificationType.Add;
csobject.ObjectType = objectClass;
csobject.DN = objectDN;
csobject.AnchorAttributes.Add(AnchorAttribute.Create(AnchorAttributeName, AnchorValue));
foreach (string key in hashTable.Keys)
{
try
{
if (Regex.IsMatch(key, string.Format(@"^(objectClass|\[objectclass\]|changeType|\[changetype\]|\[DN\]|\[ErrorName\]|\[ErrorDetail\]|{0})$", AnchorAttributeName), RegexOptions.Compiled | RegexOptions.IgnoreCase))
{
Tracer.TraceInformation("skip-control-value {0}", key);
continue;
}
if (hashTable[key] == null)
{
Tracer.TraceInformation("skip-null-value-for '{0}'", key);
continue;
}
SchemaAttribute sa = schema.Types[objectClass].Attributes[key];
Tracer.TraceInformation("attribute: {0} (type {1}, {2}): '{3}'", key, sa.DataType, sa.IsMultiValued ? "multi-value" : "single-value", hashTable[key]);
if (sa.IsMultiValued)
{
//Tracer.TraceInformation("add-multivalue '{0}' [{1}]", key, hashTable[key].GetType());
List <object> mvs = new List <object>();
if (hashTable[key].ToString().EndsWith("[]"))
{
mvs.AddRange((object[])hashTable[key]);
}
else
{
mvs.Add(hashTable[key]);
}
csobject.AttributeChanges.Add(AttributeChange.CreateAttributeAdd(key, mvs));
}
else
{
csobject.AttributeChanges.Add(AttributeChange.CreateAttributeAdd(key, hashTable[key]));
}
}
catch (KeyNotFoundException keyexception)
{
Tracer.TraceError("attribute-is-not-defined-for '{0}' / '{1}' ({2})", key, objectClass, keyexception.ToString());
}
}
#endregion
if (csobject.ErrorCodeImport != MAImportError.Success)
{
Tracer.TraceError("defective-csentrychange id: {0}, dn: {1}, errorcode: {2}, error: {3}, details: {4}", csobject.Identifier, csobject.DN, csobject.ErrorCodeImport, csobject.ErrorName, csobject.ErrorDetail);
}
Tracer.TraceInformation("returning-csentry dn: {0}, id: {1}", csobject.DN, csobject.Identifier);
csentryqueue.Add(csobject);
}
catch (Exception ex)
{
Tracer.TraceError("creating-csentrychange", ex);
}
finally
{
Tracer.TraceInformation("end-connector-space-object");
}
}
// clearing results for next loop
importResultsBatch.Clear();
}
#endregion
#region dequeue csentries
GetImportEntriesResults importReturnInfo = null;
Tracer.TraceInformation("total-import-object(s)-left {0:n0}", importResults.Count);
Tracer.TraceInformation("total-connector-space-object(s)-left {0:n0}", csentryqueue.Count);
List <CSEntryChange> batch = csentryqueue.Take(ImportRunStepPageSize).ToList();
if (csentryqueue.Count > ImportRunStepPageSize)
{
csentryqueue.RemoveRange(0, batch.Count);
}
else
{
csentryqueue.Clear();
}
importReturnInfo = new GetImportEntriesResults();
importReturnInfo.MoreToImport = MoreToImport || importResults.Count > 0 || (csentryqueue.Count > 0);
importReturnInfo.CustomData = returnedCustomData == null ? "" : returnedCustomData.ToString();
importReturnInfo.CSEntries = batch;
Tracer.TraceInformation("should-return-for-more {0}", importReturnInfo.MoreToImport);
Tracer.TraceInformation("custom-data '{0}'", importReturnInfo.CustomData);
Tracer.TraceInformation("connector-space-object(s)-returned {0:n0}", importReturnInfo.CSEntries.Count);
return(importReturnInfo);
#endregion
}
catch (Exception ex)
{
Tracer.TraceError("getimportentries", ex);
throw;
}
finally
{
Tracer.Exit("getimportentries");
}
}
public GetImportEntriesResults GetImportEntriesDetached(GetImportEntriesRunStep importRunStep)
{
Tracer.Enter(nameof(GetImportEntriesDetached));
GetImportEntriesResults results = new GetImportEntriesResults();
try
{
results.MoreToImport = false; // default to no more to import to prevent loops
if (importAnchors == null)
{
importAnchors = new List <object>();
if (ImportType == OperationType.Full)
{
importAnchors = methods.GetAllAnchors(SqlMethods.ImportType.Full).ToList();
Tracer.TraceInformation("got-anchors {0:n0}", importAnchors.Count);
}
else
{
importAnchors = methods.GetAllAnchors(SqlMethods.ImportType.Delta, CustomData).ToList();
}
}
if (importAnchors.Count > 0)
{
List <object> sqlbatch = importAnchors.Take(PageSize).ToList();
if (importAnchors.Count > PageSize)
{
importAnchors.RemoveRange(0, sqlbatch.Count);
}
else
{
importAnchors.Clear();
}
Tracer.TraceInformation("reading-objects {0:n0}", sqlbatch.Count);
DataSet records;
records = methods.ReadObjects(sqlbatch);
importCsEntryQueue.AddRange(DataSetToCsEntryChanges(records));
records.Clear();
records.Dispose();
sqlbatch.Clear();
sqlbatch = null;
}
// return this batch
Tracer.TraceInformation("sql-object-left {0:n0}", importAnchors.Count);
Tracer.TraceInformation("cs-objects-left {0:n0}", importCsEntryQueue.Count);
List <CSEntryChange> batch = importCsEntryQueue.Take(PageSize).ToList();
if (importCsEntryQueue.Count > PageSize)
{
importCsEntryQueue.RemoveRange(0, batch.Count);
}
else
{
importCsEntryQueue.Clear();
}
results.MoreToImport = (importAnchors != null && importAnchors.Count > 0) || (importCsEntryQueue.Count > 0);
results.CustomData = CustomData == null ? "" : CustomData;
results.CSEntries = batch;
Tracer.TraceInformation("more-to-import {0}", results.MoreToImport);
Tracer.TraceInformation("custom-data '{0}'", results.CustomData);
Tracer.TraceInformation("csobjects-returned {0:n0}", results.CSEntries.Count);
}
catch (Exception ex)
{
Tracer.TraceError(nameof(GetImportEntriesDetached), ex);
}
finally
{
Tracer.Exit(nameof(GetImportEntriesDetached));
}
return(results);
}
public GetImportEntriesResults GetImportEntries(GetImportEntriesRunStep importRunStep)
{
_lastRunTimeStamp = importRunStep.CustomData;
GetImportEntriesResults importReturnInfo = new GetImportEntriesResults();
List <CSEntryChange> csentries = new List <CSEntryChange>();
if (!_skipPeople)
{
foreach (JObject person in _personDiscoveryList)
{
CSEntryChange csentryPerson = CSEntryChange.Create();
csentryPerson.ObjectType = CS_OBJECTTYPE_PERSON;
csentryPerson.ObjectModificationType = ObjectModificationType.Add;
foreach (KeyValuePair <string, JToken> item in person)
{
if (item.Key.ToLower().Equals(ANCHOR))
{
string itemName = string.Format("{0}_{1}", CS_OBJECTTYPE_PERSON, ANCHOR);
string itemValue = string.Format("{0}_{1}", CS_OBJECTTYPE_PERSON, item.Value.ToString());
csentryPerson.AnchorAttributes.Add(AnchorAttribute.Create(itemName, itemValue));
}
else if (item.Key.ToLower().Equals("known_for") && item.Value.GetType().Name.Equals("JArray"))
{
if (item.Value.HasValues)
{
List <object> knowFor = new List <object>();
foreach (var movieObject in item.Value)
{
MovieDiscoveryJsonTypes.Result movie = new MovieDiscoveryJsonTypes.Result();
foreach (JToken itemValue in movieObject.Children())
{
string movieAnchor = string.Empty;
if (((JProperty)itemValue).Name.Equals("id"))
{
movieAnchor = string.Format("{0}_{1}", CS_OBJECTTYPE_MOVIE, ((JProperty)itemValue).Value);
knowFor.Add(movieAnchor);
movie.id = ((int)((JProperty)itemValue).Value);
}
else if (((JProperty)itemValue).Name.Equals("genre_ids") ||
((JProperty)itemValue).Name.Equals("origin_country"))
{
}
else
{
movie[((JProperty)itemValue).Name] = ((JValue)((JProperty)itemValue).Value).Value;
}
}
bool addMovie = true;
for (int i = 0; i < _moviesList.Count; i++)
{
if (_moviesList[i].id.Equals(movie.id))
{
addMovie = false;
break;
}
}
if (addMovie)
{
_moviesList.Add(movie);
}
}
csentryPerson.AttributeChanges.Add(AttributeChange.CreateAttributeAdd(item.Key, knowFor));
}
}
else
{
csentryPerson.AttributeChanges.Add(AttributeChange.CreateAttributeAdd(item.Key, item.Value.ToString()));
}
}
bool addPerson = true;
foreach (CSEntryChange item in csentries)
{
addPerson = !(item.AnchorAttributes[0].Value.Equals(csentryPerson.AnchorAttributes[0].Value));
}
if (addPerson)
{
csentries.Add(csentryPerson);
_objectCount++;
}
if (_objectCount >= _currentPageSize)
{
importReturnInfo.MoreToImport = true;
break;
}
}
_personCount = _objectCount++;
}
if (!importReturnInfo.MoreToImport)
{
for (int moviesPos = _objectCount - _personCount; moviesPos < _moviesList.Count; moviesPos++)
{
CSEntryChange csentryMovie = CSEntryChange.Create();
csentryMovie.ObjectType = CS_OBJECTTYPE_MOVIE;
csentryMovie.ObjectModificationType = ObjectModificationType.Add;
foreach (PropertyInfo property in _moviesList[moviesPos])
{
if (property.Name.Equals(ANCHOR))
{
string propertyName = string.Format("{0}_{1}", CS_OBJECTTYPE_MOVIE, ANCHOR);
string propertyValue = string.Format("{0}_{1}", CS_OBJECTTYPE_MOVIE, _moviesList[moviesPos][ANCHOR].ToString());
csentryMovie.AnchorAttributes.Add(AnchorAttribute.Create(propertyName, propertyValue));
}
else if (property.Name.ToLower().Equals("item"))
{
}
else if (property.Name != null && _moviesList[moviesPos][property.Name] != null)
{
csentryMovie.AttributeChanges.Add(AttributeChange.CreateAttributeAdd(property.Name, _moviesList[moviesPos][property.Name].ToString()));
}
}
csentries.Add(csentryMovie);
_objectCount++;
if (_objectCount >= _currentPageSize)
{
_skipPeople = true;
importReturnInfo.MoreToImport = true;
break;
}
}
}
importReturnInfo.CSEntries = csentries;
return(importReturnInfo);
}
private GetImportEntriesResults ConsumePageFromProducer()
{
int count = 0;
bool mayHaveMore = false;
GetImportEntriesResults results = new GetImportEntriesResults {
CSEntries = new List <CSEntryChange>()
};
if (this.importContext.ImportItems.IsCompleted)
{
results.MoreToImport = false;
return(results);
}
this.Batch++;
logger.Trace($"Producing page {this.Batch}");
while (!this.importContext.ImportItems.IsCompleted || this.importContext.CancellationTokenSource.IsCancellationRequested)
{
count++;
CSEntryChange csentry = null;
try
{
csentry = this.importContext.ImportItems.Take();
this.importContext.ImportedItemCount++;
logger.Trace($"Got record {this.importContext.ImportedItemCount}:{csentry.ErrorCodeImport}:{csentry?.ObjectModificationType}:{csentry?.ObjectType}:{csentry?.DN}");
}
catch (InvalidOperationException)
{
}
if (csentry != null)
{
results.CSEntries.Add(csentry);
}
if (count >= this.importContext.RunStep.PageSize)
{
mayHaveMore = true;
break;
}
}
if (this.importContext.Producer?.IsFaulted == true)
{
throw new ExtensibleExtensionException("The producer thread encountered an exception", this.importContext.Producer.Exception);
}
if (mayHaveMore)
{
logger.Trace($"Page {this.Batch} complete");
}
else
{
logger.Info("CSEntryChange consumption complete");
this.Batch = 0;
}
results.MoreToImport = mayHaveMore;
return(results);
}
