/// <summary>
/// Apply changes sent by a client to the server.
/// </summary>
/// <param name="serverBlob">Blob sent in the incoming request</param>
/// <param name="entities">Changes from the client</param>
/// <returns>Response containing the new knowledge and conflict/error information.</returns>
public ApplyChangesResponse ApplyChanges(byte[] serverBlob, List<IOfflineEntity> entities)
{
WebUtil.CheckArgumentNull(serverBlob, "serverBlob");
WebUtil.CheckArgumentNull(entities, "entities");
if (0 == serverBlob.Length)
{
throw new InvalidOperationException("serverBlob is empty");
}
var syncBlob = new SyncBlob();
SyncBlob incomingBlob = SyncBlob.DeSerialize(serverBlob);
PopulateClientScopeNameAndSyncId(incomingBlob);
// Set the scope name in the response blob.
syncBlob.ClientScopeName = incomingBlob.ClientScopeName;
// If the requested scope does not exists, then throw an error since we
// don't initialize scopes on upload requests.
if (!CheckIfScopeExists())
{
throw SyncServiceException.CreateResourceNotFound("Scope does not exist");
}
byte[] clientKnowledgeBlob = incomingBlob.ClientKnowledge;
// Initialize a SqlSyncProvider object.
_sqlSyncProvider = CreateSqlSyncProviderInstance(_clientScopeName, _serverConnectionString, _configuration.SyncObjectSchema);
var response = new ApplyChangesResponse();
// Deserialize the knowledge or create new empty knowledge.
SyncKnowledge clientKnowledge = GetSyncKnowledgeFromBlob(clientKnowledgeBlob);
// If there are no entities to upload, then return the client knowledge as is.
if (entities.Count == 0)
{
response.Conflicts = new List<SyncConflict>();
response.Errors = new List<SyncError>();
syncBlob.ClientKnowledge = clientKnowledge.Serialize();
response.ServerBlob = syncBlob.Serialize();
return response;
}
// Client never has any forgotten knowledge. So create a new one.
var forgottenKnowledge = new ForgottenKnowledge(_sqlSyncProvider.IdFormats, clientKnowledge);
// Convert the entities to dataset using the custom converter.
DataSet changesDS = _converter.ConvertEntitiesToDataSet(entities);
var stats = new SyncSessionStatistics();
var sessionContext = new SyncSessionContext(_sqlSyncProvider.IdFormats, new SyncCallbacks());
_sqlSyncProvider.BeginSession(SyncProviderPosition.Remote, sessionContext);
ulong tickCount = 0;
SyncKnowledge updatedClientKnowldege;
try
{
uint batchSize;
SyncKnowledge serverKnowledge;
// This gives us the server knowledge.
_sqlSyncProvider.GetSyncBatchParameters(out batchSize, out serverKnowledge);
var changeBatch = new ChangeBatch(_sqlSyncProvider.IdFormats, clientKnowledge, forgottenKnowledge);
changeBatch.SetLastBatch();
//Note: There is a possiblity of (-ve) item exceptions , between two uploads from the
// same client (for example: in case of RI failures). This would result in an incorrect value if the function
// FindMinTickCountForReplica is used to get the last tickcount. So, we need to ignore the -ve item exceptions
// when finding the tickcount for the client replica from the server knowledge.
/* Logic:
* SyncKnowledge.GetKnowledgeForItemId could be used for itemid Zero and then we can find the mintickcount for client replica id.
* This does not however seem to work, so we use the KnowledgeInspector and enumerate over each ClockVector
* and find the client clockvector and get its tickcount.
*
* Assumption: The above approach assumes that we don't have any positive exceptions in the knowledge.
*/
try
{
// Check if the client replica key exists.
uint clientReplicaKey = serverKnowledge.ReplicaKeyMap.LookupReplicaKey(_clientSyncId);
var ki = new KnowledgeInspector(1, serverKnowledge);
var clockVector = (ClockVector)ki.ScopeClockVector;
int noOfReplicaKeys = clockVector.Count;
for (int i = noOfReplicaKeys - 1; i >= 0; i--)
{
if (clockVector[i].ReplicaKey == clientReplicaKey)
{
tickCount = clockVector[i].TickCount;
break;
}
}
}
catch (ReplicaNotFoundException exception)
{
SyncTracer.Info("ReplicaNotFoundException. NEW CLIENT. Exception details: {0}",
WebUtil.GetExceptionMessage(exception));
// If the knowedge does not contain the client replica (first apply), initialize tickcount to zero.
tickCount = 0;
}
// Increment the tickcount
tickCount++;
// update the made with knowledge to include the new tickcount.
updatedClientKnowldege = new SyncKnowledge(_sqlSyncProvider.IdFormats, _clientSyncId, tickCount);
updatedClientKnowldege.Combine(clientKnowledge);
// The incoming data does not have metadata for each item, so we need to create it at this point.
AddSyncColumnsToDataSet(changesDS, tickCount);
// Make DbSyncContext
var dbSyncContext = new DbSyncContext
{
IsDataBatched = false,
IsLastBatch = true,
DataSet = changesDS,
MadeWithKnowledge = updatedClientKnowldege,
MadeWithForgottenKnowledge = forgottenKnowledge,
ScopeProgress = new DbSyncScopeProgress()
};
_conflicts = new List<SyncConflict>();
_syncErrors = new List<SyncError>();
// Subscribe to the ApplyChangeFailed event to handle conflicts.
_sqlSyncProvider.ApplyChangeFailed += SqlSyncProviderApplyChangeFailed;
// Subscribe to the ChangesApplied event to read the server tickcount incase there are any conflicts.
_sqlSyncProvider.ChangesApplied += SqlSyncProviderChangesApplied;
//NOTE: The ConflictResolutionPolicy pass into the method is IGNORED.
// Conflicts can be logged by subscribing to the failed events
_sqlSyncProvider.ProcessChangeBatch(Microsoft.Synchronization.ConflictResolutionPolicy.DestinationWins,
changeBatch,
dbSyncContext, new SyncCallbacks(), stats);
if (0 != _conflicts.Count)
{
_sqlSyncProvider.GetSyncBatchParameters(out batchSize, out serverKnowledge);
// The way the current P2P provider works, versions are bumped up when conflicts are resolved on the server.
// This would result in us sending the changes to the client on the next download request. We want
// to not enumerate that change again on the next request from the same client.
// The solution is to get the server knowledge after all changes are applied and then
// project the knowledge of each conflictign item and add it as a positive exception to the updated client knowledge.
AddConflictItemsKnowledgeToClientKnowledge(updatedClientKnowldege, serverKnowledge);
}
}
finally
{
_sqlSyncProvider.EndSession(sessionContext);
}
// Don't send any updates to the server knowledge since the client has not got any updates yet.
// This updated knowledge will only include an update to the client tickcount.
// The client would obtain the server knowledge when it does a get changes.
// If we include the serverknowlege, the client would never get any items that are
// between the current server knowledge and the client known server knowledge.
syncBlob.ClientKnowledge = updatedClientKnowldege.Serialize();
response.ServerBlob = syncBlob.Serialize();
response.Conflicts = _conflicts;
response.Errors = _syncErrors;
return response;
}
private async Task <CdcState> SetInitialStateAsync(CancellationToken token, IKafkaProducer producer, string executionId, TableSchema tableSchema, TimeSpan maxInterval)
{
byte[] initialToLsn = await _cdcReaderClient.GetMaxLsnAsync();
var existingOffset = await _cdcReaderClient.GetLastCdcOffsetAsync(executionId, tableSchema.TableName);
if (existingOffset.Result == Result.NoStoredState)
{
Console.WriteLine($"Table {tableSchema.TableName} - No previous stored LSN. Starting from first change");
var hasFirstChange = false;
ChangeBatch syncBatch = null;
ChangeRecord firstChange = null;
while (!hasFirstChange && !token.IsCancellationRequested)
{
var initialFromLsn = await _cdcReaderClient.GetMinValidLsnAsync(tableSchema.TableName);
initialToLsn = await _cdcReaderClient.GetMaxLsnAsync();
syncBatch = await _cdcReaderClient.GetChangeBatchAsync(tableSchema, initialFromLsn, initialToLsn, 1);
if (syncBatch.Changes.Any())
{
firstChange = syncBatch.Changes.First();
await producer.SendAsync(token, firstChange);
hasFirstChange = true;
}
else
{
await Task.Delay(maxInterval);
}
}
var cdcState = new CdcState()
{
FromLsn = firstChange.Lsn,
FromSeqVal = firstChange.SeqVal,
ToLsn = initialToLsn,
UnfinishedLsn = syncBatch.MoreOfLastTransaction
};
var offset = GetOffset(cdcState);
await _cdcReaderClient.StoreCdcOffsetAsync(executionId, tableSchema.TableName, offset);
return(cdcState);
}
else
{
Console.WriteLine($"Table {tableSchema.TableName} - Starting from stored LSN");
return(new CdcState()
{
FromLsn = existingOffset.State.Lsn,
FromSeqVal = existingOffset.State.SeqVal,
ToLsn = initialToLsn,
UnfinishedLsn = existingOffset.State.UnfinishedLsn
});
}
}
public override ChangeBatch GetChangeBatch(uint batchSize, SyncKnowledge destinationKnowledge, out object changeDataRetriever)
{
ChangeBatch batch = _metadataStore.Metadata.GetChangeBatch(batchSize, destinationKnowledge);
changeDataRetriever = _changeApplier;
return(batch);
}
//Simply ask the metadata store to compute my change batch for me, providing the batch size and the knowledge of the other endpoint!
//The engine is asking for the list of changes that the destination provider does not know about.
public override ChangeBatch GetChangeBatch(uint batchSize, SyncKnowledge destinationKnowledge, out object changeDataRetriever)
{
ChangeBatch batch = _metadata.GetChangeBatch(batchSize, destinationKnowledge);
changeDataRetriever = this; //this is where the transfer mechanism/protocol would go. For an in memory provider, this is sufficient
return(batch);
}
public ChangeBatch GetChangeBatch(uint batchSize, SyncKnowledge destinationKnowledge, out ForgottenKnowledge forgottenKnowledge, out object changeDataRetriever)
{
// Increment the tick count
GetNextTickCount();
// Get local changes
List <ItemChange> changes = DetectChanges(destinationKnowledge, batchSize);
// Update the knowledge with an updated local tick count
SyncKnowledge.SetLocalTickCount(tickCount);
// Construct the ChangeBatch and return it
ChangeBatch changeBatch = new ChangeBatch(IdFormats, destinationKnowledge, ForgottenKnowledge);
changeBatch.BeginUnorderedGroup();
changeBatch.AddChanges(changes);
if (changes.Count < batchSize || changes.Count == 0)
{
changeBatch.SetLastBatch();
}
changeBatch.EndUnorderedGroup(SyncKnowledge, changeBatch.IsLastBatch);
// Return the forgotten knowledge
forgottenKnowledge = ForgottenKnowledge;
changeDataRetriever = this;
return(changeBatch);
}
public List <ItemChange> GetChanges(ChangeBatch sourceChanges)
{
// Increment the tick count
GetNextTickCount();
// Increase local knowledge tick count.
SyncKnowledge.SetLocalTickCount(tickCount);
// Create a collection to hold the changes we'll put into our batch
List <ItemChange> changes = new List <ItemChange>();
foreach (ItemChange ic in sourceChanges)
{
ItemMetadata item;
ItemChange change;
// Iterate through each item to get the corresponding version in the local store
if (metadataStore.TryGetItem(ic.ItemId, out item))
{
// Found the corresponding item in the local metadata
// Get the local creation version and change (update) version from the metadata
change = new ItemChange(IdFormats, ReplicaId, item.ItemId, item.IsTombstone ? ChangeKind.Deleted: ChangeKind.Update, item.CreationVersion, item.ChangeVersion);
}
else
{
// Remote item has no local counterpart
// This item is unknown to us
change = new ItemChange(IdFormats, ReplicaId, ic.ItemId, ChangeKind.UnknownItem, SyncVersion.UnknownVersion, SyncVersion.UnknownVersion);
}
// Add our change to the change list
changes.Add(change);
}
return(changes);
}
public byte[] GetChangeBatch(uint batchSize, byte[] rawDestinationKnowledge,
out byte[] changeDataRetriever)
{
GenericRemoteSyncProvider <EntityObjectHierarchy> provider = GetSessionProvider();
byte[] retVal = null;
try
{
SyncKnowledge destinationKnowledge = SyncKnowledge.Deserialize(provider.IdFormats, rawDestinationKnowledge);
object dataRetriever;
ChangeBatch changeBatch = provider.GetChangeBatch(batchSize, destinationKnowledge, out dataRetriever);
CachedChangeDataRetriever cachedChangeDataRetriever =
new CachedChangeDataRetriever(dataRetriever as IChangeDataRetriever, changeBatch);
string debugCachedRetr = SerializerHelper.XmlSerialize(cachedChangeDataRetriever);
changeDataRetriever = SerializerHelper.BinarySerialize(cachedChangeDataRetriever);
retVal = changeBatch.Serialize();
}
catch (SyncException e)
{
throw SoapErrorCreator.RaiseException(HttpContext.Current.Request.Url.ToString(),
new SyncronizationServiceError(SyncronizationServiceError.eServiceErrorType.SyncFramework, e),
true);
}
catch (Exception e)
{
throw SoapErrorCreator.RaiseException(HttpContext.Current.Request.Url.ToString(),
new SyncronizationServiceError(SyncronizationServiceError.eServiceErrorType.SyncProvider, e),
true);
}
return(retVal);
}
private async Task <bool> WriteToRedshiftAsync(string tableName, ChangeBatch batch)
{
if (batch.Changes.Any())
{
var rowChanges = new List <RowChange>();
foreach (var record in batch.Changes)
{
rowChanges.Add(new RowChange()
{
ChangeKey = record.ChangeKey,
ChangeType = (CdcTools.Redshift.Changes.ChangeType)record.ChangeType,
Data = record.Data,
Lsn = record.LsnStr,
SeqVal = record.SeqValStr
});
}
try
{
await _redshiftClient.UploadAsCsvAsync(tableName, rowChanges);
return(true);
}
catch (Exception ex)
{
Console.WriteLine($"{tableName} upload failed. {ex}");
return(false);
}
}
return(true);
}
public virtual byte[] ProcessRemoteChangeBatch(
ConflictResolutionPolicy resolutionPolicy,
ChangeBatch sourceChanges,
object changeDataRetriever,
byte[] changeApplierInfo)
{
BeginTransaction();
//Get all my local change versions from the metadata store
IEnumerable <ItemChange> localChanges = _metaData.GetLocalVersions(sourceChanges);
//Create a changeapplier object to make change application easier (make the engine call me
//when it needs data and when I should save data)
NotifyingChangeApplier changeApplier = new NotifyingChangeApplier(IdFormats);
// The following step is required because we are remote change application
changeApplier.LoadChangeApplierInfo(changeApplierInfo);
changeApplier.ApplyChanges(
resolutionPolicy,
sourceChanges,
changeDataRetriever as IChangeDataRetriever,
localChanges,
base._metaData.GetKnowledge(),
base._metaData.GetForgottenKnowledge(),
this,
null, // Note that we do not pass a sync session context
new SyncCallbacks());
CommitTransaction();
// Return the ChangeApplierInfo
return(changeApplier.GetChangeApplierInfo());
}
public byte[] ProcessChangeBatch(int resolutionPolicy, byte[] sourceChanges,
byte[] rawChangeDataRetriever, byte[] changeApplierInfo)
{
GenericRemoteSyncProvider <EntityObjectHierarchy> provider = GetSessionProvider();
byte[] retVal = null;
try
{
ChangeBatch sourceChangeBatch =
ChangeBatch.Deserialize(provider.IdFormats, sourceChanges);
CachedChangeDataRetriever chachedDataRetriever =
SerializerHelper.BinaryDeserialize <CachedChangeDataRetriever>(rawChangeDataRetriever);
retVal = provider.ProcessRemoteChangeBatch((ConflictResolutionPolicy)resolutionPolicy, sourceChangeBatch,
chachedDataRetriever, changeApplierInfo);
}
catch (SyncException e)
{
throw SoapErrorCreator.RaiseException(HttpContext.Current.Request.Url.ToString(),
new SyncronizationServiceError(SyncronizationServiceError.eServiceErrorType.SyncFramework, e),
true);
}
catch (Exception e)
{
throw SoapErrorCreator.RaiseException(HttpContext.Current.Request.Url.ToString(),
new SyncronizationServiceError(SyncronizationServiceError.eServiceErrorType.SyncProvider, e),
true);
}
return(retVal);
}
//Change application!
public byte[] ProcessRemoteChangeBatch(
ConflictResolutionPolicy resolutionPolicy,
ChangeBatch sourceChanges,
CachedChangeDataRetriever changeDataRetriever,
byte[] changeApplierInfo)
{
_metadataStore.BeginTransaction();
//Get all my local change versions from the metadata store
IEnumerable <ItemChange> localChanges = _metadata.GetLocalVersions(sourceChanges);
NotifyingChangeApplier changeApplier = new NotifyingChangeApplier(_idFormats);
// The following step is required because we are remote change application
changeApplier.LoadChangeApplierInfo(changeApplierInfo);
changeApplier.ApplyChanges(
resolutionPolicy,
sourceChanges,
changeDataRetriever,
localChanges,
_metadata.GetKnowledge(),
_metadata.GetForgottenKnowledge(),
this,
null, // Note that we do not pass a sync session context
new SyncCallbacks());
_metadataStore.CommitTransaction();
// Return the ChangeApplierInfo
return(changeApplier.GetChangeApplierInfo());
}
private async Task <ChangeResourceRecordSetsResponse> createAws_A_ResourceRecords(string hostedZoneId, string hostName)
{
ChangeBatch changes = new ChangeBatch();
string newHostName = hostName.Substring(0, hostName.Length - 1);
logger.LogInformation($"Creating Alias Record for hosted zone { newHostName }...");
ResourceRecordSet aliasRs = new ResourceRecordSet()
{
AliasTarget = new AliasTarget()
{
HostedZoneId = "Z3AQBSTGFYJSTF", //use zone id from here: https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
DNSName = "s3-website-us-east-1.amazonaws.com",
EvaluateTargetHealth = false
},
Type = RRType.A,
Name = newHostName
};
Change AliasRecord = new Change(ChangeAction.CREATE, aliasRs);
changes.Changes.Add(AliasRecord);
ChangeResourceRecordSetsRequest request = new ChangeResourceRecordSetsRequest()
{
ChangeBatch = changes,
HostedZoneId = hostedZoneId
};
ChangeResourceRecordSetsResponse response = await TheRoute53Client().ChangeResourceRecordSetsAsync(request);
return(response);
}
private bool ApplyDnsChange(HostedZone zone, ResourceRecordSet recordSet, ChangeAction action)
{
// Prepare change as Batch
Change changeDetails = new Change()
{
ResourceRecordSet = recordSet,
Action = action
};
ChangeBatch changeBatch = new ChangeBatch()
{
Changes = new List <Change> {
changeDetails
}
};
// Prepare zone's resource record sets
var recordsetRequest = new ChangeResourceRecordSetsRequest()
{
HostedZoneId = zone.Id,
ChangeBatch = changeBatch
};
logger.Debug($"Route53 :: ApplyDnsChange : ChangeResourceRecordSets: {recordsetRequest.ChangeBatch} ");
var recordsetResponse = route53Client.ChangeResourceRecordSets(recordsetRequest);
logger.Debug($"Route53 :: ApplyDnsChange : ChangeResourceRecordSets Response: {recordsetResponse} ");
logger.Info("DNS change completed.");
return(true);
}
public ChangeResourceRecordSetsResponse updateRRSet(
string hostedZoneId, ResourceRecordSet oldRRset, ResourceRecordSet newRRset)
{
logger.Info("Calling ChangeResourceRecordSets");
Change delete = new Change()
{
Action = Action.DELETE.ToString(),
ResourceRecordSet = oldRRset
};
Change create = new Change()
{
Action = Action.CREATE.ToString(),
ResourceRecordSet = newRRset
};
List <Change> changes = new List <Change>()
{
delete, create
};
ChangeBatch batch = new ChangeBatch()
{
Changes = changes
};
return(client.ChangeResourceRecordSets(new ChangeResourceRecordSetsRequest()
{
HostedZoneId = hostedZoneId,
ChangeBatch = batch
}));
}
/// <summary>
/// Возвращает пакет изменений, содержащий метаданные элементов, которые отсутствовали в указанном наборе знаний от поставщика назначения
/// </summary>
/// <param name="batchSize">Size of the batch.</param>
/// <param name="destinationKnowledge">The destination knowledge.</param>
/// <returns></returns>
public override ChangeBatch GetChangeBatch(uint batchSize, SyncKnowledge destinationKnowledge)
{
ChangeBatch retVal = null;
ulong tickCount = GetNextTickCount();
List <ItemChange> changes = DetectChanges(destinationKnowledge, batchSize);
retVal = new ChangeBatch(IdFormats, destinationKnowledge, Replica.ForgottenKnowledge);
// Add the changes to the ChangeBatch with our made with knowledge
// (Made width knowledge is the knowledge the other side will "learn" if they apply these
// changes successfully)
retVal.BeginUnorderedGroup();
retVal.AddChanges(changes);
// If last change batch, mark accordingly
// (We always enumerate full batches, so if our batch is less than the batch size we
// must be at the last batch. The second condition is spurious.)
bool isLastBatch = false;
if ((changes.Count < batchSize) || (changes.Count == 0))
{
retVal.SetLastBatch();
isLastBatch = true;
}
retVal.EndUnorderedGroup(Replica.CurrentKnowledge, isLastBatch);
return(retVal);
}
public async Task <bool> UpdateIpAddressForSubdomain(string hostedZoneId, string fqdn, string newIpAddress)
{
using (AmazonRoute53Client route53Client = GetAmazonRoute53Client())
{
ListResourceRecordSetsResponse records = await route53Client.ListResourceRecordSetsAsync(new ListResourceRecordSetsRequest(hostedZoneId));
// Look for an A record matching the FQDN that was passed in
ResourceRecordSet matchingRecordSet = records?.ResourceRecordSets.FirstOrDefault(prop => prop.Name == fqdn && prop.Type == RRType.A);
if (matchingRecordSet != null && matchingRecordSet.ResourceRecords.Any())
{
if (matchingRecordSet.ResourceRecords.First().Value != newIpAddress)
{
matchingRecordSet.ResourceRecords.First().Value = newIpAddress;
ChangeBatch change = new ChangeBatch();
change.Changes.Add(new Change(ChangeAction.UPSERT, matchingRecordSet));
ChangeResourceRecordSetsResponse changeRequest = await route53Client.ChangeResourceRecordSetsAsync(new ChangeResourceRecordSetsRequest(hostedZoneId, change));
Log.Information("[Runtime = {StartTime}] Change request submitted to change subdomain {Subdomain} IP address to {IPAddress}.", Settings.StartTime, fqdn, newIpAddress);
return(changeRequest.HttpStatusCode == System.Net.HttpStatusCode.OK);
}
else
{
Log.Information("[Runtime = {StartTime}] Subdomain {Subdomain} found, but the IP address was already {IPAddress}.", Settings.StartTime, fqdn, newIpAddress);
}
}
return(false);
}
}
public override void ProcessChangeBatch(ConflictResolutionPolicy resolutionPolicy, ChangeBatch sourceChanges,
object changeDataRetriever, SyncCallbacks syncCallback, SyncSessionStatistics sessionStatistics)
{
_metadataStore.BeginTransaction();
IEnumerable<ItemChange> localChanges = _metadataStore.Metadata.GetLocalVersions(sourceChanges);
NotifyingChangeApplier changeApplier = new NotifyingChangeApplier(_idFormats);
changeApplier.ApplyChanges(resolutionPolicy, sourceChanges, changeDataRetriever as IChangeDataRetriever, localChanges, _metadataStore.Metadata.GetKnowledge(),
_metadataStore.Metadata.GetForgottenKnowledge(), _changeApplier, _currentSessionContext, syncCallback);
_metadataStore.CommitTransaction();
}
public void ApplyChanges(
ConflictResolutionPolicy resolutionPolicy,
ChangeBatch sourceChanges,
object changeData,
ref SyncSessionStatistics sessionStatistics)
{
SyncCallbacks syncCallback = new SyncCallbacks();
peerProvider.ProcessChangeBatch(resolutionPolicy, sourceChanges, changeData, syncCallback,
sessionStatistics);
}
public override ChangeBatch GetChangeBatch(uint batchSize, SyncKnowledge destinationKnowledge, out object changeDataRetriever)
{
var temp = _metadata.GetChangeBatch(uint.MaxValue, destinationKnowledge);
ChangeBatch batch = _metadata.GetChangeBatch(batchSize, destinationKnowledge);
changeDataRetriever = this; //this is where the transfer mechanism/protocol would go. For an in memory provider, this is sufficient
var batchCount = (uint)batch.Count();
batch.BatchWorkEstimate = batchCount;
batch.RemainingSessionWorkEstimate = (uint)temp.Count();
return(batch);
}
public ItemsChangeInfo GetChanges(string folderPath, ChangeBatch sourceChanges, string[] filters)
{
using (RemoteSyncDetails sync = new RemoteSyncDetails(folderPath, filters)) {
List <ItemChangeMetadata> itemChanges = sync.GetMetadataForChanges(sourceChanges);
ItemsChangeInfo lst = new ItemsChangeInfo();
lst.IdFormats = sync.IdFormats;
lst.ReplicaId = sync.ReplicaId;
lst.ItemChanges = itemChanges;
return(lst);
}
}
private static async Task UpdateResourceRecordSet(ResourceRecordSet resourceRecordSet, Domain domain)
{
var change = new Change(ChangeAction.UPSERT, resourceRecordSet);
var changeBatch = new ChangeBatch(new List <Change> {
change
});
var request = new ChangeResourceRecordSetsRequest(domain.HostedZoneID, changeBatch);
//var response = await client.ChangeResourceRecordSetsAsync(request);
}
public byte[] ProcessChangeBatch(
ConflictResolutionPolicy resolutionPolicy,
ChangeBatch sourceChanges,
Sync101.CachedChangeDataRetriever changeDataRetriever,
byte[] changeApplierInfo)
{
return(base.Channel.ProcessChangeBatch(
resolutionPolicy,
sourceChanges,
changeDataRetriever,
changeApplierInfo));
}
public byte[] ProcessChangeBatch(
ConflictResolutionPolicy resolutionPolicy,
ChangeBatch sourceChanges,
CachedChangeDataRetriever changeDataRetriever,
byte[] changeApplierInfo)
{
return(provider.ProcessRemoteChangeBatch(
resolutionPolicy,
sourceChanges,
changeDataRetriever,
changeApplierInfo));
}
/// <summary>
/// Create or change a DNS record for a hosted zone.
/// </summary>
/// <param name="hostedZoneId">The ID of the hosted zone that contains the resource record sets that you want to change</param>
/// <param name="name">The name of the DNS record set.</param>
/// <param name="type">The type of the DNS record set.</param>
/// <param name="value">The value of the record set.</param>
/// <param name="ttl">The time to live of the record set.</param>
/// <param name="settings">The <see cref="Route53Settings"/> required to upload to Amazon S3.</param>
/// <param name="cancellationToken">A cancellation token that can be used by other objects or threads to receive notice of cancellation.</param>
public async Task <string> CreateResourceRecordSet(string hostedZoneId, string name, RRType type, string value, long ttl, Route53Settings settings, CancellationToken cancellationToken = default(CancellationToken))
{
var recordSet = new ResourceRecordSet()
{
Name = name,
TTL = ttl,
Type = type,
ResourceRecords = new List <ResourceRecord>
{
new ResourceRecord {
Value = value
}
}
};
var change1 = new Change()
{
ResourceRecordSet = recordSet,
Action = ChangeAction.UPSERT
};
var changeBatch = new ChangeBatch()
{
Changes = new List <Change> {
change1
}
};
var recordsetRequest = new ChangeResourceRecordSetsRequest()
{
HostedZoneId = hostedZoneId,
ChangeBatch = changeBatch
};
AmazonRoute53Client client = this.GetClient(settings);
ChangeResourceRecordSetsResponse response = await client.ChangeResourceRecordSetsAsync(recordsetRequest);
if (response.HttpStatusCode == HttpStatusCode.OK)
{
await this.WaitForChange(client, response.ChangeInfo.Id, 10000, 60);
_Log.Verbose("Updated record set");
return(response.ChangeInfo.Id);
}
else
{
_Log.Error("Could not change resource records");
return("");
}
}
/// <summary>
/// Calculates the next SpSyncAnchor from the current SpSyncAnchor object and the change batch just returned from the server
/// </summary>
/// <param name="currentAnchor">the current SpSyncAnchor object</param>
/// <param name="changes"> the ChangeBatch object</param>
/// <returns>the new SpSyncAnchor for the next incremental select</returns>
protected SpSyncAnchor CalculateNextAnchor(SpSyncAnchor currentAnchor, ChangeBatch changes)
{
SpSyncAnchor nextChanges = currentAnchor.NextChangesAnchor ?? new SpSyncAnchor(changes.NextChangeBatch, null);
SpSyncAnchor nextAnchor = nextChanges;
if (changes.HasMoreData())
{
nextAnchor = new SpSyncAnchor(currentAnchor.NextChangesToken, changes.NextPage);
nextAnchor.NextChangesAnchor = nextChanges;
}
nextAnchor.HasMoreData = changes.HasMoreChanges();
return(nextAnchor);
}
/// <summary>
/// When overridden in a derived class, gets a change batch that contains item metadata for items that are not contained in the specified knowledge from the destination provider.
/// </summary>
/// <param name="batchSize">The number of changes to include in the change batch.</param>
/// <param name="destinationKnowledge">The knowledge from the destination provider. This knowledge must be mapped by calling <see cref="M:Microsoft.Synchronization.SyncKnowledge.MapRemoteKnowledgeToLocal(Microsoft.Synchronization.SyncKnowledge)"/> on the source knowledge before it can be used for change enumeration.</param>
/// <param name="changeDataRetriever">Returns an object that can be used to retrieve change data. It can be an <see cref="T:Microsoft.Synchronization.IChangeDataRetriever"/> object or a provider-specific object.</param>
/// <returns>
/// A change batch that contains item metadata for items that are not contained in the specified knowledge from the destination provider. Cannot be a null.
/// </returns>
public override ChangeBatch GetChangeBatch(uint batchSize, SyncKnowledge destinationKnowledge, out object changeDataRetriever)
{
byte[] rawDestinationKnowledge = destinationKnowledge.Serialize();
byte[] rawChangeDataRetriever;
byte[] rawChangeBatch = _syncService.GetChangeBatch(batchSize, rawDestinationKnowledge,
out rawChangeDataRetriever);
CachedChangeDataRetriever cachedRetriever =
SerializerHelper.BinaryDeserialize <CachedChangeDataRetriever>(rawChangeDataRetriever);
changeDataRetriever = cachedRetriever;
return(ChangeBatch.Deserialize(IdFormats, rawChangeBatch));
}
public override ChangeBatch GetChangeBatch(uint batchSize, SyncKnowledge destinationKnowledge, out object changeDataRetriever)
{
ChangeBatch batch = _metaData.GetChangeBatch(batchSize, destinationKnowledge);
changeDataRetriever = this; //this is where the transfer mechanism/protocol would go. For an in memory provider, this is sufficient
//Calculate estimate work
batch.RemainingSessionWorkEstimate = RemainingSessionWorkEstimate;
batch.BatchWorkEstimate = batch.IsLastBatch ? (uint)batch.Count() : batchSize;
RemainingSessionWorkEstimate = batch.IsLastBatch ? 0 : RemainingSessionWorkEstimate - batchSize;
return(batch);
}
public ChangeResourceRecordSetsResponse updateRRSet(string hostedZoneId, ResourceRecordSet oldRRset, ResourceRecordSet newRRset)
{
logger.Info("Calling ChangeResourceRecordSets");
List <Change> changes = new List <Change>()
{
new Change().WithAction(Action.DELETE.ToString()).WithResourceRecordSet(oldRRset),
new Change().WithAction(Action.CREATE.ToString()).WithResourceRecordSet(newRRset)
};
ChangeBatch batch = new ChangeBatch().WithChanges(changes.ToArray());
return(client.ChangeResourceRecordSets(new ChangeResourceRecordSetsRequest()
.WithHostedZoneId(hostedZoneId)
.WithChangeBatch(batch)));
}
private static void CreateHostedZone(IAmazonRoute53 r53Client, string domainName)
{
var zoneRequest = new CreateHostedZoneRequest
{
Name = domainName,
CallerReference = "testingss"
};
var zoneResponse = r53Client.CreateHostedZone(zoneRequest);
var recordSet = new ResourceRecordSet
{
Name = domainName,
TTL = 60,
Type = RRType.A,
ResourceRecords = new List <ResourceRecord> {
new ResourceRecord {
Value = IpAddress
}
}
};
var change1 = new Change
{
ResourceRecordSet = recordSet,
Action = ChangeAction.CREATE
};
var changeBatch = new ChangeBatch
{
Changes = new List <Change> {
change1
}
};
var recordsetRequest = new ChangeResourceRecordSetsRequest
{
HostedZoneId = zoneResponse.HostedZone.Id,
ChangeBatch = changeBatch
};
var recordsetResponse = r53Client.ChangeResourceRecordSets(recordsetRequest);
var changeRequest = new GetChangeRequest
{
Id = recordsetResponse.ChangeInfo.Id
};
Console.WriteLine(changeRequest);
}
public override ChangeBatch GetChangeBatch(
uint batchSize,
SyncKnowledge destinationKnowledge,
out object changeDataRetriever)
{
CachedChangeDataRetriever cachedChangeDataRetriever;
ChangeBatch changeBatch = this.client.GetChangeBatch(
batchSize,
destinationKnowledge,
out cachedChangeDataRetriever);
changeDataRetriever = cachedChangeDataRetriever;
return(changeBatch);
}
private static void CreateDNSChangeAction(string domain, string hostName, string value, RRType type, ChangeAction action, int ttl)
{
string zoneId = FindHostedZoneID(domain);
if (zoneId == null)
{
throw new Exception("Zone not found");
}
ResourceRecord resourceRecord = null;
if (value != null)
{
resourceRecord = new ResourceRecord()
{
Value = value
};
}
var change = new Change
{
Action = action,
ResourceRecordSet = new ResourceRecordSet
{
Name = $"{hostName}.{domain}",
Type = type,
TTL = ttl,
ResourceRecords = resourceRecord != null ? new List <ResourceRecord>()
{
resourceRecord
} : null
},
};
var changeBatch = new ChangeBatch();
changeBatch.Changes.Add(change);
var changeResourceRecordSetsRequest = new ChangeResourceRecordSetsRequest
{
ChangeBatch = changeBatch,
HostedZoneId = zoneId
};
var changeResourceResponse = c.ChangeResourceRecordSetsAsync(changeResourceRecordSetsRequest).Result;
Console.WriteLine($"{changeResourceResponse.ChangeInfo.Status} {changeResourceResponse.ChangeInfo.Comment}");
}
private async Task <bool> ApplyDnsChange(HostedZone zone, ResourceRecordSet recordSet, ChangeAction action)
{
// prepare change
var changeDetails = new Change()
{
ResourceRecordSet = recordSet,
Action = action
};
var changeBatch = new ChangeBatch()
{
Changes = new List <Change> {
changeDetails
}
};
// Update the zone's resource record sets
var recordsetRequest = new ChangeResourceRecordSetsRequest()
{
HostedZoneId = zone.Id,
ChangeBatch = changeBatch
};
_log?.Debug($"Route53 :: ApplyDnsChange : ChangeResourceRecordSetsAsync: {JsonConvert.SerializeObject(recordsetRequest.ChangeBatch)} ");
var recordsetResponse = await _route53Client.ChangeResourceRecordSetsAsync(recordsetRequest);
_log?.Debug($"Route53 :: ApplyDnsChange : ChangeResourceRecordSetsAsync Response: {JsonConvert.SerializeObject(recordsetResponse)} ");
// Monitor the change status
var changeRequest = new GetChangeRequest()
{
Id = recordsetResponse.ChangeInfo.Id
};
while (ChangeStatus.PENDING == (await _route53Client.GetChangeAsync(changeRequest)).ChangeInfo.Status)
{
System.Diagnostics.Debug.WriteLine("DNS change is pending.");
await Task.Delay(1500);
}
_log?.Information("DNS change completed.");
return(true);
}
public SyncSessionStatistics ApplyChanges(ConflictResolutionPolicy resolutionPolicy, ChangeBatch sourceChanges, object changeData)
{
DbSyncContext dataRetriever = changeData as DbSyncContext;
if (dataRetriever != null && dataRetriever.IsDataBatched)
{
string remotePeerId = dataRetriever.MadeWithKnowledge.ReplicaId.ToString();
//Data is batched. The client should have uploaded this file to us prior to calling ApplyChanges.
//So look for it.
//The Id would be the DbSyncContext.BatchFileName which is just the batch file name without the complete path
string localBatchFileName = null;
if (!this.batchIdToFileMapper.TryGetValue(dataRetriever.BatchFileName, out localBatchFileName))
{
//Service has not received this file. Throw exception
throw new FaultException<WebSyncFaultException>(new WebSyncFaultException("No batch file uploaded for id " + dataRetriever.BatchFileName, null));
}
dataRetriever.BatchFileName = localBatchFileName;
}
SyncSessionStatistics sessionStatistics = new SyncSessionStatistics();
this.peerProvider.ProcessChangeBatch(resolutionPolicy, sourceChanges, changeData, new SyncCallbacks(), sessionStatistics);
return sessionStatistics;
}
/// <summary>
/// Возвращает пакет изменений, содержащий версии элементов и базовые единицы, которые хранятся в данной реплике.
/// Они соответствуют элементам и базовым единицам, на которые были ссылки в пакете изменений, полученном от другого поставщика.
/// </summary>
/// <param name="sourceChanges">The source changes.</param>
/// <returns></returns>
public override IEnumerable<ItemChange> GetLocalVersions(ChangeBatch sourceChanges)
{
if (sourceChanges == null)
{
throw new ArgumentNullException("sourceChanges");
}
ulong tickCount = GetNextTickCount();
Replica.CurrentKnowledge.SetLocalTickCount(tickCount);
// Iterate over changes in the source ChangeBatch
foreach (ItemChange itemChange in sourceChanges)
{
ItemChange change = null;
ItemMetadata item = Replica.FindItemMetadataById(itemChange.ItemId);
// Iterate through each item to get the corresponding version in the local store
if (item != null)
{
// Found the corresponding item in the local metadata
// Get the local creation version and change (update) version from the metadata
change = new ItemChange(IdFormats, ReplicaId, item.GlobalId,
item.IsDeleted ? ChangeKind.Deleted : ChangeKind.Update, // If local item is a tombstone, mark it accordingly
item.CreationVersion, item.ChangeVersion);
}
else
{
// Remote item has no local counterpart
// This item is unknown to us
change = new ItemChange(IdFormats, ReplicaId, itemChange.ItemId,
ChangeKind.UnknownItem, // Mark the change as unknown
SyncVersion.UnknownVersion, SyncVersion.UnknownVersion);
}
// Add our change to the change list
yield return change;
}
}
public override void ProcessChangeBatch(ConflictResolutionPolicy resolutionPolicy, ChangeBatch sourceChanges, object changeDataRetriever, SyncCallbacks syncCallbacks, SyncSessionStatistics sessionStatistics)
{
DbSyncContext context = changeDataRetriever as DbSyncContext;
if (context != null && context.IsDataBatched)
{
string fileName = new FileInfo(context.BatchFileName).Name;
//Retrieve the remote peer id from the MadeWithKnowledge.ReplicaId. MadeWithKnowledge is the local knowledge of the peer
//that is enumerating the changes.
string peerId = context.MadeWithKnowledge.ReplicaId.ToString();
//Check to see if service already has this file
if (!this.proxy.HasUploadedBatchFile(fileName, peerId))
{
//Upload this file to remote service
FileStream stream = new FileStream(context.BatchFileName, FileMode.Open, FileAccess.Read);
byte[] contents = new byte[stream.Length];
using (stream)
{
stream.Read(contents, 0, contents.Length);
}
this.proxy.UploadBatchFile(fileName, contents, peerId);
}
context.BatchFileName = fileName;
}
SyncSessionStatistics stats = this.proxy.ApplyChanges(resolutionPolicy, sourceChanges, changeDataRetriever);
sessionStatistics.ChangesApplied += stats.ChangesApplied;
sessionStatistics.ChangesFailed += stats.ChangesFailed;
}
/// <summary>
/// When overridden in a derived class, processes a set of changes by detecting conflicts and applying changes to the item store.
/// </summary>
/// <param name="resolutionPolicy">The conflict resolution policy to use when this method applies changes.</param>
/// <param name="sourceChanges">A batch of changes from the source provider to be applied locally.</param>
/// <param name="changeDataRetriever">An object that can be used to retrieve change data. It can be an <see cref="T:Microsoft.Synchronization.IChangeDataRetriever"/> object or a provider-specific object.</param>
/// <param name="syncCallbacks">An object that receives event notifications during change application.</param>
/// <param name="sessionStatistics">Tracks change statistics. For a provider that uses custom change application, this object must be updated with the results of the change application.</param>
public override void ProcessChangeBatch(ConflictResolutionPolicy resolutionPolicy, ChangeBatch sourceChanges, object changeDataRetriever, SyncCallbacks syncCallbacks, SyncSessionStatistics sessionStatistics)
{
CachedChangeDataRetriever cachedChangeDataRetriever = new CachedChangeDataRetriever(
changeDataRetriever as IChangeDataRetriever,
sourceChanges);
byte[] rawSourceChanges = sourceChanges.Serialize();
byte[] rawCachedChangeDataRetriever = SerializerHelper.BinarySerialize(cachedChangeDataRetriever);
byte[] newChangeApplierInfo = _syncService.ProcessChangeBatch(
(int)resolutionPolicy,
rawSourceChanges,
rawCachedChangeDataRetriever,
_syncSessionContext.ChangeApplierInfo);
_syncSessionContext.ChangeApplierInfo = newChangeApplierInfo;
}
/// <inheritdoc />
public override void ProcessChangeBatch(ConflictResolutionPolicy resolutionPolicy, ChangeBatch sourceChanges, object changeDataRetriever, SyncCallbacks syncCallbacks, SyncSessionStatistics sessionStatistics)
{
// anyone know how to implement ???
throw new NotImplementedException();
}
/// <summary>
/// Возвращает пакет изменений, содержащий метаданные элементов, которые отсутствовали в указанном наборе знаний от поставщика назначения
/// </summary>
/// <param name="batchSize">Size of the batch.</param>
/// <param name="destinationKnowledge">The destination knowledge.</param>
/// <returns></returns>
public override ChangeBatch GetChangeBatch(uint batchSize, SyncKnowledge destinationKnowledge)
{
ChangeBatch retVal = null;
ulong tickCount = GetNextTickCount();
List<ItemChange> changes = DetectChanges(destinationKnowledge, batchSize);
retVal = new ChangeBatch(IdFormats, destinationKnowledge, Replica.ForgottenKnowledge);
// Add the changes to the ChangeBatch with our made with knowledge
// (Made width knowledge is the knowledge the other side will "learn" if they apply these
// changes successfully)
retVal.BeginUnorderedGroup();
retVal.AddChanges(changes);
// If last change batch, mark accordingly
// (We always enumerate full batches, so if our batch is less than the batch size we
// must be at the last batch. The second condition is spurious.)
bool isLastBatch = false;
if ((changes.Count < batchSize) || (changes.Count == 0))
{
retVal.SetLastBatch();
isLastBatch = true;
}
retVal.EndUnorderedGroup(Replica.CurrentKnowledge, isLastBatch);
return retVal;
}
/// <summary>
/// Download Mechanism
/// </summary>
/// <param name="resolutionPolicy"></param>
/// <param name="sourceChanges"></param>
/// <param name="changeDataRetriever"></param>
/// <param name="syncCallback"></param>
/// <param name="sessionStatistics"></param>
public override void ProcessChangeBatch(ConflictResolutionPolicy resolutionPolicy, ChangeBatch sourceChanges,
object changeDataRetriever, SyncCallbacks syncCallback, SyncSessionStatistics sessionStatistics)
{
ChangeBatch localVersions = sync.GetChanges(sourceChanges);
ForgottenKnowledge destinationForgottenKnowledge = new ForgottenKnowledge(sync.IdFormats, sync.SyncKnowledge);
NotifyingChangeApplier changeApplier = new NotifyingChangeApplier(sync.IdFormats);
changeApplier.ApplyChanges(resolutionPolicy, CollisionConflictResolutionPolicy.Merge, sourceChanges,
(IChangeDataRetriever)changeDataRetriever, localVersions, sync.SyncKnowledge.Clone(),
destinationForgottenKnowledge, this, _memConflictLog, currentSessionContext, syncCallback);
}
public ChangeBatch GetChanges(ChangeBatch sourceChanges)
{
GetNextTickCount();
myKnowledge.SetLocalTickCount(tickCount);
List<ItemChange> changes = new List<ItemChange>();
foreach (ItemChange ic in sourceChanges)
{
ItemMetadata item;
ItemChange change;
if (metadataStore.TryGetItem(ic.ItemId, out item))
{
System.Diagnostics.Debug.WriteLine("Remote item has local counterpart::" + item.Uri);
change = new ItemChange(IdFormats, ReplicaId, item.ItemId,
(item.IsTombstone ? ChangeKind.Deleted : ChangeKind.Update),
item.CreationVersion, item.ChangeVersion);
}
else
{
if (item == null)
System.Diagnostics.Debug.WriteLine("Remote item has no local counterpart: item.uri is null");
else
System.Diagnostics.Debug.WriteLine("Remote item has no local counterpart:" + item.Uri);
change = new ItemChange(IdFormats, replicaId, ic.ItemId, ChangeKind.UnknownItem,
SyncVersion.UnknownVersion, SyncVersion.UnknownVersion);
}
changes.Add(change);
}
ChangeBatch changeBatchBuilder = new ChangeBatch(IdFormats, myKnowledge , myForgottenKnowledge);
changeBatchBuilder.BeginUnorderedGroup();
changeBatchBuilder.AddChanges(changes);
changeBatchBuilder.EndUnorderedGroup(myKnowledge, true);
return changeBatchBuilder;
}
public ChangeBatch GetChangeBatch(string path, uint batchSize, SyncKnowledge destinationKnowledge, out object changeDataRetriever)
{
folderPath = path;
GetNextTickCount();
List<ItemChange> changes = DetectChanges(destinationKnowledge, batchSize);
myKnowledge.SetLocalTickCount(tickCount);
ChangeBatch changeBatchBuilder = new ChangeBatch(IdFormats,destinationKnowledge, myForgottenKnowledge);
changeBatchBuilder.BeginUnorderedGroup();
changeBatchBuilder.AddChanges(changes);
changeBatchBuilder.EndUnorderedGroup(myKnowledge, true);
if ((changes.Count < batchSize) || (changes.Count == 0))
{
changeBatchBuilder.SetLastBatch();
}
changeDataRetriever = this;
return changeBatchBuilder;
}
/// <summary>
/// Upload mechanism
/// </summary>
/// <param name="resolutionPolicy"></param>
/// <param name="sourceChanges">Local File Changes</param>
/// <param name="changeDataRetriever"></param>
/// <param name="syncCallback"></param>
/// <param name="sessionStatistics"></param>
public override void ProcessChangeBatch(ConflictResolutionPolicy resolutionPolicy, ChangeBatch sourceChanges, object changeDataRetriever, SyncCallbacks syncCallback, SyncSessionStatistics sessionStatistics)
{
myForgottenKnowledge = new ForgottenKnowledge(IdFormats, myKnowledge);
NotifyingChangeApplier changeApplier = new NotifyingChangeApplier(IdFormats);
changeApplier.ApplyChanges(resolutionPolicy, sourceChanges, (IChangeDataRetriever)changeDataRetriever, myKnowledge.Clone(), myForgottenKnowledge,
this, currentSessionContext, syncCallback);
}
public override IEnumerable<ItemChange> GetFilteredLocalVersions(ChangeBatch sourceChanges, ReplicaMetadata.ItemFilterCallback filterCallback)
{
throw new NotImplementedException();
}
//Change application!
public override void ProcessChangeBatch(ConflictResolutionPolicy resolutionPolicy, ChangeBatch sourceChanges,
object changeDataRetriever, SyncCallbacks syncCallback, SyncSessionStatistics sessionStatistics)
{
_metadataStore.BeginTransaction();
//Get all my local change versions from the metadata store
IEnumerable<ItemChange> localChanges = _metadata.GetLocalVersions(sourceChanges);
//Create a changeapplier object to make change application easier (make the engine call me
//when it needs data and when I should save data)
NotifyingChangeApplier changeApplier = new NotifyingChangeApplier(_idFormats);
changeApplier.ApplyChanges(resolutionPolicy, sourceChanges, changeDataRetriever as IChangeDataRetriever, localChanges, _metadata.GetKnowledge(),
_metadata.GetForgottenKnowledge(), this, _currentSessionContext, syncCallback);
_metadataStore.CommitTransaction();
}
