public bool ContainsOverlapLink(LinkMetadata metadata)
{
bool flag = metadata != null && metadata.AttributeName.Equals(this.LinkName, StringComparison.OrdinalIgnoreCase) && metadata.TargetDistinguishedName.Equals(this.LinkTarget, StringComparison.OrdinalIgnoreCase) && metadata.Version == this.LinkVersion;
ExTraceGlobals.BackSyncTracer.TraceDebug <bool>((long)SyncConfiguration.TraceId, "FullSyncObjectCookie.ContainsOverlapLink return {0}", flag);
return(flag);
}
public void Initialize()
{
ILinkMetadata linkMetaData = new LinkMetadata();
linkMetaData.DisplayName = "Home";
linkMetaData.ParentName = "ZoeProg Center";
linkMetaData.Source = $"/ZoeProg.PlugIns.Home;component/Views/{nameof(HomeView)}.xaml";
this.linkMetadataService.Registry(linkMetaData);
}
public void SetNextPageData(SyncObjectId objectId, LinkMetadata overlapLink, int nextRangeStart, long usnChanged)
{
this.ObjectId = objectId;
ExTraceGlobals.BackSyncTracer.TraceDebug <string>((long)SyncConfiguration.TraceId, "FullSyncObjectCookie.SetNextPageData this.ObjectId = {0}", (this.ObjectId != null) ? this.ObjectId.ObjectId : "NULL");
this.EnumerateLinks = true;
this.LinkName = overlapLink.AttributeName;
this.LinkRangeStart = nextRangeStart;
this.LinkTarget = overlapLink.TargetDistinguishedName;
this.LinkVersion = overlapLink.Version;
this.UsnChanged = usnChanged;
}
private bool CheckLinkOverlap(MultiValuedProperty <LinkMetadata> metadata)
{
int num = Math.Min(10, metadata.Count);
for (int i = metadata.Count - num; i < metadata.Count; i++)
{
LinkMetadata metadata2 = metadata[i];
if (this.PageToken.ObjectCookie.ContainsOverlapLink(metadata2))
{
ExTraceGlobals.BackSyncTracer.TraceDebug((long)SyncConfiguration.TraceId, "ObjectFullSyncConfiguration.CheckLinkOverlap return true");
return(true);
}
}
ExTraceGlobals.BackSyncTracer.TraceDebug((long)SyncConfiguration.TraceId, "ObjectFullSyncConfiguration.CheckLinkOverlap return false");
return(false);
}
public EventData CreateLink(EventMetadata em, IEvent ev, Type projectionType, Guid projectionVersion)
{
var category = ServiceConventions.GetCategoryFromNamespaceFunc(em.AggregateType.Namespace);
var content = $"{em.Version}@{category}-{em.AggregateId}";
var metadata = new LinkMetadata(projectionType, em.CorrelationId, projectionVersion);
var contentBytes = Encoding.UTF8.GetBytes(content);
var metadataBytes = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(metadata));
Log.Debug("Creating a link {link}", content);
//return new EventData(Uuid.NewUuid(), "$>", contentBytes, metadataBytes);
//return new EventData(Uuid.FromGuid(ev.Id), "$>", contentBytes, metadataBytes, "application/octet-stream");
return(new EventData(Uuid.NewUuid(), "$>", contentBytes, metadataBytes, "application/octet-stream"));
//return new EventData(ev.Id, "$>", false, contentBytes, metadataBytes);
}
public void Initialize()
{
ILinkMetadata linkMetaData = new LinkMetadata();
linkMetaData.DisplayName = "Junks";
linkMetaData.ParentName = "Clean explorer";
linkMetaData.Source = $"/ZoeProg.PlugIns.Cleanup;component/Views/{nameof(CleanupView)}.xaml";
this.moduleMetadataService.Registry(linkMetaData);
//Cookies
ILinkMetadata cookieLinkMetaData = new LinkMetadata();
cookieLinkMetaData.DisplayName = "Cookies";
cookieLinkMetaData.ParentName = "Clean explorer";
cookieLinkMetaData.Source = $"/ZoeProg.PlugIns.Cleanup;component/Views/{nameof(CookieView)}.xaml";
this.moduleMetadataService.Registry(cookieLinkMetaData);
}
private static SyncLink ConvertLinkMetadataToSyncLink(LinkMetadata metadata)
{
return(new SyncLink(new ADObjectId(metadata.TargetDistinguishedName), metadata.IsDeleted ? LinkState.Removed : LinkState.Added));
}
public static async Task <(GenerationReturn generationReturn, LinkMetadata metadata)> LinkMetadataFromUrl(
string url, IProgress <string> progress)
{
if (string.IsNullOrWhiteSpace(url))
{
return(await GenerationReturn.Error("No URL?"), null);
}
progress?.Report("Setting up and Downloading Site");
var toReturn = new LinkMetadata();
var config = Configuration.Default.WithDefaultLoader().WithJs();
var context = BrowsingContext.New(config);
var document = await context.OpenAsync(url);
progress?.Report("Looking for Title");
var titleString = document.Head?.Children.FirstOrDefault(x => x.TagName == "TITLE")?.TextContent;
if (string.IsNullOrWhiteSpace(titleString))
{
titleString = document.QuerySelector("meta[property='og:title']")?.Attributes
.FirstOrDefault(x => x.LocalName == "content")?.Value;
}
if (string.IsNullOrWhiteSpace(titleString))
{
titleString = document.QuerySelector("meta[name='DC.title']")?.Attributes
.FirstOrDefault(x => x.LocalName == "content")?.Value;
}
if (string.IsNullOrWhiteSpace(titleString))
{
titleString = document.QuerySelector("meta[name='twitter:title']")?.Attributes
.FirstOrDefault(x => x.LocalName == "value")?.Value;
}
if (!string.IsNullOrWhiteSpace(titleString))
{
toReturn.Title = titleString;
}
progress?.Report("Looking for Author");
var authorString = document.QuerySelector("meta[property='og:author']")?.Attributes
.FirstOrDefault(x => x.LocalName == "content")?.Value;
if (string.IsNullOrWhiteSpace(authorString))
{
authorString = document.QuerySelector("meta[name='DC.contributor']")?.Attributes
.FirstOrDefault(x => x.LocalName == "content")?.Value;
}
if (string.IsNullOrWhiteSpace(authorString))
{
authorString = document.QuerySelector("meta[property='article:author']")?.Attributes
.FirstOrDefault(x => x.LocalName == "content")?.Value;
}
if (string.IsNullOrWhiteSpace(authorString))
{
authorString = document.QuerySelector("meta[name='author']")?.Attributes
.FirstOrDefault(x => x.LocalName == "content")?.Value;
}
if (string.IsNullOrWhiteSpace(authorString))
{
authorString = document.QuerySelector("a[rel~=\"author\"]")?.TextContent;
}
if (string.IsNullOrWhiteSpace(authorString))
{
authorString = document.QuerySelector(".author__name")?.TextContent;
}
if (string.IsNullOrWhiteSpace(authorString))
{
authorString = document.QuerySelector(".author_name")?.TextContent;
}
if (!string.IsNullOrWhiteSpace(authorString))
{
toReturn.Author = authorString;
}
progress?.Report($"Looking for Author - Found {toReturn.Author}");
progress?.Report("Looking for Date Time");
var linkDateString = document.QuerySelector("meta[property='article:modified_time']")?.Attributes
.FirstOrDefault(x => x.LocalName == "content")?.Value;
if (string.IsNullOrWhiteSpace(linkDateString))
{
linkDateString = document.QuerySelector("meta[property='og:updated_time']")?.Attributes
.FirstOrDefault(x => x.LocalName == "content")?.Value;
}
if (string.IsNullOrWhiteSpace(linkDateString))
{
linkDateString = document.QuerySelector("meta[property='article:published_time']")?.Attributes
.FirstOrDefault(x => x.LocalName == "content")?.Value;
}
if (string.IsNullOrWhiteSpace(linkDateString))
{
linkDateString = document.QuerySelector("meta[property='article:published_time']")?.Attributes
.FirstOrDefault(x => x.LocalName == "content")?.Value;
}
if (string.IsNullOrWhiteSpace(linkDateString))
{
linkDateString = document.QuerySelector("meta[name='DC.date.created']")?.Attributes
.FirstOrDefault(x => x.LocalName == "content")?.Value;
}
progress?.Report($"Looking for Date Time - Found {linkDateString}");
if (!string.IsNullOrWhiteSpace(linkDateString))
{
if (DateTime.TryParse(linkDateString, out var parsedDateTime))
{
toReturn.LinkDate = parsedDateTime;
progress?.Report($"Looking for Date Time - Parsed to {parsedDateTime}");
}
else
{
progress?.Report("Did not parse Date Time");
}
}
progress?.Report("Looking for Site Name");
var siteString = document.QuerySelector("meta[property='og:site_name']")?.Attributes
.FirstOrDefault(x => x.LocalName == "content")?.Value;
if (string.IsNullOrWhiteSpace(siteString))
{
siteString = document.QuerySelector("meta[name='DC.publisher']")?.Attributes
.FirstOrDefault(x => x.LocalName == "content")?.Value;
}
if (string.IsNullOrWhiteSpace(siteString))
{
siteString = document.QuerySelector("meta[name='twitter:site']")?.Attributes
.FirstOrDefault(x => x.LocalName == "value")?.Value.Replace("@", "");
}
if (!string.IsNullOrWhiteSpace(siteString))
{
toReturn.Site = siteString;
}
progress?.Report($"Looking for Site Name - Found {toReturn.Site}");
progress?.Report("Looking for Description");
var descriptionString = document.QuerySelector("meta[name='description']")?.Attributes
.FirstOrDefault(x => x.LocalName == "content")?.Value;
if (string.IsNullOrWhiteSpace(descriptionString))
{
descriptionString = document.QuerySelector("meta[property='og:description']")?.Attributes
.FirstOrDefault(x => x.LocalName == "content")?.Value;
}
if (string.IsNullOrWhiteSpace(descriptionString))
{
descriptionString = document.QuerySelector("meta[name='twitter:description']")?.Attributes
.FirstOrDefault(x => x.LocalName == "content")?.Value;
}
if (!string.IsNullOrWhiteSpace(descriptionString))
{
toReturn.Description = descriptionString;
}
progress?.Report($"Looking for Description - Found {toReturn.Description}");
return(await GenerationReturn.Success($"Parsed URL Metadata for {url} without error"), toReturn);
}
private IEnumerable <ADRawEntry> ReadObjects()
{
ExTraceGlobals.BackSyncTracer.TraceDebug((long)SyncConfiguration.TraceId, "ObjectFullSyncConfiguration.ReadObjects entering");
bool includeLinks = SyncConfiguration.InlcudeLinks(this.PageToken.SyncOptions);
ExTraceGlobals.BackSyncTracer.TraceDebug <bool>((long)SyncConfiguration.TraceId, "ObjectFullSyncConfiguration.ReadObjects includeLinks = {0}", includeLinks);
PropertyDefinition[] properties = FullSyncConfiguration.GetPropertyDefinitions(includeLinks);
int objectCount = Math.Min(this.PageToken.ObjectIds.Count, FullSyncConfiguration.ObjectsPerPageLimit);
ExTraceGlobals.BackSyncTracer.TraceDebug <int>((long)SyncConfiguration.TraceId, "ObjectFullSyncConfiguration.ReadObjects objectCount = {0}", objectCount);
string[] externalObjectIds = new string[objectCount];
IList <SyncObjectId> objectIds = this.PageToken.ObjectIds.ToList <SyncObjectId>();
for (int k = 0; k < objectCount; k++)
{
externalObjectIds[k] = objectIds[k].ObjectId;
ExTraceGlobals.BackSyncTracer.TraceDebug <int, string>((long)SyncConfiguration.TraceId, "ObjectFullSyncConfiguration.ReadObjects externalObjectIds[{0}] = \"{1}\"", k, externalObjectIds[k]);
}
ExTraceGlobals.BackSyncTracer.TraceDebug((long)SyncConfiguration.TraceId, "ObjectFullSyncConfiguration.ReadObjects FindByExternalDirectoryObjectIds ...");
Result <ADRawEntry>[] results = this.FindByExternalDirectoryObjectIds(externalObjectIds, true, properties);
List <ADRawEntry> completeEntries = new List <ADRawEntry>();
List <ADRawEntry> deletedEntries = new List <ADRawEntry>();
List <ADRawEntry> pagingRequiredEntries = new List <ADRawEntry>();
for (int l = 0; l < externalObjectIds.Length; l++)
{
ExTraceGlobals.BackSyncTracer.TraceDebug <int, string>((long)SyncConfiguration.TraceId, "ObjectFullSyncConfiguration.ReadObjects externalObjectIds[{0}] = \"{1}\"", l, externalObjectIds[l]);
if (results[l].Data != null)
{
ADRawEntry data = results[l].Data;
ExTraceGlobals.BackSyncTracer.TraceDebug <int, ADObjectId>((long)SyncConfiguration.TraceId, "ObjectFullSyncConfiguration.ReadObjects found entry[{0}] = \"{1}\"", l, data.Id);
data.propertyBag.SetField(SyncObjectSchema.SyncObjectId, objectIds[l]);
if ((bool)data[SyncObjectSchema.Deleted])
{
ExTraceGlobals.BackSyncTracer.TraceDebug((long)SyncConfiguration.TraceId, "ObjectFullSyncConfiguration.ReadObjects add entry to deletedEntries");
deletedEntries.Add(data);
}
else
{
MultiValuedProperty <LinkMetadata> multiValuedProperty = (MultiValuedProperty <LinkMetadata>)data[FullSyncConfiguration.InitialLinkMetadataRangedProperty];
data.propertyBag.SetField(ADRecipientSchema.LinkMetadata, multiValuedProperty);
if (FullSyncConfiguration.NotAllLinksRetrieved(multiValuedProperty))
{
ExTraceGlobals.BackSyncTracer.TraceDebug((long)SyncConfiguration.TraceId, "ObjectFullSyncConfiguration.ReadObjects add entry to pagingRequiredEntries");
pagingRequiredEntries.Add(data);
}
else
{
ExTraceGlobals.BackSyncTracer.TraceDebug((long)SyncConfiguration.TraceId, "ObjectFullSyncConfiguration.ReadObjects add entry to completeEntries");
completeEntries.Add(data);
}
}
}
else if (results[l].Error == ProviderError.NotFound)
{
ExTraceGlobals.BackSyncTracer.TraceDebug <int>((long)SyncConfiguration.TraceId, "ObjectFullSyncConfiguration.ReadObjects not found entry[{0}]", l);
SyncObjectId syncObjectId = objectIds[l];
base.ExcludedObjectReporter.ReportExcludedObject(syncObjectId, DirectoryObjectErrorCode.ObjectNotFound, ProcessingStage.ObjectFullSyncConfiguration);
this.PageToken.ObjectIds.Remove(syncObjectId);
}
}
ExTraceGlobals.BackSyncTracer.TraceDebug((long)SyncConfiguration.TraceId, "ObjectFullSyncConfiguration.ReadObjects process deletedEntries ...");
int i = 0;
while (i < deletedEntries.Count && this.ShouldReadMoreData())
{
ExTraceGlobals.BackSyncTracer.TraceDebug <ADObjectId>((long)SyncConfiguration.TraceId, "ObjectFullSyncConfiguration.ReadObjects remove {0} from this.PageToken.ObjectIds", deletedEntries[i].Id);
this.PageToken.ObjectIds.Remove((SyncObjectId)deletedEntries[i][SyncObjectSchema.SyncObjectId]);
base.ReturnedObjectCount++;
yield return(deletedEntries[i]);
i++;
}
ExTraceGlobals.BackSyncTracer.TraceDebug((long)SyncConfiguration.TraceId, "ObjectFullSyncConfiguration.ReadObjects process completeEntries ...");
int j = 0;
while (j < completeEntries.Count && this.ShouldReadMoreData())
{
ExTraceGlobals.BackSyncTracer.TraceDebug <ADObjectId>((long)SyncConfiguration.TraceId, "ObjectFullSyncConfiguration.ReadObjects remove {0} from this.PageToken.ObjectIds", completeEntries[j].Id);
this.PageToken.ObjectIds.Remove((SyncObjectId)completeEntries[j][SyncObjectSchema.SyncObjectId]);
base.ReturnedObjectCount++;
base.ReturnedLinkCount += ((MultiValuedProperty <LinkMetadata>)completeEntries[j][FullSyncConfiguration.InitialLinkMetadataRangedProperty]).Count;
yield return(completeEntries[j]);
j++;
}
if (this.ShouldReadMoreData() && pagingRequiredEntries.Count > 0)
{
ExTraceGlobals.BackSyncTracer.TraceDebug((long)SyncConfiguration.TraceId, "ObjectFullSyncConfiguration.ReadObjects read more data ...");
ADRawEntry linkEntry = pagingRequiredEntries[0];
ExTraceGlobals.BackSyncTracer.TraceDebug <ADObjectId>((long)SyncConfiguration.TraceId, "ObjectFullSyncConfiguration.ReadObjects linkEntry {0}", linkEntry.Id);
base.ReturnedObjectCount++;
MultiValuedProperty <LinkMetadata> metadata = (MultiValuedProperty <LinkMetadata>)linkEntry[FullSyncConfiguration.InitialLinkMetadataRangedProperty];
LinkMetadata overlapLink = metadata[0];
base.ReturnedLinkCount += metadata.Count;
int nextPageLinkRangeStart = metadata.ValueRange.UpperBound - 10 + 1;
this.PageToken.ObjectCookie = new FullSyncObjectCookie((SyncObjectId)linkEntry[SyncObjectSchema.SyncObjectId], overlapLink, nextPageLinkRangeStart, (long)linkEntry[ADRecipientSchema.UsnChanged], this.PageToken.ServiceInstanceId);
yield return(linkEntry);
}
yield break;
}
private ADRawEntry ReadNextLinkPage()
{
ExTraceGlobals.BackSyncTracer.TraceDebug((long)SyncConfiguration.TraceId, "ObjectFullSyncConfiguration.ReadNextLinkPage entering");
SyncObjectId objectId = this.PageToken.ObjectCookie.ObjectId;
if (this.PageToken.ObjectCookie.ReadRestartsCount > 10)
{
if (base.EventLogger != null)
{
base.EventLogger.LogTooManyObjectReadRestartsEvent(objectId.ToString(), 10);
}
ExTraceGlobals.BackSyncTracer.TraceDebug <string>((long)SyncConfiguration.TraceId, "ObjectFullSyncConfiguration.ReadNextLinkPage TooManyObjectReadRestarts {0}", objectId.ToString());
base.ExcludedObjectReporter.ReportExcludedObject(objectId, DirectoryObjectErrorCode.Busy, ProcessingStage.ObjectFullSyncConfiguration);
this.PageToken.RemoveObjectFromList(objectId, true);
return(null);
}
int num = Math.Max(0, FullSyncConfiguration.LinksPerPageLimit - base.ReturnedLinkCount);
if (num <= 10)
{
return(null);
}
List <PropertyDefinition> list = new List <PropertyDefinition>();
int linkRangeStart = this.PageToken.ObjectCookie.LinkRangeStart;
ExTraceGlobals.BackSyncTracer.TraceDebug <int>((long)SyncConfiguration.TraceId, "ObjectFullSyncConfiguration.ReadNextLinkPage this.PageToken.ObjectCookie.LinkRangeStart = {0}", this.PageToken.ObjectCookie.LinkRangeStart);
ADPropertyDefinition adpropertyDefinition = RangedPropertyHelper.CreateRangedProperty(ADRecipientSchema.LinkMetadata, new IntRange(linkRangeStart, linkRangeStart + num - 1));
list.Add(adpropertyDefinition);
list.Add(ADRecipientSchema.UsnChanged);
bool flag = !this.PageToken.ObjectCookie.EnumerateLinks;
if (flag)
{
list.AddRange(SyncSchema.Instance.AllBackSyncBaseProperties.Cast <PropertyDefinition>());
list.AddRange(SyncObject.BackSyncProperties.Cast <PropertyDefinition>());
}
else
{
list.AddRange(SyncObject.FullSyncLinkPageBackSyncProperties.Cast <PropertyDefinition>());
}
ExTraceGlobals.BackSyncTracer.TraceDebug <string>((long)SyncConfiguration.TraceId, "ObjectFullSyncConfiguration.ReadNextLinkPage objectId.ObjectId = {0}", objectId.ObjectId);
string[] externalObjectIds = new string[]
{
objectId.ObjectId
};
Result <ADRawEntry>[] array = this.FindByExternalDirectoryObjectIds(externalObjectIds, true, list.ToArray());
Result <ADRawEntry> result = array[0];
if (result.Data != null)
{
ADRawEntry data = result.Data;
ExTraceGlobals.BackSyncTracer.TraceDebug <ADObjectId>((long)SyncConfiguration.TraceId, "ObjectFullSyncConfiguration.ReadNextLinkPage entry = {0}", data.Id);
long num2 = (long)data[ADRecipientSchema.UsnChanged];
ExTraceGlobals.BackSyncTracer.TraceDebug <long>((long)SyncConfiguration.TraceId, "ObjectFullSyncConfiguration.ReadNextLinkPage usnChanged = {0}", num2);
MultiValuedProperty <LinkMetadata> multiValuedProperty = (MultiValuedProperty <LinkMetadata>)data[adpropertyDefinition];
if (!flag)
{
if (num2 != this.PageToken.ObjectCookie.UsnChanged)
{
ExTraceGlobals.BackSyncTracer.TraceDebug <long>((long)SyncConfiguration.TraceId, "ObjectFullSyncConfiguration.ReadNextLinkPage RestartObjectReadAfterObjectChange usnChanged = {0}", num2);
this.PageToken.ObjectCookie.RestartObjectReadAfterObjectChange(num2);
return(null);
}
if (this.PageToken.ObjectCookie.LinkRangeStart > 0 && !this.CheckLinkOverlap(multiValuedProperty))
{
ExTraceGlobals.BackSyncTracer.TraceDebug((long)SyncConfiguration.TraceId, "ObjectFullSyncConfiguration.ReadNextLinkPage RestartObjectReadAfterTargetLinksChange");
this.PageToken.ObjectCookie.RestartObjectReadAfterTargetLinksChange();
return(null);
}
}
result.Data.propertyBag.SetField(ADRecipientSchema.LinkMetadata, multiValuedProperty);
LinkMetadata overlapLink = multiValuedProperty[0];
base.ReturnedLinkCount += multiValuedProperty.Count;
if (multiValuedProperty.ValueRange != null)
{
if (multiValuedProperty.ValueRange.UpperBound == 2147483647)
{
this.PageToken.RemoveObjectFromList(objectId, true);
ExTraceGlobals.BackSyncTracer.TraceDebug((long)SyncConfiguration.TraceId, "ObjectFullSyncConfiguration.ReadNextLinkPage last link page read");
}
else
{
int num3 = multiValuedProperty.ValueRange.UpperBound - 10 + 1;
ExTraceGlobals.BackSyncTracer.TraceDebug <int>((long)SyncConfiguration.TraceId, "ObjectFullSyncConfiguration.ReadNextLinkPage set this.PageToken.nextLinkPageRangeStart to {0}", num3);
this.PageToken.ObjectCookie.SetNextPageData(objectId, overlapLink, num3, (long)result.Data[ADRecipientSchema.UsnChanged]);
}
}
else
{
this.PageToken.RemoveObjectFromList(objectId, true);
ExTraceGlobals.BackSyncTracer.TraceDebug <string>((long)SyncConfiguration.TraceId, "ObjectFullSyncConfiguration.ReadNextLinkPage this.PageToken.RemoveObjectFromList objectId = {0}", objectId.ObjectId);
}
return(result.Data);
}
ExTraceGlobals.BackSyncTracer.TraceDebug((long)SyncConfiguration.TraceId, "ObjectFullSyncConfiguration.ReadNextLinkPage FindByExternalDirectoryObjectIds returned no data.");
this.PageToken.ObjectCookie.RestartObjectRead(false);
return(null);
}
internal FullSyncObjectCookie(SyncObjectId objectId, LinkMetadata overlapLink, int nextRangeStart, long usnChanged, ServiceInstanceId serviceInstanceid) : this()
{
this.ReadRestartsCount = 0;
this.SetNextPageData(objectId, overlapLink, nextRangeStart, usnChanged);
this.ServiceInstanceId = serviceInstanceid;
}
