public static async Task UploadDocuments(List <Document> documents, string activeEtag, Action <int, int> progress)
{
if (documents == null || !documents.Any())
{
return;
}
// upload documents
await ParallelUtility.ParallelForEach(documents, document =>
{
return(CosmosDBAccessor <Document> .UpsertAsync(document));
}, 400, 200, progress);
// switch active etag
var doc = documents.First();
var active = new Active
{
ActiveEtag = activeEtag,
IsActive = false,
Branch = doc.Branch,
Locale = doc.Locale,
Docset = doc.Docset,
id = HashUtility.GetSha1HashString($"{doc.Docset}|{doc.Branch}|{doc.Locale}")
};
await CosmosDBAccessor <Active> .UpsertAsync(active);
}
public static async Task <(string pageUrl, string pageHash)> UploadPage(Stream pageStream, bool isDynamicRender, string contentType)
{
string pageUrl;
string hash = HashUtility.GetSha1HashString(pageStream);
pageStream.Position = 0;
if (isDynamicRender)
{
Page page = await CosmosDBAccessor <Page> .GetAsync(hash);
if (page is null)
{
using (var sr = new StreamReader(pageStream))
{
page = new Page()
{
id = hash,
Hash = hash,
Content = sr.ReadToEnd(),
};
}
await CosmosDBAccessor <Page> .UpsertAsync(page);
}
pageUrl = Config.Get("cosmos_endpoint") + CosmosDBAccessor <Page> .GetDocumentUri(hash).ToString();
}
else
{
var blob = BlobAccessor.cloudBlobContainer.GetBlockBlobReference(hash);
bool blobExist = await blob.ExistsAsync();
if (!blobExist)
{
blob.Properties.ContentType = contentType;
await blob.UploadFromStreamAsync(pageStream);
}
pageUrl = blob.Uri.AbsoluteUri;
}
return(pageUrl, hash);
}
public static async Task <Document> QueryDocument(string url, string branch, string locale, string moniker)
{
if (string.IsNullOrWhiteSpace(url))
{
return(null);
}
// pr branch could fall back
HashSet <string> branches = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
if (string.IsNullOrWhiteSpace(branch))
{
branch = DefaultBranch;
}
branches.Add(branch);
if (branch.StartsWith("pr-", StringComparison.OrdinalIgnoreCase))
{
branches.Add(DefaultFallbackBranch);
}
// non en-us locale could fall back
HashSet <string> locales = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
if (string.IsNullOrWhiteSpace(locale))
{
locale = DefaultLocale;
}
locales.Add(locale);
if (LocaleFallbackNonEnUsRules.ContainsKey(locale))
{
locales.Add(LocaleFallbackNonEnUsRules[locale]);
}
//normalize moniker
if (string.IsNullOrWhiteSpace(moniker))
{
moniker = null;
}
// get all potential documents
var docs = await CosmosDBAccessor <Document> .QueryAsync(
doc =>
doc.Url == url.ToLowerInvariant() &&
branches.Contains(doc.Branch) &&
locales.Contains(doc.Locale));
if (!docs.Any())
{
return(null);
}
// get all actives
HashSet <string> activeEtags = new HashSet <string>(docs.Select(doc => doc.ActiveEtag));
var actives = await CosmosDBAccessor <Active> .QueryAsync(active => activeEtags.Contains(active.ActiveEtag));
activeEtags = new HashSet <string>(actives.Select(active => active.ActiveEtag));
if (!activeEtags.Any())
{
return(null);
}
// best match
Document result = null;
int bestMatch = -1;
string finalMoniker = null;
foreach (Document doc in docs.Where(d => activeEtags.Contains(d.ActiveEtag)))
{
// branch fallback
int score = string.Equals(branch, doc.Branch, StringComparison.OrdinalIgnoreCase) ? 200000 : 100000;
// moniker fallback
string matchedMoniker = null;
if (doc.Monikers != null && doc.Monikers.Contains(moniker, StringComparer.OrdinalIgnoreCase))
{
score += 2000;
matchedMoniker = moniker;
}
else
{
score += 1000;
matchedMoniker = doc.Monikers?.FirstOrDefault();
}
// locale fallback
if (string.Equals(locale, doc.Locale, StringComparison.OrdinalIgnoreCase))
{
score += 40;
}
else if (LocaleFallbackNonEnUsRules.TryGetValue(locale, out string fallbackLocale) && string.Equals(fallbackLocale, doc.Locale, StringComparison.OrdinalIgnoreCase))
{
score += 20;
}
else
{
score += 10;
}
if (score > bestMatch)
{
result = doc;
bestMatch = score;
finalMoniker = matchedMoniker;
}
}
result.Monikers = new List <string> {
finalMoniker
};
return(result);
}
public static async Task <string> GetPageContent(string uri)
{
var page = await CosmosDBAccessor <Page> .GetByUriAsync(uri);
return(page?.Content);
}