//public GetDocumentsResponse GetDocuments()
//{
// GetDocumentsResponse response = new GetDocumentsResponse();
// try
// {
// IEnumerable<DocumentView> documents = _documentRepository.FindAll()
// .ConvertToDocumentViews();
// response.DocumentViews = documents;
// }
// catch (Exception ex)
// {
// }
// return response;
//}
public GetDocumentsResponse GetDocumentsBy(Guid customerID, IList <Sort> sort)
{
GetDocumentsResponse response = new GetDocumentsResponse();
try
{
Infrastructure.Querying.Query query = new Infrastructure.Querying.Query();
Criterion criterion = new Criterion("Customer.ID", customerID, CriteriaOperator.Equal);
query.Add(criterion);
Response <Document> documents = _documentRepository.FindBy(query, -1, -1, sort);
IEnumerable <DocumentView> documentViews = documents.data.ConvertToDocumentViews();
// اصلاح آدرس عکسها و پاس دادن آدرس جدید
foreach (var document in documentViews)
{
document.Photo = document.Photo.Replace(@"\", "/").Substring(document.Photo.IndexOf("data"));;
}
response.DocumentViews = documentViews;
}
catch (Exception ex)
{
}
return(response);
}
public ICollectionReader GetDocuments(ICollection<string> documentKeys)
{
if (documentKeys == null)
throw new ArgumentNullException("documentKeys");
if (documentKeys.Count < 1)
throw new ArgumentException("No DocumentKey specified");
List<JSONDocument> documents = new List<JSONDocument>();
foreach (string documentKey in documentKeys)
{
if (documentKey != null)
{
JSONDocument jdoc = new JSONDocument();
jdoc.Key = documentKey;
documents.Add(jdoc);
}
}
GetDocumentsOperation getDocumentsOperation = new GetDocumentsOperation();
getDocumentsOperation.DocumentIds = documents.Cast<IJSONDocument>().ToList();
getDocumentsOperation.Database = _database.DatabaseName;
getDocumentsOperation.Collection = _collectionName;
GetDocumentsResponse getDocumentsResponse = (GetDocumentsResponse)_database.ExecutionMapper.GetDocuments(getDocumentsOperation);
if (getDocumentsResponse.IsSuccessfull)
{
CollectionReader reader = new CollectionReader((DataChunk)getDocumentsResponse.DataChunk, _database.ExecutionMapper, _database.DatabaseName, _collectionName);
return reader;
}
else
throw new Exception("Operation failed Error: " + Common.ErrorHandling.ErrorMessages.GetErrorMessage(getDocumentsResponse.ErrorCode, getDocumentsResponse.ErrorParams));
}
public ActionResult <GetDocumentsResponse> GetDocuments()
{
var response = new GetDocumentsResponse();
var documents = documentService.GetDocuments();
response.Documents = documents.Select(x => new Document
{
Location = x.Location,
Name = x.Name,
FileSize = x.FileSize,
documentId = x.Id,
SortOrder = x.SortOrder
}).ToList();
return(response);
}
public JsonResult Document_Read(Guid customerID, string sort)
{
GetDocumentsResponse response = new GetDocumentsResponse();
#region Access Check
bool hasPermission = GetEmployee().IsGuaranteed("Document_Read");
if (!hasPermission)
{
ModelState.AddModelError("", "AccessDenied");
return(Json(response.DocumentViews, JsonRequestBehavior.AllowGet));
}
#endregion
response = _documentService.GetDocumentsBy(customerID, ConvertJsonToObject(sort));
return(Json(response.DocumentViews, JsonRequestBehavior.AllowGet));
}
public static void TestDotnerApiBloomFilters(string clientName, string accessKey)
{
double falsePositiveRate = 0.00001;
string baseUri = "https://op-dhs-prod-read-nus.azurewebsites.net/";// "https://op-dhs-sandbox-read.azurewebsites.net/"; //
IDocumentHostingService dhsClient = new DocumentHostingServiceClient(new Uri(baseUri), clientName, accessKey);
IList <BloomFilter> result = new List <BloomFilter>();
IList <string> assetIds = new List <string>();
Dictionary <string, List <string> > conflicts = new Dictionary <string, List <string> >(StringComparer.OrdinalIgnoreCase);
IList <GetDepotResponse> depots = dhsClient.GetAllDepotsBySiteBasePath("docs", "docs.microsoft.com/dotnet/", null, CancellationToken.None).Result;
foreach (GetDepotResponse depot in depots)
{
if (depot.SystemMetadata.GetValueOrDefault <bool>(MetadataConstants.ActiveKey))
{
string depotName = depot.DepotName;
string continueAt = null;
Console.WriteLine($"{depotName} Start.");
IEnumerable <GetDocumentResponse> allDocuments = new List <GetDocumentResponse>();
int i = 0;
do
{
for (int retry = 0; retry < 3; i++)
{
try
{
GetDocumentsResponse documents = dhsClient.GetDocumentsPaginated(depot.DepotName, "en-us", "live", true, continueAt, null, null, CancellationToken.None).Result;
continueAt = documents.ContinueAt;
allDocuments = allDocuments.Concat(documents.Documents);
break;
}
catch (Exception e)
{
Console.WriteLine(e);
Console.WriteLine($"Retry for {retry + 1} times");
}
}
i++;
Console.WriteLine($"{i:000} ..................");
}while (!string.IsNullOrEmpty(continueAt));
Console.WriteLine($"{depotName} Size: {allDocuments.Count()}.");
var bloomFilter = new BloomFilter(allDocuments.Count(), falsePositiveRate);
foreach (var document in allDocuments)
{
bloomFilter.Add(document.AssetId);
assetIds.Add(document.AssetId);
}
Console.WriteLine($"{depotName} Bloom Filter Size: {bloomFilter.BitLength / 1024 / 8} KB.");
result.Add(bloomFilter);
Console.WriteLine($"{depotName} Done.");
}
}
using (StreamWriter file = new StreamWriter(@"output.json", true))
{
file.WriteLine(JsonConvert.SerializeObject(result));
}
int onlyOneCount = 0;
var conflictHashDict = new Dictionary <int, int>();
foreach (var assetId in assetIds)
{
var conflictCount = result.Where(r => r.Contains(assetId)).Count();
if (conflictCount == 1)
{
onlyOneCount++;
}
else
{
if (conflictHashDict.ContainsKey(conflictCount))
{
conflictHashDict[conflictCount] += 1;
}
else
{
conflictHashDict[conflictCount] = 1;
}
}
}
Console.WriteLine($"Only one count: {onlyOneCount}, total count: {assetIds.Count()}");
Console.WriteLine($"Duplicate count: {conflictHashDict.Keys.Count}");
var output = conflictHashDict.OrderBy(p => p.Value).Reverse();
using (StreamWriter file = new StreamWriter(@"conflictCount.txt", true))
{
foreach (var item in output)
{
file.WriteLine($"{item.Key}: {item.Value}");
}
}
Console.ReadLine();
}
public static void GenerateDotnetApiBloomFilter(string clientName, string accessKey)
{
double falsePositiveRate = 0.00001;
string baseUri = "https://op-dhs-prod-read-nus.azurewebsites.net/";// "https://op-dhs-sandbox-read.azurewebsites.net/"; //
IDocumentHostingService dhsClient = new DocumentHostingServiceClient(new Uri(baseUri), clientName, accessKey);
IList <DepotBloomFilter> result = new List <DepotBloomFilter>();
Dictionary <string, List <string> > conflicts = new Dictionary <string, List <string> >(StringComparer.OrdinalIgnoreCase);
IList <GetDepotResponse> depots = dhsClient.GetAllDepotsBySiteBasePath("docs", "docs.microsoft.com/dotnet/", null, CancellationToken.None).Result;
foreach (GetDepotResponse depot in depots.Skip(20).Take(1))
{
if (depot.SystemMetadata.GetValueOrDefault <bool>(MetadataConstants.ActiveKey))
{
string depotName = depot.DepotName;
string continueAt = null;
Console.WriteLine($"{depotName} Start.");
IEnumerable <GetDocumentResponse> allDocuments = new List <GetDocumentResponse>();
int i = 0;
do
{
for (int retry = 0; retry < 3; i++)
{
try
{
GetDocumentsResponse documents = dhsClient.GetDocumentsPaginated(depot.DepotName, "en-us", "live", true, continueAt, null, null, CancellationToken.None).Result;
continueAt = documents.ContinueAt;
allDocuments = allDocuments.Concat(documents.Documents);
break;
}
catch (Exception e)
{
Console.WriteLine(e);
Console.WriteLine($"Retry for {retry + 1} times");
}
}
i++;
Console.WriteLine($"{i:000} ..................");
}while (!string.IsNullOrEmpty(continueAt));
Console.WriteLine($"{depotName} Size: {allDocuments.Count()}.");
var bloomFilter = new BloomFilter(allDocuments.Count(), falsePositiveRate);
foreach (var document in allDocuments)
{
bloomFilter.Add(document.AssetId);
}
Console.WriteLine($"{depotName} Bloom Filter Size: {bloomFilter.BitLength / 1024 / 8} KB.");
result.Add(new DepotBloomFilter
{
DepotName = depotName,
BloomFilter = bloomFilter.BitArray,
Count = allDocuments.Count(),
FalsePositiveRate = falsePositiveRate
});
Console.WriteLine($"{depotName} Done.");
}
}
/*
* using (StreamWriter file = new StreamWriter(@"output.json", true))
* {
* file.WriteLine(JsonConvert.SerializeObject(result));
* }
*/
MemoryStream ms = new MemoryStream();
using (BsonWriter writer = new BsonWriter(ms))
{
JsonSerializer serializer = new JsonSerializer();
serializer.Serialize(writer, result);
}
Console.ReadLine();
}
private static void GetMergedRepo(IDocumentHostingService sandboxClient, IDocumentHostingService prodClient)
{
string[] depotNames =
{
//"MSDN.azuredotnet",
//"MSDN.aspnetAPIDocs",
//"MSDN.coredocs-demo",
// Prod
"MSDN.aspnetAPIDocs",
"VS.core-docs",
};
CancellationToken cancellationToken = new CancellationToken();
foreach (var depotName in depotNames)
{
Console.WriteLine(depotName);
string continueAt = null;
int count = 0;
while (true)
{
GetDocumentsResponse documents = prodClient.GetDocumentsPaginated(depotName, "en-us", "master", false, continueAt, null, 100, cancellationToken).Result;
continueAt = documents.ContinueAt;
count += documents.Documents.Count;
Console.WriteLine($"{count}, {DateTime.Now:HH:mm:ss tt zz}");
foreach (var document in documents.Documents)
{
var key = document.AssetId;
if (key.StartsWith(HardCodedPrefix))
{
key = key.Substring(HardCodedPrefix.Length);
}
document.AssetId = key;
if (!DocumentDict.ContainsKey(key))
{
DocumentDict.Add(key, new List <GetDocumentResponse>());
}
DocumentDict[key].Add(document);
}
var putDocumentsRequest = new PutDocumentsRequest();
putDocumentsRequest.Documents.AddRange(documents.Documents.Select(d => new PutDocumentsRequestItem
{
AssetId = d.AssetId,
ProductVersion = d.ProductVersion,
ContentSourceUri = GetCorrectedCanonicalUrl(d.ContentUri, d.AssetId),
//ContentSourceUri = d.ContentUri,
Locale = d.Locale,
Metadata = d.Metadata
}));
sandboxClient.PutDocuments(newDepotName, "master", putDocumentsRequest, null, cancellationToken).Wait();
if (string.IsNullOrEmpty(continueAt))
{
break;
}
}
}
Console.WriteLine("Merge comeplete. Press any key to continue...");
}
