/// <summary>
/// 根据枚举代码获取枚举列表
/// </summary>
/// <param name="assemblyName">程序集名称</param>
/// <param name="namespace">命名空间名称</param>
/// <param name="enumClassName">枚举名称</param>
/// <returns></returns>
public static Array GetEnumListByCode(string assemblyName, string @namespace, string enumClassName)
{
if (!(AssemblyList?.Select(o => o.Key)).Contains(assemblyName))
{
AssemblyList.TryAdd(assemblyName, Assembly.Load(assemblyName));
}
List <Type> enumlist = new List <Type>();
foreach (var assembly in AssemblyList)
{
var enumInfo = assembly.Value.CreateInstance($"{@namespace}.{enumClassName}", false);
if (enumInfo != null)
{
enumlist.Add(enumInfo.GetType());
}
}
if (enumlist.Count == 0)
{
return(null);
}
if (enumlist.Count > 1)
{
throw new EnumException($"枚举【{enumClassName}】存在多个，请检查命名空间【{ @namespace}】");
}
return(Enum.GetValues(enumlist.FirstOrDefault()));
}
/// <summary>
/// 根据枚举代码获取枚举列表
/// </summary>
/// <param name="assemblys">程序集名称集合</param>
/// <param name="namespaces">命名空间名称集合</param>
/// <param name="enumCode">枚举名称</param>
/// <returns></returns>
public static List <EnumDto> GetEnumListByCode(IEnumerable <string> assemblys, IEnumerable <string> namespaces, string enumCode)
{
assemblys = assemblys.Distinct();
namespaces = namespaces.Distinct();
var Inexistentassembly = assemblys.Except(AssemblyList?.Select(o => o.Key));
foreach (var assembly in Inexistentassembly)
{
//AppAssembly.Assemblies.Where(o=>o.FullName == assembly);
AssemblyList.TryAdd(assembly, System.Reflection.Assembly.Load(assembly));
}
List <Type> enumlist = new List <Type>();
List <string> enumNamespanList = new List <string>();
foreach (var enumNamespace in namespaces)
{
foreach (var assembly in AssemblyList)
{
var enumInfo = assembly.Value.CreateInstance($"{enumNamespace}.{enumCode}", false);
if (enumInfo != null)
{
enumNamespanList.Add(enumNamespace);
enumlist.Add(enumInfo.GetType());
}
}
}
if (enumlist.Count == 0)
{
return(default);
public static void PostCreatedFiles(ConcurrentDictionary<string, LocalFile> files, string uploadUrl)
{
var localFiles = files.ToList();
// Create post content
var content = new MultipartFormDataContent();
// Check we have content to send
if (!LoadFileToDataContent(content, files.Select(x => x.Value).ToList()))
{
Console.WriteLine("Failed to load file or no content as loaded. Cancel upload");
return;
}
SendPostData(content, uploadUrl);
if (BaseConfig.Synchronize)
{
localFiles.ForEach(x => x.Value.Synchronized = true);
}
else
{
// Remove files from queue
foreach (var file in localFiles)
{
LocalFile deleted;
if (!files.TryRemove(file.Key, out deleted))
{
Console.WriteLine("Failed to remove");
}
}
}
}
public async Task<IActionResult> MissingRepos()
{
var gitHubName = HttpContext.User.Identity.Name;
var gitHubAccessToken = await HttpContext.Authentication.GetTokenAsync("access_token");
var gitHubClient = GitHubUtils.GetGitHubClient(gitHubAccessToken);
var repoDataSet = await RepoSetProvider.GetRepoDataSet();
var repoSetLists = repoDataSet.GetRepoSetLists();
var distinctOrgs =
repoSetLists
.SelectMany(
repoSet => repoSet.Value.Repos.Select(repoDefinition => repoDefinition.Owner))
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(org => org).ToList();
var allOrgRepos = new ConcurrentDictionary<string, string[]>(StringComparer.OrdinalIgnoreCase);
var result = AsyncParallelUtils.ForEachAsync(distinctOrgs, 5, async org =>
{
var reposInOrg = await gitHubClient.Repository.GetAllForOrg(org);
allOrgRepos[org] = reposInOrg.Where(repo => !repo.Fork).Select(repo => repo.Name).ToArray();
});
await result;
var missingOrgRepos = allOrgRepos.Select(org =>
new MissingRepoSet
{
Org = org.Key,
MissingRepos =
org.Value
.Except(
repoSetLists
.SelectMany(repoSetList => repoSetList.Value.Repos)
.Select(repoDefinition => repoDefinition.Name), StringComparer.OrdinalIgnoreCase)
.OrderBy(repo => repo, StringComparer.OrdinalIgnoreCase)
.ToList(),
})
.OrderBy(missingRepoSet => missingRepoSet.Org, StringComparer.OrdinalIgnoreCase)
.ToList();
return View(new MissingReposViewModel
{
GitHubUserName = gitHubName,
RepoSetNames = repoDataSet.GetRepoSetLists().Select(repoSetList => repoSetList.Key).ToArray(),
MissingRepos = missingOrgRepos,
});
}
/// <summary>
/// This method is used to update any calculated fields for the specific service statistics.
/// </summary>
/// <param name="statistics">The current statistics.</param>
protected override void StatisticsRecalculate(SerializationStatistics statistics)
{
base.StatisticsRecalculate(statistics);
try
{
statistics.ItemCount = mPayloadSerializers?.Count ?? 0;
statistics.CacheCount = mLookUpCache?.Count ?? 0;
statistics.Serialization = mPayloadSerializers?.Select((c) => $"{BitConverter.ToString(c.Key)}: {c.Value.GetType().Name}").ToArray();
statistics.Cache = mLookUpCache?.Select((c) => $"{c.Key.Name}: {c.Value.GetType().Name}").ToArray();
}
catch (Exception)
{
}
}
private HttpResponseMessage ReplicateAllIndexes(Func<ReplicationDestination, bool> additionalDestinationPredicate = null)
{
//check for replication document before doing work on getting index definitions.
//if there is no replication set up --> no point in doing any other work
HttpResponseMessage erroResponseMessage;
var replicationDocument = GetReplicationDocument(out erroResponseMessage);
if (replicationDocument == null)
return erroResponseMessage;
var indexDefinitions = Database.IndexDefinitionStorage
.IndexDefinitions
.Select(x => x.Value)
.ToList();
var httpRavenRequestFactory = new HttpRavenRequestFactory { RequestTimeoutInMs = Database.Configuration.Replication.ReplicationRequestTimeoutInMilliseconds };
var enabledReplicationDestinations = replicationDocument.Destinations
.Where(dest => dest.Disabled == false && dest.SkipIndexReplication == false)
.ToList();
if (additionalDestinationPredicate != null)
enabledReplicationDestinations = enabledReplicationDestinations.Where(additionalDestinationPredicate).ToList();
if (enabledReplicationDestinations.Count == 0)
return GetMessageWithObject(new { Message = "Replication is configured, but no enabled destinations found." }, HttpStatusCode.NotFound);
var replicationRequestTasks = new List<Task>(enabledReplicationDestinations.Count * indexDefinitions.Count);
var failedDestinations = new ConcurrentDictionary<string, Exception>();
foreach (var definition in indexDefinitions)
{
replicationRequestTasks.AddRange(
enabledReplicationDestinations
.Select(destination =>
Task.Run(() =>
{
try
{
ReplicateIndex(definition.Name, destination,
RavenJObject.FromObject(definition),
httpRavenRequestFactory);
}
catch (Exception e)
{
failedDestinations.TryAdd(destination.Humane ?? "<null?>", e);
log.WarnException("Could not replicate " + definition.Name + " to " + destination.Humane, e);
}
})));
}
Task.WaitAll(replicationRequestTasks.ToArray());
return GetMessageWithObject(new
{
IndexesCount = indexDefinitions.Count,
EnabledDestinationsCount = enabledReplicationDestinations.Count,
SuccessfulReplicationCount = ((enabledReplicationDestinations.Count * indexDefinitions.Count) - failedDestinations.Count),
FailedDestinationUrls = failedDestinations.Select(x=>new{ Server = x.Key, Error = x.Value.ToString()}).ToArray()
});
}
public IEnumerator <T> GetEnumerator()
{
return(_Behaviors.Select(k => k.Key).GetEnumerator());
}
public static CodeCompileUnit[] LoadSvcData(string[] serviceEndpoints, string globalNamespaceName)
{
var concurrentDic = new ConcurrentDictionary <string, CodeCompileUnit>();
//var logWriter = new IndentedTextWriter(new StreamWriter(@"c:\\log2.txt"));
foreach (var serviceEndpoint in serviceEndpoints)
{
Console.WriteLine($"{serviceEndpoint} start");
MetadataSet metadataSet = null;
Task <MetadataSet> mexClientData = null;
MetadataExchangeClient mexClient = null;
var serviceUri =
new Uri(serviceEndpoint);
var isHttps = 0 == serviceUri.ToString().IndexOf("https://", StringComparison.Ordinal);
var basicHttpBinding = new BasicHttpBinding
{
MaxReceivedMessageSize = int.MaxValue,
MaxBufferSize = int.MaxValue,
OpenTimeout = new TimeSpan(0, 0, 10, 0),
SendTimeout = new TimeSpan(0, 0, 10, 0),
ReceiveTimeout = new TimeSpan(0, 0, 10, 0),
CloseTimeout = new TimeSpan(0, 0, 10, 0),
AllowCookies = false,
ReaderQuotas = new XmlDictionaryReaderQuotas
{
MaxNameTableCharCount = int.MaxValue,
MaxStringContentLength = int.MaxValue,
MaxArrayLength = 32768,
MaxBytesPerRead = 4096,
MaxDepth = 32
},
Security =
{
Mode = isHttps ? BasicHttpSecurityMode.Transport : BasicHttpSecurityMode.None,
Transport =
{
ClientCredentialType =
isHttps ? HttpClientCredentialType.Certificate : HttpClientCredentialType.Basic
}
}
};
mexClient =
new MetadataExchangeClient(basicHttpBinding)
{
MaximumResolvedReferences = 1000,
HttpCredentials = new NetworkCredential(GlobalConfig.SitLogin, GlobalConfig.SitPassword),
ResolveMetadataReferences = true
};
mexClientData = mexClient.GetMetadataAsync(serviceUri, MetadataExchangeClientMode.HttpGet);
do
{
try
{
mexClientData.Wait();
metadataSet = mexClientData.Result;
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
//Console.WriteLine(serviceUri.ToString());
}
//System.Threading.Thread.Sleep(1000);
} while (mexClientData == null || metadataSet == null);
object dataContractImporter;
XsdDataContractImporter xsdDcImporter;
var options = new ImportOptions();
var wsdl = new WsdlImporter(metadataSet);
if (!wsdl.State.TryGetValue(typeof(XsdDataContractImporter), out dataContractImporter))
{
xsdDcImporter = new XsdDataContractImporter {
Options = options
};
wsdl.State.Add(typeof(XsdDataContractImporter), xsdDcImporter);
}
else
{
xsdDcImporter = (XsdDataContractImporter)dataContractImporter;
if (xsdDcImporter.Options == null)
{
xsdDcImporter.Options = options;
}
}
//IEnumerable<IWsdlImportExtension> exts = wsdl.WsdlImportExtensions;
var newExts = new List <IWsdlImportExtension>();
newExts.AddRange(wsdl.WsdlImportExtensions);
newExts.Add(new WsdlDocumentationImporter());
IEnumerable <IPolicyImportExtension> polExts = wsdl.PolicyImportExtensions;
wsdl = new WsdlImporter(metadataSet, polExts, newExts);
var contracts = wsdl.ImportAllContracts();
wsdl.ImportAllEndpoints();
wsdl.ImportAllBindings();
var generator = new ServiceContractGenerator();
foreach (var contract in contracts)
{
generator.GenerateServiceContractType(contract);
var nsname = GetHumanString(contract.Namespace);
generator.TargetCompileUnit.UserData.Add("ModuleName", nsname);
generator.TargetCompileUnit.UserData.Add("NamespaceName", globalNamespaceName.TrimEnd('.') + '.');
concurrentDic.TryAdd(nsname, generator.TargetCompileUnit);
//logWriter.WriteLine(nsname);
}
Console.WriteLine($"{serviceEndpoint} end");
}
//logWriter.Flush(); logWriter.Close();
return(concurrentDic.Select(t => t.Value).ToArray());
}
public IActionResult Index(string repoSet)
{
var gitHubAccessToken = Context.Session.GetString("GitHubAccessToken");
var gitHubName = Context.Session.GetString("GitHubName");
// If session state didn't have our data, either there's no one logged in, or they just logged in
// but the claims haven't yet been read.
if (string.IsNullOrEmpty(gitHubAccessToken))
{
if (!User.Identity.IsAuthenticated)
{
// Not authenticated at all? Go to GitHub to authorize the app
return new ChallengeResult("GitHub", new AuthenticationProperties { RedirectUri = "/" + repoSet });
}
// Authenticated but haven't read the claims? Process the claims
gitHubAccessToken = Context.User.FindFirst("access_token")?.Value;
gitHubName = Context.User.Identity.Name;
Context.Session.SetString("GitHubAccessToken", gitHubAccessToken);
Context.Session.SetString("GitHubName", gitHubName);
}
// Authenticated and all claims have been read
var repos =
RepoSets.HasRepoSet(repoSet ?? string.Empty)
? RepoSets.GetRepoSet(repoSet)
: RepoSets.GetAllRepos();
var allIssuesByRepo = new ConcurrentDictionary<string, Task<IReadOnlyList<Issue>>>();
var allPullRequestsByRepo = new ConcurrentDictionary<string, Task<IReadOnlyList<PullRequest>>>();
Parallel.ForEach(repos, repo => allIssuesByRepo[repo] = GetIssuesForRepo(repo, GetGitHubClient(gitHubAccessToken)));
Parallel.ForEach(repos, repo => allPullRequestsByRepo[repo] = GetPullRequestsForRepo(repo, GetGitHubClient(gitHubAccessToken)));
Task.WaitAll(allIssuesByRepo.Select(x => x.Value).ToArray());
Task.WaitAll(allPullRequestsByRepo.Select(x => x.Value).ToArray());
var allIssues = allIssuesByRepo.SelectMany(
issueList => issueList.Value.Result.Select(
issue => new IssueWithRepo
{
Issue = issue,
RepoName = issueList.Key,
})).ToList();
var allPullRequests = allPullRequestsByRepo.SelectMany(
pullRequestList => pullRequestList.Value.Result.Select(
pullRequest => new PullRequestWithRepo
{
PullRequest = pullRequest,
RepoName = pullRequestList.Key,
}))
.OrderBy(pullRequestWithRepo => pullRequestWithRepo.PullRequest.CreatedAt)
.ToList();
return View(new HomeViewModel
{
TotalIssues = allIssues.Count,
Name = gitHubName,
GroupByAssignee = new GroupByAssigneeViewModel
{
Assignees =
allIssues
.GroupBy(issue => issue.Issue.Assignee?.Login)
.Select(group =>
new GroupByAssigneeAssignee
{
Assignee = group.Key,
Issues = group.ToList().AsReadOnly(),
})
.OrderBy(group => group.Assignee, StringComparer.OrdinalIgnoreCase)
.ToList()
.AsReadOnly()
},
GroupByMilestone = new GroupByMilestoneViewModel
{
Milestones =
allIssues
.GroupBy(issue => issue.Issue.Milestone?.Title)
.Select(group =>
new GroupByMilestoneMilestone
{
Milestone = group.Key,
Issues = group.ToList().AsReadOnly(),
})
.OrderBy(group => group.Milestone, StringComparer.OrdinalIgnoreCase)
.ToList()
.AsReadOnly()
},
GroupByRepo = new GroupByRepoViewModel
{
Repos =
allIssues
.GroupBy(issue => issue.RepoName)
.Select(group =>
new GroupByRepoRepo
{
RepoName = group.Key,
Issues = group.ToList().AsReadOnly(),
})
.OrderByDescending(group => group.Issues.Count)
.ToList()
.AsReadOnly()
},
PullRequests = allPullRequests,
});
}
private void WriteCheckpointInfo(AbsolutePath path, ConcurrentDictionary <string, string> newCheckpointInfo)
{
// Format is newline (IncrementalCheckpointInfoEntrySeparator) separated entries with {Key}={Value}
File.WriteAllText(path.Path, string.Join(IncrementalCheckpointInfoEntrySeparator, newCheckpointInfo.Select(s => $"{s.Key}={s.Value}")));
}
/// <summary>
/// Get remote all
/// </summary>
/// <returns></returns>
public static List <GameSession> GetRemoteAll()
{
return(_remoteHash.Select(pair => Get(pair.Value)).ToList());
}
public string GetStatus(bool isInternal)
{
var reducers = HostEnvironment.Instance.GetAvailablePeerConnections(10000);
ConcurrentDictionary<string, string> results = new ConcurrentDictionary<string, string>();
List<Task> statusTasks = new List<Task>();
foreach (var reducer in reducers)
{
Task t = new Task((r) =>
{
var rie = r as PeerInfo;
string id = rie.Id.Split('_').LastOrDefault();
try
{
if (rie.Equals(Endpoint))
{
var memStatus = SystemInfo.GetMemoryStatusEx();
var usedMem = Math.Round((double)(memStatus.ullTotalPhys - memStatus.ullAvailPhys) / 1073741824d, 2); //Math.Round( (double)GC.GetTotalMemory(false) / 1073741824d, 4);
var availMem = Math.Round((double)memStatus.ullAvailPhys / 1073741824d, 2);
string sub = string.Format(" Memory Usage: Used={0}GB, Available={1}GB", usedMem, availMem);
long totalMentions = 0;
//foreach (var sql in SqlInterface.StoredSqlInterfaces)
foreach (var sql in ComputeNode.Node.Catalogs.Values)
{
var count = sql.GetCount(null);
totalMentions += count;
sub += "\r\n Subdomain:" + sql.Name + " Mentions:" + count;
}
sub += "\r\n Total Mentions:" + totalMentions;
results[id] = sub;
}
else
{
if (!isInternal)
{
using (var client = HostEnvironment.GetServiceClient(rie))
{
results[id] = client.Ping("status");
}
}
}
}
catch (Exception ex)
{
results[id] = "[Failed Status On Node] " + ex.Message;
}
},
reducer,
TaskCreationOptions.LongRunning);
statusTasks.Add(t);
t.Start();
}
Task.WaitAll(statusTasks.ToArray());
if (isInternal) return string.Join("\r\n", results.Values);
else return string.Join("\r\n", results.Select(x => "[@" + x.Key + "] ==> " + x.Value));
}
private static bool RunDictionaryTest(int cLevel, int initSize, int threads, int addsPerThread, TestMethod testMethod)
{
TestHarness.TestLog("* RunDictionaryTest_{0}, Level={1}, initSize={2}, threads={3}, addsPerThread={4})",
PrintTestMethod(testMethod), cLevel, initSize, threads, addsPerThread);
ConcurrentDictionary <int, int> dict = new ConcurrentDictionary <int, int>(cLevel, 1);
int count = threads;
using (ManualResetEvent mre = new ManualResetEvent(false))
{
for (int i = 0; i < threads; i++)
{
int ii = i;
ThreadPool.QueueUserWorkItem(
(o) =>
{
for (int j = 0; j < addsPerThread; j++)
{
//call either of the two overloads of GetOrAdd
if (j + ii % 2 == 0)
{
dict.GetOrAdd(j, -j);
}
else
{
dict.GetOrAdd(j, x => - x);
}
}
if (Interlocked.Decrement(ref count) == 0)
{
mre.Set();
}
});
}
mre.WaitOne();
}
bool passed = true;
if (dict.Any(pair => pair.Key != -pair.Value))
{
TestHarness.TestLog(" > Invalid value for some key in the dictionary.");
passed = false;
}
var gotKeys = dict.Select(pair => pair.Key).OrderBy(i => i).ToArray();
var expectKeys = Enumerable.Range(0, addsPerThread);
if (!gotKeys.SequenceEqual(expectKeys))
{
TestHarness.TestLog(" > The set of keys in the dictionary is invalid.");
passed = false;
}
// Finally, let's verify that the count is reported correctly.
int expectedCount = addsPerThread;
int count1 = dict.Count, count2 = dict.ToArray().Length,
count3 = dict.ToList().Count;
if (count1 != expectedCount || count2 != expectedCount || count3 != expectedCount)
{
TestHarness.TestLog(" > Incorrect count of elements reported for the dictionary. Expected {0}, Dict.Count {1}, ToArray.Length {2}, ToList.Count {3}",
expectedCount, count1, count2, count3);
passed = false;
}
return(passed);
}
IEnumerator <IGrouping <TKey, TElement> > GetEnumeratorInternal()
{
return((IEnumerator <System.Linq.IGrouping <TKey, TElement> >)dictionary.Select((pair) => new ConcurrentGrouping <TKey, TElement> (pair.Key, pair.Value)).GetEnumerator());
}
public ShapeTable GetShapeTable(string themeId)
{
var cacheKey = $"ShapeTable:{themeId}";
if (!_memoryCache.TryGetValue(cacheKey, out ShapeTable shapeTable))
{
if (_logger.IsEnabled(LogLevel.Information))
{
_logger.LogInformation("Start building shape table");
}
HashSet <string> excludedFeatures;
// Here we don't use a lock for thread safety but for atomicity.
lock (_syncLock)
{
excludedFeatures = new HashSet <string>(_shapeDescriptors.Select(kv => kv.Value.Feature.Id));
}
var shapeDescriptors = new Dictionary <string, FeatureShapeDescriptor>();
foreach (var bindingStrategy in _bindingStrategies)
{
var strategyFeature = _typeFeatureProvider.GetFeatureForDependency(bindingStrategy.GetType());
var builder = new ShapeTableBuilder(strategyFeature, excludedFeatures);
bindingStrategy.Discover(builder);
var builtAlterations = builder.BuildAlterations();
BuildDescriptors(bindingStrategy, builtAlterations, shapeDescriptors);
}
// Here we don't use a lock for thread safety but for atomicity.
lock (_syncLock)
{
foreach (var kv in shapeDescriptors)
{
_shapeDescriptors[kv.Key] = kv.Value;
}
}
var enabledAndOrderedFeatureIds = _shellFeaturesManager
.GetEnabledFeaturesAsync()
.GetAwaiter()
.GetResult()
.Select(f => f.Id)
.ToList();
// let the application acting as a super theme for shapes rendering.
if (enabledAndOrderedFeatureIds.Remove(_hostingEnvironment.ApplicationName))
{
enabledAndOrderedFeatureIds.Add(_hostingEnvironment.ApplicationName);
}
var descriptors = _shapeDescriptors
.Where(sd => enabledAndOrderedFeatureIds.Contains(sd.Value.Feature.Id))
.Where(sd => IsModuleOrRequestedTheme(sd.Value.Feature, themeId))
.OrderBy(sd => enabledAndOrderedFeatureIds.IndexOf(sd.Value.Feature.Id))
.GroupBy(sd => sd.Value.ShapeType, StringComparer.OrdinalIgnoreCase)
.Select(group => new ShapeDescriptorIndex
(
shapeType: group.Key,
alterationKeys: group.Select(kv => kv.Key),
descriptors: _shapeDescriptors
))
.ToList();
shapeTable = new ShapeTable
(
descriptors: descriptors.ToDictionary(sd => sd.ShapeType, x => (ShapeDescriptor)x, StringComparer.OrdinalIgnoreCase),
bindings: descriptors.SelectMany(sd => sd.Bindings).ToDictionary(kv => kv.Key, kv => kv.Value, StringComparer.OrdinalIgnoreCase)
);
if (_logger.IsEnabled(LogLevel.Information))
{
_logger.LogInformation("Done building shape table");
}
_memoryCache.Set(cacheKey, shapeTable, new MemoryCacheEntryOptions {
Priority = CacheItemPriority.NeverRemove
});
}
return(shapeTable);
}
private static void UploadExperiments(ConcurrentDictionary <int, ExperimentEntity> experiments, IDictionary <int, TimeSpan> experimentInfo, AzureExperimentStorage storage)
{
storage.ImportExperiments(experiments.Select(e => e.Value)).Wait();
}
/// <summary>
/// Suggests other potential options based on the entities passed
/// </summary>
/// <param name="rows">The rows being imported</param>
/// <returns>Entites with suggestions</returns>
public Suggestions SuggestEntites(ImportRow[] rows)
{
var rowSuggestionsConcurrentDictionary = new ConcurrentDictionary<int, RowSuggestions>();
var suggestionsToReturn = new Suggestions();
var clients = new ConcurrentDictionary<Guid, FoundOps.Api.Models.Client>();
var locations = new ConcurrentDictionary<Guid, FoundOps.Api.Models.Location>();
var contactInfoSets = new ConcurrentDictionary<Guid, FoundOps.Api.Models.ContactInfo>();
Parallel.For((long)0, rows.Count(), rowIndex =>
{
var row = rows[rowIndex];
var rowSuggestions = new RowSuggestions();
#region Location
if (row.Location != null)
{
//Find all the Locations to be suggested by finding all Locations for the Client of the row
var locationSuggestions = row.Client != null
? _locations.Where(l => l.Value.ClientId == row.Client.Id).ToArray()
: null;
if (locationSuggestions != null)
{
//Add any of the suggestions to the rows suggestions
rowSuggestions.LocationSuggestions.AddRange(locationSuggestions.Select(l => l.Key));
var convertedLocationSuggestions = locationSuggestions.Select(l => l.Value).Select(FoundOps.Api.Models.Location.ConvertModel);
//Add all suggested Locations to the list of Locations to be returned
foreach (var location in convertedLocationSuggestions)
locations.GetOrAdd(location.Id, location);
}
//Add the matched/new location as the first suggestion
rowSuggestions.LocationSuggestions.Add(row.Location.Id);
//Add the location passed to the list of location entites
locations.GetOrAdd(row.Location.Id, row.Location);
}
#endregion
#region Client
if (row.Client != null)
{
//Find all the Clients to be suggested by finding all Clients for the Location of the row
var clientSuggestions = row.Location != null
? _clients.Where(c => c.Key == row.Location.ClientId).ToArray()
: null;
if (clientSuggestions != null)
{
//Add any of the suggestions to the rows suggestions
rowSuggestions.ClientSuggestions.AddRange(clientSuggestions.Select(c => c.Key));
var convertedClientSuggestions = clientSuggestions.Select(c => c.Value).Select(FoundOps.Api.Models.Client.ConvertModel);
//Add all suggested Clients to the list of Clients to be returned
foreach (var client in convertedClientSuggestions)
clients.GetOrAdd(client.Id, client);
}
//Add the matched/new client as the first suggestion
rowSuggestions.ClientSuggestions.Add(row.Client.Id);
//Add the Client passed to the list of client entites
clients.GetOrAdd(row.Client.Id, row.Client);
}
#endregion
//Repeat
if (row.Repeat != null)
rowSuggestions.Repeats.Add(row.Repeat);
//Contact Info
if (row.ContactInfoSet.Count != 0)
{
rowSuggestions.ContactInfoSuggestions.AddRange(row.ContactInfoSet.Select(ci => ci.Id));
foreach (var contactInfoSet in row.ContactInfoSet)
contactInfoSets.GetOrAdd(contactInfoSet.Id, contactInfoSet);
}
//Add this row's suggestions to the list to be returned
rowSuggestionsConcurrentDictionary.GetOrAdd((int)rowIndex, rowSuggestions);
});
//Order the row suggestions by rowIndex
suggestionsToReturn.RowSuggestions.AddRange(rowSuggestionsConcurrentDictionary.OrderBy(kvp => kvp.Key).Select(kvp => kvp.Value));
//Only add distinct Clients
var distinctClients = clients.Distinct();
suggestionsToReturn.Clients.AddRange(distinctClients.Select(dc => dc.Value));
//Only add distinct Locations
var distinctLocations = locations.Distinct();
suggestionsToReturn.Locations.AddRange(distinctLocations.Select(dl => dl.Value));
//Only add distinct ContactInfo
var distinctContactInfo = contactInfoSets.Select(ci => ci.Value).Distinct();
suggestionsToReturn.ContactInfoSet.AddRange(distinctContactInfo);
return suggestionsToReturn;
}
private void MigratePermissions0_9(IUnitOfWork uow)
{
var PermissionsDict = new ConcurrentDictionary<ulong, ServerPermissions0_9>();
if (!Directory.Exists("data/permissions/"))
{
_log.Warn("No data from permissions will be migrated.");
return;
}
foreach (var file in Directory.EnumerateFiles("data/permissions/"))
{
try
{
var strippedFileName = Path.GetFileNameWithoutExtension(file);
if (string.IsNullOrWhiteSpace(strippedFileName)) continue;
var id = ulong.Parse(strippedFileName);
var data = JsonConvert.DeserializeObject<ServerPermissions0_9>(File.ReadAllText(file));
PermissionsDict.TryAdd(id, data);
}
catch { }
}
var i = 0;
PermissionsDict
.Select(p => new { data = p.Value, gconfig = uow.GuildConfigs.For(p.Key) })
.AsParallel()
.ForAll(perms =>
{
try
{
var data = perms.data;
var gconfig = perms.gconfig;
gconfig.PermissionRole = data.PermissionsControllerRole;
gconfig.VerbosePermissions = data.Verbose;
gconfig.FilteredWords = new HashSet<FilteredWord>(data.Words.Select(w => w.ToLowerInvariant())
.Distinct()
.Select(w => new FilteredWord() { Word = w }));
gconfig.FilterWords = data.Permissions.FilterWords;
gconfig.FilterInvites = data.Permissions.FilterInvites;
gconfig.FilterInvitesChannelIds = new HashSet<FilterChannelId>();
gconfig.FilterInvitesChannelIds.AddRange(data.ChannelPermissions.Where(kvp => kvp.Value.FilterInvites)
.Select(cp => new FilterChannelId()
{
ChannelId = cp.Key
}));
gconfig.FilterWordsChannelIds = new HashSet<FilterChannelId>();
gconfig.FilterWordsChannelIds.AddRange(data.ChannelPermissions.Where(kvp => kvp.Value.FilterWords)
.Select(cp => new FilterChannelId()
{
ChannelId = cp.Key
}));
gconfig.CommandCooldowns = new HashSet<CommandCooldown>(data.CommandCooldowns
.Where(cc => !string.IsNullOrWhiteSpace(cc.Key) && cc.Value > 0)
.Select(cc => new CommandCooldown()
{
CommandName = cc.Key,
Seconds = cc.Value
}));
_log.Info("Migrating data from permissions folder for {0} done ({1})", gconfig.GuildId, ++i);
}
catch (Exception ex)
{
_log.Error(ex);
}
});
try { Directory.Move("data/permissions", "data/DELETE_ME_permissions"); } catch { }
}
/// <summary>
/// 获取所提供的所有数据库提供者名称。
/// </summary>
/// <returns></returns>
public static string[] GetSupportedProviders()
{
return(dicProviders.Select(s => s.Key).ToArray());
}
private async Task RemoveObjectsFromTenantAsync()
{
var client = TestClients.GetSAuthc1Client();
var results = new ConcurrentDictionary<string, Exception>();
// Delete applications
var deleteApplicationTasks = this.CreatedApplicationHrefs.Select(async href =>
{
try
{
var application = await client.GetResourceAsync<IApplication>(href);
var deleteResult = await application.DeleteAsync();
results.TryAdd(href, null);
}
catch (ResourceException rex)
{
if (rex.Code == 404)
{
// Already deleted
results.TryAdd(href, null);
}
}
catch (Exception e)
{
results.TryAdd(href, e);
}
});
// Delete directories
var deleteDirectoryTasks = this.CreatedDirectoryHrefs.Select(async href =>
{
try
{
var directory = await client.GetResourceAsync<IDirectory>(href);
var deleteResult = await directory.DeleteAsync();
results.TryAdd(href, null);
}
catch (ResourceException rex)
{
if (rex.Code == 404)
{
// Already deleted
results.TryAdd(href, null);
}
}
catch (Exception e)
{
results.TryAdd(href, e);
}
});
// Delete organizations
var deleteOrganizationTasks = this.CreatedOrganizationHrefs.Select(async href =>
{
try
{
var org = await client.GetResourceAsync<IOrganization>(href);
var deleteResult = await org.DeleteAsync();
results.TryAdd(href, null);
}
catch (ResourceException rex)
{
if (rex.Code == 404)
{
// Already deleted
results.TryAdd(href, null);
}
}
catch (Exception e)
{
results.TryAdd(href, e);
}
});
await Task.WhenAll(
Task.WhenAll(deleteApplicationTasks),
Task.WhenAll(deleteDirectoryTasks),
Task.WhenAll(deleteOrganizationTasks));
// All done! Throw errors if any occurred
bool anyErrors = results.Any(kvp => kvp.Value != null);
if (anyErrors)
{
throw new ApplicationException(
"Errors occurred during test cleanup. Full log: " + Environment.NewLine
+ string.Join(Environment.NewLine, results.Select(kvp => $"{kvp.Key} : '{(kvp.Value == null ? "Good" : kvp.Value.Message)}'")));
}
}
public async Task <List <HttpResponseMessage> > Execute()
{
System.Net.ServicePointManager.DefaultConnectionLimit = 1000;
var tasksForUser1 = new List <Task <HttpResponseMessage> >();
var tasksForUser2 = new List <Task <HttpResponseMessage> >();
using (var client = new HttpClient())
{
Stopwatch sw = new Stopwatch();
var learnersTasks = new List <List <Task <HttpResponseMessage> > >();
Console.Write("Please enter number of learners: (10 or 100 or 500 etc):");
var readFromUser = Console.ReadLine();
int totalLearners;
int.TryParse(readFromUser, out totalLearners);
Console.WriteLine("Request Process Started!");
var allReqts = new ConcurrentDictionary <Guid, List <HttpRequestMessage> >();
var reqs = new ConcurrentDictionary <Guid, HttpRequestMessage>();
var reqsListNotThreadSafe = new List <HttpRequestMessage>();
int counter = 0;
Parallel.For(0, totalLearners, (index) =>
{
var tasksPerLearner = new List <HttpRequestMessage>();
for (int i = 0; i < 600; i++)
{
counter++;
//allTasks.TryAdd(counter, client.SendAsync(CreateRequestObject()));
tasksPerLearner.Add(CreateRequestObject());
}
allReqts.TryAdd(Guid.NewGuid(), tasksPerLearner);
//learnersTasks.Add(tasksForUser);
});
//for (int i = 0; i < totalLearners; i++)
//{
// for (int j = 0; j < 600; j++)
// {
// reqsListNotThreadSafe.Add(CreateRequestObject());
// }
//}
try
{
sw.Start();
Console.WriteLine("Request waiting for the learners tasks to complete! - Total requests: " + allReqts.Count());
// Parallel.ForEach(learnersTasks, async (tasks) => await Task.WhenAll(tasks));
// await learnersTasks.ParallelForEachAsync(async tasks => await Task.WhenAll(tasks));
var reqList = reqs.Select(x => x.Value).ToList();
var responses = new List <HttpResponseMessage>();
foreach (var item in allReqts)
{
responses.AddRange(await item.Value.DownloadAsync(1000));
}
;
// var responses = await reqList.DownloadAsync(1000);
//await Task.WhenAll(allTasks.ToList().Select(x=>x.Value));
//foreach (var tasks in learnersTasks)
//{
// await Task.WhenAll(tasks);
//}
sw.Stop();
// var result1 = responses[0];
// Console.WriteLine("Total requests with success status:" + responses.Count(x => x.IsSuccessStatusCode));
// Console.WriteLine("Total requests with Error status:" + responses.Count(x => !x.IsSuccessStatusCode));
// result.ForEach(x => Console.WriteLine(x.ToString()));
Console.WriteLine("Request Completed for all learners!");
Console.WriteLine("Time it took - in seconds: " + sw.Elapsed + " , in milliseconds: " + sw.ElapsedMilliseconds);
return(null);
}
catch (Exception ex)
{
Console.WriteLine("Error occured while making http requests");
Console.WriteLine("Error details: " + ex.ToString());
}
return(null);
}
}
public void CanDoCompetingConsumers(int messageCount, int threadCount)
{
using (CreateQueue(InputQueueName).Purge()) { }
var keepRunning = true;
var lastMessageReceivedTime = DateTime.UtcNow;
var receivedMessagesDistribution = new ConcurrentDictionary<int, int>();
var receivers = Enumerable.Range(0, threadCount)
.Select(i =>
{
var queue = TrackDisposable(CreateQueue(InputQueueName));
var number = i + 1;
return new Thread(() =>
{
Console.WriteLine("Receiver {0} started", number);
while (keepRunning)
{
using (var tx = new TransactionScope())
{
var receivedMessage = queue.ReceiveMessage(new AmbientTransactionContext());
if (receivedMessage != null)
{
receivedMessagesDistribution.AddOrUpdate(number, (key) => 1, (key, value) => value + 1);
Console.Write(".");
lastMessageReceivedTime = DateTime.UtcNow;
}
tx.Complete();
}
}
Console.WriteLine("Receiver {0} stopped", number);
});
})
.ToList();
var sender = CreateQueue("test_competing_sender");
Console.WriteLine("Sending {0} messages", messageCount);
messageCount.Times(() => sender.Send(InputQueueName, new TransportMessageToSend
{
Headers = new Dictionary<string, object>(),
Body = Encoding.UTF8.GetBytes("w00000t!")
}, new NoTransaction()));
Console.WriteLine("Starting {0} receivers", receivers.Count);
receivers.ForEach(r => r.Start());
lastMessageReceivedTime = DateTime.UtcNow;
while (lastMessageReceivedTime.ElapsedUntilNow() < 3.Seconds())
{
Console.WriteLine("Waiting...");
Thread.Sleep(2.Seconds());
}
Console.WriteLine("Stopping receivers...");
keepRunning = false;
receivers.ForEach(r => r.Join());
Console.WriteLine("Got {0} messages distributed among workers like this:", receivedMessagesDistribution.Sum(d => d.Value));
Console.WriteLine(string.Join(Environment.NewLine, receivedMessagesDistribution.Select(kvp => string.Format("{0:000}: {1}", kvp.Key, new string('=', kvp.Value)))));
}
IEnumerator IEnumerable.GetEnumerator()
{
return(_families.Select(x => x.Value).ToArray().GetEnumerator());
}
/// <summary>
/// Initializes a new instance of the <see cref="DashboardViewModel"/> class.
/// </summary>
public DashboardViewModel()
{
var universities = new UniversityBuilder().Build(2).ToList();
this.DataPoints = new ObservableCollection<DataPoint>();
var mongoDbRepository = new MongoDbRepository(ConfigurationManager.AppSettings["MongoDbConnectionString"]);
var entityFrameworkRepository = new EntityFrameworkRepository(ConfigurationManager.AppSettings["EntityFrameworkConnectionString"]);
var adoRepository = new AdoRepository(ConfigurationManager.AppSettings["AdoConnectionString"]);
var redisRepository = new RedisRepository(ConfigurationManager.AppSettings["RedisConnectionString"]);
var repositories = new List<IRepository> { mongoDbRepository, entityFrameworkRepository, adoRepository, redisRepository };
var progress = new ConcurrentDictionary<IRepository, int>(repositories.Select(r => new KeyValuePair<IRepository, int>(r, 0)));
this.AdoNetRepository = new ChartableRepository
{
Title = "ADO.NET",
Key = "Ado",
Brush = new SolidColorBrush((Color)ColorConverter.ConvertFromString("#EC098C")),
Repository = adoRepository
};
this.Ef5Repository = new ChartableRepository
{
Title = "EF5",
Key = "EntityFramework",
Brush = new SolidColorBrush((Color)ColorConverter.ConvertFromString("#00B159")),
Repository = entityFrameworkRepository
};
this.MongoDbRepository = new ChartableRepository
{
Title = "MongoDB",
Key = "MongoDB",
Brush = new SolidColorBrush((Color)ColorConverter.ConvertFromString("#7C4199")),
Repository = mongoDbRepository
};
this.RedisRepository = new ChartableRepository
{
Title = "Redis",
Key = "Redis",
Brush = new SolidColorBrush((Color)ColorConverter.ConvertFromString("#F37735")),
Repository = redisRepository
};
this.ChartableRepositories = new List<ChartableRepository>{ this.AdoNetRepository, this.Ef5Repository, this.MongoDbRepository, this.RedisRepository };
var timer = new TickOnceDispatcherTimer(new TimeSpan(0, 0, 0, 0, 100));
timer.Tick += delegate
{
var datapoint = new DataPoint
{
Ticks = (this.DataPoints.Count + 1) / 10.0,
MongoDB = progress[mongoDbRepository],
EntityFramework = progress[entityFrameworkRepository],
Ado = progress[adoRepository],
Redis = progress[redisRepository]
};
this.DataPoints.Add(datapoint);
if (this.AdoNetRepository.LastDuration == default(TimeSpan) && datapoint.Ado == universities.Count)
{
this.AdoNetRepository.LastDuration = Stopwatch.Elapsed;
}
if (this.Ef5Repository.LastDuration == default(TimeSpan) && datapoint.EntityFramework == universities.Count)
{
this.Ef5Repository.LastDuration = Stopwatch.Elapsed;
}
if (this.MongoDbRepository.LastDuration == default(TimeSpan) && datapoint.MongoDB == universities.Count)
{
this.MongoDbRepository.LastDuration = Stopwatch.Elapsed;
}
if (this.RedisRepository.LastDuration == default(TimeSpan) && datapoint.Redis == universities.Count)
{
this.RedisRepository.LastDuration = Stopwatch.Elapsed;
}
this.OnPropertyChanged("Stopwatch");
};
Action<IRepository> notifyProgress = delegate(IRepository repository)
{
if (!timer.IsEnabled)
{
timer.Start();
}
progress[repository]++;
};
EventHandler commandCompleteEventHandler = delegate
{
Task.Factory.StartNew(timer.StopWhenTicked);
this.Stopwatch.Stop();
};
this.ResetCommand = new RelayCommand(delegate
{
this.DataPoints.Clear();
this.DataPoints.Add(new DataPoint { Ticks = 0, MongoDB = 0, EntityFramework = 0, Ado = 0 });
foreach (IRepository repository in progress.Select(x => x.Key))
{
progress[repository] = 0;
}
this.Ef5Repository.LastDuration = default(TimeSpan);
this.AdoNetRepository.LastDuration = default(TimeSpan);
this.MongoDbRepository.LastDuration = default(TimeSpan);
this.RedisRepository.LastDuration = default(TimeSpan);
this.Stopwatch.Restart();
});
this.Stopwatch = new Stopwatch();
this.ToggleCommand = new ToggleCommand();
this.BuildCommand = new BuildCommand(repositories);
var createCommand = new CreateCommand(this.ChartableRepositories.Where(x => x.Selected).Select(x => x.Repository), universities, notifyProgress);
createCommand.CommandComplete += commandCompleteEventHandler;
this.CreateCommand = new CompositeCommand(this.ResetCommand, createCommand);
var retrieveCommand = new RetrieveCommand(this.ChartableRepositories.Where(x => x.Selected).Select(x => x.Repository), universities, notifyProgress);
retrieveCommand.CommandComplete += commandCompleteEventHandler;
this.RetrieveCommand = new CompositeCommand(this.ResetCommand, retrieveCommand);
var updateCommand = new UpdateCommand(this.ChartableRepositories.Where(x => x.Selected).Select(x => x.Repository), universities, notifyProgress);
updateCommand.CommandComplete += commandCompleteEventHandler;
this.UpdateCommand = new CompositeCommand(this.ResetCommand, updateCommand);
var queryCommand = new QueryCommand(this.ChartableRepositories.Where(x => x.Selected && x != this.RedisRepository).Select(x => x.Repository), new StudentBuilder().GetForenames(1000), notifyProgress);
queryCommand.CommandComplete += commandCompleteEventHandler;
this.QueryCommand = new CompositeCommand(this.ResetCommand, queryCommand);
var deleteCommand = new DeleteCommand(this.ChartableRepositories.Where(x => x.Selected).Select(x => x.Repository), universities, notifyProgress);
deleteCommand.CommandComplete += commandCompleteEventHandler;
this.DeleteCommand = new CompositeCommand(this.ResetCommand, deleteCommand);
}
private static bool RunDictionaryTest_Update1(int cLevel, int threads, int updatesPerThread)
{
TestHarness.TestLog("* RunDictionaryTest_Update1(cLevel={0}, threads={1}, updatesPerThread={2})", cLevel, threads, updatesPerThread);
IDictionary<int, int> dict = new ConcurrentDictionary<int, int>(cLevel, 1);
for (int i = 1; i <= updatesPerThread; i++) dict[i] = i;
int running = threads;
using (ManualResetEvent mre = new ManualResetEvent(false))
{
for (int i = 0; i < threads; i++)
{
int ii = i;
ThreadPool.QueueUserWorkItem(
(o) =>
{
for (int j = 1; j <= updatesPerThread; j++)
{
dict[j] = (ii + 2) * j;
}
if (Interlocked.Decrement(ref running) == 0) mre.Set();
});
}
mre.WaitOne();
}
if ((from pair in dict
let div = pair.Value / pair.Key
let rem = pair.Value % pair.Key
select rem != 0 || div < 2 || div > threads + 1)
.Any(res => res))
{
TestHarness.TestLog(" > Invalid value for some key in the dictionary.");
return false;
}
var gotKeys = dict.Select(pair => pair.Key).OrderBy(i => i).ToArray();
var expectKeys = Enumerable.Range(1, updatesPerThread);
if (!gotKeys.SequenceEqual(expectKeys))
{
TestHarness.TestLog(" > The set of keys in the dictionary is invalid.");
return false;
}
return true;
}
public IEnumerable <string> GetErrors()
{
return(_errors.Select(e => e.Value).ToList());
}
private static bool RunDictionaryTest_Add1(int cLevel, int initSize, int threads, int addsPerThread)
{
TestHarness.TestLog(
"* RunDictionaryTest_Add1(cLevel={0}, initSize={1}, threads={2}, addsPerThread={3})",
cLevel, initSize, threads, addsPerThread);
IDictionary<int, int> dict = new ConcurrentDictionary<int, int>(cLevel, 1);
int count = threads;
using (ManualResetEvent mre = new ManualResetEvent(false))
{
for (int i = 0; i < threads; i++)
{
int ii = i;
ThreadPool.QueueUserWorkItem(
(o) =>
{
for (int j = 0; j < addsPerThread; j++)
{
dict.Add(j + ii * addsPerThread, -(j + ii * addsPerThread));
}
if (Interlocked.Decrement(ref count) == 0) mre.Set();
});
}
mre.WaitOne();
}
if (dict.Any(pair => pair.Key != -pair.Value))
{
TestHarness.TestLog(" > Invalid value for some key in the dictionary.");
return false;
}
var gotKeys = dict.Select(pair => pair.Key).OrderBy(i => i).ToArray();
var expectKeys = Enumerable.Range(0, threads * addsPerThread);
if (!gotKeys.SequenceEqual(expectKeys))
{
TestHarness.TestLog(" > The set of keys in the dictionary is invalid.");
return false;
}
// Finally, let's verify that the count is reported correctly.
int expectedCount = threads * addsPerThread;
if (dict.Count != expectedCount || dict.ToArray().Length != expectedCount || dict.ToList().Count() != expectedCount)
{
TestHarness.TestLog(" > Incorrect count of elements reported for the dictionary.");
return false;
}
return true;
}
private static void DebugReport(ConcurrentDictionary <string, BData> report)
{
Debug.Log(string.Join(Environment.NewLine + Environment.NewLine,
report.Select(x => string.Format("{0}:{2}{1}", x.Key, x.Value.ToString(), Environment.NewLine))
.ToArray()));
}
/// <summary>
/// Manipulates the input strings to existing or new entities.
/// Then it passes those entities to SuggestEntities to generate suggestions.
/// <note>This method is only used for the initial step of the Import. Once entites are generated; use SuggestEntities</note>
/// </summary>
/// <param name="rowsWithHeaders">List of string[]. The first string[] is the headers, the rest are data to be imported</param>
/// <returns>Entites with suggestions</returns>
public Suggestions ValidateThenSuggestEntities(List<string[]> rowsWithHeaders)
{
var headers = rowsWithHeaders[0];
rowsWithHeaders.RemoveAt(0);
var rows = rowsWithHeaders;
#region Assign Column placement
//Client
var clientNameCol = Array.IndexOf(headers, "Client Name");
//Location
var addressLineOneCol = Array.IndexOf(headers, "Address Line One");
var addressLineTwoCol = Array.IndexOf(headers, "Address Line Two");
var cityCol = Array.IndexOf(headers, "AdminDistrictTwo");
var stateCol = Array.IndexOf(headers, "AdminDistrictOne");
var zipCodeCol = Array.IndexOf(headers, "PostalCode");
var countryCodeCol = Array.IndexOf(headers, "Country Code");
var regionNameCol = Array.IndexOf(headers, "Region Name");
var latitudeCol = Array.IndexOf(headers, "Latitude");
var longitudeCol = Array.IndexOf(headers, "Longitude");
//Repeat
var frequencyCol = Array.IndexOf(headers, "Frequency");
var repeatEveryCol = Array.IndexOf(headers, "Repeat Every");
var startDateCol = Array.IndexOf(headers, "Start Date");
var endDateCol = Array.IndexOf(headers, "End Date");
var endAfterCol = Array.IndexOf(headers, "End After Times");
var frequencyDetailCol = Array.IndexOf(headers, "Frequency Detail");
//Contact Info
var phoneNumberValueCols = headers.Where(h => h.Contains("Phone Value")).ToArray();
var phoneNumberLabelCols = headers.Where(h => h.Contains("Phone Label")).ToArray();
var emailValueCols = headers.Where(h => h.Contains("Email Value")).ToArray();
var emailLabelCols = headers.Where(h => h.Contains("Email Label")).ToArray();
var websiteValueCols = headers.Where(h => h.Contains("Website Value")).ToArray();
var websiteLabelCols = headers.Where(h => h.Contains("Website Label")).ToArray();
var otherValueCols = headers.Where(h => h.Contains("Other Value")).ToArray();
var otherLabelCols = headers.Where(h => h.Contains("Other Label")).ToArray();
#endregion
var importRowsConcurrentDictionary = new ConcurrentDictionary<int, ImportRow>();
var rowCount = rows.Count();
Parallel.For((long)0, rowCount, rowIndex =>
{
var row = rows[(int)rowIndex];
var importRow = new ImportRow();
#region Location
var importedLocation = new Api.Models.Location();
string clientName;
//Checks if at least one location column is passed
if (new[] { addressLineOneCol, addressLineTwoCol, cityCol, stateCol, countryCodeCol, zipCodeCol, latitudeCol, longitudeCol }.Any(col => col != -1))
{
var latitude = latitudeCol != -1 ? row[latitudeCol] : null;
var longitude = longitudeCol != -1 ? row[longitudeCol] : null;
var addressLineOne = addressLineOneCol != -1 ? row[addressLineOneCol] : null;
var addressLineTwo = addressLineTwoCol != -1 ? row[addressLineTwoCol] : null;
var adminDistrictTwo = cityCol != -1 ? row[cityCol] : null;
var adminDistrictOne = stateCol != -1 ? row[stateCol] : null;
var postalCode = zipCodeCol != -1 ? row[zipCodeCol] : null;
var regionName = regionNameCol != -1 ? row[regionNameCol] : null;
//If lat/lon dont have values, try to Geocode
if (latitude == null && longitude == null)
{
var address = new Address
{
AddressLineOne = addressLineOne,
City = adminDistrictTwo,
State = adminDistrictOne,
ZipCode = postalCode
};
//Attempt to Geocode the address
var geocodeResult = BingLocationServices.TryGeocode(address).FirstOrDefault(gc => gc != null);
latitude = geocodeResult != null ? geocodeResult.Latitude : null;
longitude = geocodeResult != null ? geocodeResult.Longitude : null;
//If they still do not have values, throw error on the location
if (latitude == null && longitude == null)
importedLocation.StatusInt = (int)ImportStatus.Error;
//Matched the entire address to a location (AddressLineOne, AddressLineTwo, City, State and ZipCode)
var matchedLocation = _locations.FirstOrDefault(l => l.Value.AddressLineOne == addressLineOne && l.Value.AddressLineTwo == addressLineTwo
&& l.Value.AdminDistrictTwo == adminDistrictTwo && l.Value.AdminDistrictOne == adminDistrictOne && l.Value.PostalCode == postalCode);
if (matchedLocation.Key != Guid.Empty)
importedLocation = ConvertLocationSetRegionAndStatus(matchedLocation.Value, regionName, ImportStatus.Linked);
else
{
clientName = clientNameCol != -1 ? row[clientNameCol] : null;
//Matched the address entered and client name to a location
matchedLocation = _locations.FirstOrDefault(l => clientName != null && l.Value.ClientId != null &&
(addressLineTwo != null && (l.Value.AddressLineOne == addressLineOne && l.Value.AddressLineTwo == addressLineTwo
&& _clients.First(c => c.Key == l.Value.ClientId).Value.Name == clientName)));
if (matchedLocation.Key != Guid.Empty)
importedLocation = ConvertLocationSetRegionAndStatus(matchedLocation.Value, regionName, ImportStatus.Linked);
}
}
//If lat/lon exist, try and match based on that
if (latitude != null && longitude != null && importedLocation.StatusInt != (int)ImportStatus.Error)
{
var setError = false;
//check for invalid deciamls
Decimal convertedLatitude = 0;
Decimal convertedLongitude = 0;
Decimal tempdeciaml;
if (Decimal.TryParse(latitude, out tempdeciaml))
convertedLatitude = tempdeciaml;
else
setError = true;
if (Decimal.TryParse(longitude, out tempdeciaml))
convertedLongitude = tempdeciaml;
else
setError = true;
if (!setError)
{
var roundedLatitude = Math.Round(convertedLatitude, 6);
var roundedLongitude = Math.Round(convertedLongitude, 6);
//Try and match a location to one in the FoundOPS system
var matchedLocation = addressLineTwo != null
//Use this statement if Address Line Two is not null
? _locations.FirstOrDefault(l => l.Value.AddressLineTwo == addressLineTwo && (l.Value.Latitude != null && l.Value.Longitude != null)
&& (decimal.Round(l.Value.Latitude.Value, 6) == roundedLatitude
&& decimal.Round(l.Value.Longitude.Value, 6) == roundedLongitude))
//Use this statement if Address Line Two is null
: _locations.FirstOrDefault(l => (l.Value.Latitude != null && l.Value.Longitude != null)
&& (decimal.Round(l.Value.Latitude.Value, 6) == roundedLatitude
&& decimal.Round(l.Value.Longitude.Value, 6) == roundedLongitude));
//If a match is found, assign Linked status to all cells
if (matchedLocation.Key != Guid.Empty)
importedLocation = ConvertLocationSetRegionAndStatus(matchedLocation.Value, regionName, ImportStatus.Linked);
}
else
importedLocation.StatusInt = (int) ImportStatus.Error;
}
//If no linked location is found, it means that the location is new
if (importedLocation.StatusInt != (int)ImportStatus.Linked)
{
var status = (int) ImportStatus.New;
if (importedLocation.StatusInt == (int) ImportStatus.Error)
status = (int) ImportStatus.Error;
importedLocation = new Api.Models.Location
{
Id = Guid.NewGuid(),
AddressLineOne = addressLineOne,
AddressLineTwo = addressLineTwo ?? null,
AdminDistrictTwo = adminDistrictTwo,
AdminDistrictOne = adminDistrictOne,
PostalCode = postalCode,
CountryCode = "US",
ContactInfoSet = new List<ContactInfo>(),
Latitude = latitude,
Longitude = longitude,
StatusInt = status,
Region = SetRegion(regionName)
};
//Since the location is new, we add it to the list of locations to be searched next time we try and match
_locations.GetOrAdd(importedLocation.Id, FoundOps.Api.Models.Location.ConvertBack(importedLocation));
}
importRow.Location = importedLocation;
}
#endregion
#region Client
clientName = clientNameCol != -1 ? row[clientNameCol] : null;
//Make sure a Client is being Imported
if (clientName != null)
{
//Find an existing Client based on the Name
var existingClient = _clients.FirstOrDefault(c => c.Value.Name == clientName);
Models.Client importedClient;
//If a Client was found, set it to be imported with a Linked Status
if (existingClient.Key != Guid.Empty)
{
importedClient = FoundOps.Api.Models.Client.ConvertModel(existingClient.Value);
importedClient.StatusInt = (int)ImportStatus.Linked;
}
//If not, create a new Client and add it to _clients for matching later
else
{
importedClient = new FoundOps.Api.Models.Client
{
Id = Guid.NewGuid(),
Name = clientName,
ContactInfoSet = new List<ContactInfo>(),
StatusInt = (int)ImportStatus.New
};
//Since the client is new, we add it to the list of clients to be searched next time we try and match
_clients.GetOrAdd(importedClient.Id, FoundOps.Api.Models.Client.ConvertBack(importedClient));
}
importRow.Client = importedClient;
}
#endregion
#region Repeat
if (new[] { startDateCol, endDateCol, endAfterCol, repeatEveryCol, frequencyCol, frequencyDetailCol }.Any(col => col != -1))
{
var setError = false;
//check for invalid DateTime's
DateTime startDate = DateTime.UtcNow.Date;
DateTime? endDate = null;
DateTime tempDate;
if (startDateCol != -1 && !String.IsNullOrEmpty(row[startDateCol]))
{
if (DateTime.TryParse(row[startDateCol], out tempDate))
startDate = tempDate;
else
{
startDate = DateTime.UtcNow.Date;
setError = true;
}
}
if (endDateCol != -1 && !String.IsNullOrEmpty(row[endDateCol]))
{
if (DateTime.TryParse(row[endDateCol], out tempDate))
endDate = tempDate;
else
{
endDate = null;
setError = true;
}
}
//Check for invalid Int's
int? endAfterTimes = null;
int? repeatEveryTimes = null;
int tempInt;
if (endAfterCol != -1 && !String.IsNullOrEmpty(row[endAfterCol]))
{
if (Int32.TryParse(row[endAfterCol], out tempInt))
endAfterTimes = tempInt;
else
{
endAfterTimes = null;
setError = true;
}
}
if (repeatEveryCol != -1 && !String.IsNullOrEmpty(row[repeatEveryCol]))
{
if (Int32.TryParse(row[repeatEveryCol], out tempInt))
repeatEveryTimes = tempInt;
else
{
repeatEveryTimes = null;
setError = true;
}
}
//Create the Repeat object
var repeat = new Repeat
{
Id = Guid.NewGuid(),
StartDate = startDate,
EndDate = endDate,
EndAfterTimes = endAfterTimes,
RepeatEveryTimes = repeatEveryTimes
};
#region Frequency
var val = frequencyCol != -1 ? row[frequencyCol].ToLower() : String.Empty;
switch (val)
{
case "o":
case "once":
repeat.FrequencyInt = (int)Frequency.Once;
break;
case "d":
case "daily":
repeat.FrequencyInt = (int)Frequency.Daily;
break;
case "w":
case "weekly":
repeat.FrequencyInt = (int)Frequency.Weekly;
break;
case "m":
case "monthly":
repeat.FrequencyInt = (int)Frequency.Monthly;
break;
case "y":
case "yearly":
repeat.FrequencyInt = (int)Frequency.Yearly;
break;
default:
repeat.FrequencyInt = null;
break;
}
#endregion
#region Frequency Detail
val = frequencyDetailCol != -1 ? row[frequencyDetailCol].ToLower() : String.Empty;
val = val.Replace(" ", "");
if (repeat.Frequency == Frequency.Weekly)
{
var startDayOfWeek = repeat.StartDate.DayOfWeek;
//If it is empty assume the Start Date
if (string.IsNullOrEmpty(val))
repeat.FrequencyDetailAsWeeklyFrequencyDetail = new[] { startDayOfWeek };
else
{
var dayStrings = val.Split(',');
var daysOfWeek = new List<DayOfWeek>();
if (dayStrings.Any(s => s == "s" || s == "su" || s == "sun" || s == "sunday"))
daysOfWeek.Add(DayOfWeek.Sunday);
if (dayStrings.Any(s => s == "m" || s == "mo" || s == "mon" || s == "monday"))
daysOfWeek.Add(DayOfWeek.Monday);
if (dayStrings.Any(s => s == "t" || s == "tu" || s == "tue" || s == "tues" || s == "tuesday"))
daysOfWeek.Add(DayOfWeek.Tuesday);
if (dayStrings.Any(s => s == "w" || s == "we" || s == "wed" || s == "wednesday"))
daysOfWeek.Add(DayOfWeek.Wednesday);
if (dayStrings.Any(s => s == "r" || s == "th" || s == "tr" || s == "thur" || s == "thurs" || s == "thursday"))
daysOfWeek.Add(DayOfWeek.Thursday);
if (dayStrings.Any(s => s == "f" || s == "fr" || s == "fri" || s == "friday"))
daysOfWeek.Add(DayOfWeek.Friday);
if (dayStrings.Any(s => s == "s" || s == "sa" || s == "sat" || s == "saturday"))
daysOfWeek.Add(DayOfWeek.Saturday);
//Make sure the days include the startdate
if (!daysOfWeek.Contains(startDayOfWeek))
daysOfWeek.Add(startDayOfWeek);
repeat.FrequencyDetailAsWeeklyFrequencyDetail = daysOfWeek.OrderBy(e => (int)e).ToArray();
}
}
if (repeat.Frequency == Frequency.Monthly)
{
if (string.IsNullOrEmpty(val) || val == "date")
{
repeat.FrequencyDetailAsMonthlyFrequencyDetail = MonthlyFrequencyDetail.OnDayInMonth;
}
else if (val == "day")
{
var detailsAvailable = repeat.AvailableMonthlyFrequencyDetailTypes.ToList();
if (detailsAvailable.Count() > 1)
detailsAvailable.Remove(MonthlyFrequencyDetail.OnDayInMonth);
repeat.FrequencyDetailAsMonthlyFrequencyDetail = detailsAvailable.First();
}
}
#endregion
repeat.StatusInt = repeat.RepeatEveryTimes == null || repeat.FrequencyInt == null || setError
? (int)ImportStatus.Error
: (int)ImportStatus.New;
importRow.Repeat = repeat;
}
#endregion
#region Contact Info
//Create label and value dictionaries for Phone Number contact information
var phoneValueDictionary = phoneNumberValueCols.ToDictionary(p => Convert.ToInt32(p.Split(' ').ElementAt(2)), p => row[Array.IndexOf(headers, p)]);
var phoneLabelDictionary = phoneNumberLabelCols.ToDictionary(p => Convert.ToInt32(p.Split(' ').ElementAt(2)), p => row[Array.IndexOf(headers, p)]);
//Create label and value dictionaries for Email Address contact information
var emailValueDictionary = emailValueCols.ToDictionary(e => Convert.ToInt32(e.Split(' ').ElementAt(2)), e => row[Array.IndexOf(headers, e)]);
var emailLabelDictionary = emailLabelCols.ToDictionary(e => Convert.ToInt32(e.Split(' ').ElementAt(2)), e => row[Array.IndexOf(headers, e)]);
//Create label and value dictionaries for Website contact information
var websiteValueDictionary = websiteValueCols.ToDictionary(w => Convert.ToInt32(w.Split(' ').ElementAt(2)), w => row[Array.IndexOf(headers, w)]);
var websiteLabelDictionary = websiteLabelCols.ToDictionary(w => Convert.ToInt32(w.Split(' ').ElementAt(2)), w => row[Array.IndexOf(headers, w)]);
//Create label and value dictionaries for any other types of contact information
var otherValueDictionary = otherValueCols.ToDictionary(o => Convert.ToInt32(o.Split(' ').ElementAt(2)), o => row[Array.IndexOf(headers, o)]);
var otherLabelDictionary = otherLabelCols.ToDictionary(o => Convert.ToInt32(o.Split(' ').ElementAt(2)), o => row[Array.IndexOf(headers, o)]);
//Find which type of contact info is being imported the most
//This way we only have one loop
var maxLabel = Math.Max(Math.Max(phoneLabelDictionary.Count, emailLabelDictionary.Count), Math.Max(websiteLabelDictionary.Count, otherLabelDictionary.Count));
var maxValue = Math.Max(Math.Max(phoneValueDictionary.Count, emailValueDictionary.Count), Math.Max(websiteValueDictionary.Count, otherValueDictionary.Count));
var max = Math.Max(maxLabel, maxValue);
//Dictionary of Cantact Information to be imported for the row
var concurrentContactInfoDictionary = new ConcurrentDictionary<Guid, ContactInfo>();
Parallel.For((long)1, max + 1, contactIndex =>
{
//Phone
if (phoneLabelDictionary.Count >= contactIndex || phoneValueDictionary.Count >= contactIndex)
MatchContactInfo(phoneLabelDictionary, phoneValueDictionary, contactIndex, concurrentContactInfoDictionary, "Phone Number");
//Email
if (emailLabelDictionary.Count >= contactIndex || emailValueDictionary.Count >= contactIndex)
MatchContactInfo(emailLabelDictionary, emailValueDictionary, contactIndex, concurrentContactInfoDictionary, "Email Address");
//Website
if (websiteLabelDictionary.Count >= contactIndex || websiteValueDictionary.Count >= contactIndex)
MatchContactInfo(websiteLabelDictionary, websiteValueDictionary, contactIndex, concurrentContactInfoDictionary, "Website");
//Other
if (otherLabelDictionary.Count >= contactIndex || otherValueDictionary.Count >= contactIndex)
MatchContactInfo(otherLabelDictionary, otherValueDictionary, contactIndex, concurrentContactInfoDictionary, "Other");
});
//Once all the contact info sets are made or matched, add them to the ImportRow
importRow.ContactInfoSet.AddRange(concurrentContactInfoDictionary.Select(ci => ci.Value));
#endregion
//Add the ImportRow to the concurrent dictionary of ImportRows
importRowsConcurrentDictionary.GetOrAdd((int)rowIndex, importRow);
});
//Order the ImportRows by rowIndex
var importRows = importRowsConcurrentDictionary.OrderBy(kvp => kvp.Key).Select(kvp => kvp.Value).ToArray();
//Send the validated ImportRows to get suggestions and return
return SuggestEntites(importRows);
}
public async Task UpdateUsersOnlineCount()
{
var conectId = OnlineUsers.Select(x => x.Key).Distinct().Count();
await Clients.All.SendAsync("GetOnlineUsers", conectId);
}
public List <MqttValueContract> GetAllContracts() => _values.Select(entry => entry.Value).ToList();
private static bool RunDictionaryTest(int cLevel, int initSize, int threads, int addsPerThread, TestMethod testMethod)
{
TestHarness.TestLog("* RunDictionaryTest_{0}, Level={1}, initSize={2}, threads={3}, addsPerThread={4})",
PrintTestMethod(testMethod), cLevel, initSize, threads, addsPerThread);
ConcurrentDictionary<int, int> dict = new ConcurrentDictionary<int, int>(cLevel, 1);
int count = threads;
using (ManualResetEvent mre = new ManualResetEvent(false))
{
for (int i = 0; i < threads; i++)
{
int ii = i;
ThreadPool.QueueUserWorkItem(
(o) =>
{
for (int j = 0; j < addsPerThread; j++)
{
//call either of the two overloads of GetOrAdd
if (j + ii % 2 == 0)
{
dict.GetOrAdd(j, -j);
}
else
{
dict.GetOrAdd(j, x => -x);
}
}
if (Interlocked.Decrement(ref count) == 0) mre.Set();
});
}
mre.WaitOne();
}
bool passed = true;
if (dict.Any(pair => pair.Key != -pair.Value))
{
TestHarness.TestLog(" > Invalid value for some key in the dictionary.");
passed = false;
}
var gotKeys = dict.Select(pair => pair.Key).OrderBy(i => i).ToArray();
var expectKeys = Enumerable.Range(0, addsPerThread);
if (!gotKeys.SequenceEqual(expectKeys))
{
TestHarness.TestLog(" > The set of keys in the dictionary is invalid.");
passed = false;
}
// Finally, let's verify that the count is reported correctly.
int expectedCount = addsPerThread;
int count1 = dict.Count, count2 = dict.ToArray().Length,
count3 = dict.ToList().Count;
if (count1 != expectedCount || count2 != expectedCount || count3 != expectedCount)
{
TestHarness.TestLog(" > Incorrect count of elements reported for the dictionary. Expected {0}, Dict.Count {1}, ToArray.Length {2}, ToList.Count {3}",
expectedCount, count1, count2, count3);
passed = false;
}
return passed;
}
public string[] GetIds() => payments.Select(p => p.Value.Id).ToArray();
/// <nodoc />
public IReadOnlyList <ModuleAndContext> GetVisitedModules()
{
return
(m_visitedModules?.Select(kvp => new ModuleAndContext(kvp.Key, kvp.Value.ContextTree)).ToArray()
?? CollectionUtilities.EmptyArray <ModuleAndContext>());
}
public IEnumerable <T> ReadAll() => elements.Select(e => e.Value);
public HttpResponseMessage IndexReplicate([FromBody] ReplicationDestination replicationDestination)
{
var op = GetQueryStringValue("op");
if (string.Equals(op, "replicate-all", StringComparison.InvariantCultureIgnoreCase))
return ReplicateAllIndexes();
if (string.Equals(op, "replicate-all-to-destination", StringComparison.InvariantCultureIgnoreCase))
return ReplicateAllIndexes(dest => dest.IsEqualTo(replicationDestination));
var indexName = GetQueryStringValue("indexName");
if(indexName == null)
throw new InvalidOperationException("indexName query string must be specified if op=replicate-all or op=replicate-all-to-destination isn't specified");
//check for replication document before doing work on getting index definitions.
//if there is no replication set up --> no point in doing any other work
HttpResponseMessage erroResponseMessage;
var replicationDocument = GetReplicationDocument(out erroResponseMessage);
if (replicationDocument == null)
return erroResponseMessage;
if (indexName.EndsWith("/")) //since id is part of the url, perhaps a trailing forward slash appears there
indexName = indexName.Substring(0, indexName.Length - 1);
indexName = HttpUtility.UrlDecode(indexName);
var indexDefinition = Database.IndexDefinitionStorage.GetIndexDefinition(indexName);
if (indexDefinition == null)
{
return GetMessageWithObject(new
{
Message = string.Format("Index with name: {0} not found. Cannot proceed with replication...", indexName)
}, HttpStatusCode.NotFound);
}
var serializedIndexDefinition = RavenJObject.FromObject(indexDefinition);
var httpRavenRequestFactory = new HttpRavenRequestFactory { RequestTimeoutInMs = Database.Configuration.Replication.ReplicationRequestTimeoutInMilliseconds };
var failedDestinations = new ConcurrentDictionary<string, Exception>();
Parallel.ForEach(replicationDocument.Destinations.Where(dest => dest.Disabled == false && dest.SkipIndexReplication == false),
destination =>
{
try
{
ReplicateIndex(indexName, destination, serializedIndexDefinition, httpRavenRequestFactory);
}
catch (Exception e)
{
failedDestinations.TryAdd(destination.Humane ?? "<null?>", e);
log.WarnException("Could not replicate index " + indexName + " to " + destination.Humane, e);
}
});
return GetMessageWithObject(new
{
SuccessfulReplicationCount = (replicationDocument.Destinations.Count - failedDestinations.Count),
FailedDestinationUrls = failedDestinations.Select(x => new { Server = x.Key, Error = x.Value.ToString() }).ToArray()
});
}
/// <summary>
/// 关闭所有Channel
/// </summary>
public void Shutdown()
{
_channels.Select(q => q.Value).ToList().ForEach(q => q.Channel.ShutdownAsync().Wait());
_channels.Clear();
}
private void MigrateServerSpecificConfigs0_9(IUnitOfWork uow)
{
const string specificConfigsPath = "data/ServerSpecificConfigs.json";
if (!File.Exists(specificConfigsPath))
{
_log.Warn($"No data from {specificConfigsPath} will be migrated.");
return;
}
var configs = new ConcurrentDictionary<ulong, ServerSpecificConfig>();
try
{
configs = JsonConvert
.DeserializeObject<ConcurrentDictionary<ulong, ServerSpecificConfig>>(
File.ReadAllText(specificConfigsPath), new JsonSerializerSettings()
{
Error = (s, e) =>
{
if (e.ErrorContext.Member.ToString() == "GenerateCurrencyChannels")
{
e.ErrorContext.Handled = true;
}
}
});
}
catch (Exception ex)
{
_log.Warn(ex, "ServerSpecificConfig deserialization failed");
return;
}
var i = 0;
var selfAssRoles = new ConcurrentHashSet<SelfAssignedRole>();
configs
.Select(p => new { data = p.Value, gconfig = uow.GuildConfigs.For(p.Key) })
.AsParallel()
.ForAll(config =>
{
try
{
var guildConfig = config.gconfig;
var data = config.data;
guildConfig.AutoAssignRoleId = data.AutoAssignedRole;
guildConfig.DeleteMessageOnCommand = data.AutoDeleteMessagesOnCommand;
guildConfig.DefaultMusicVolume = data.DefaultMusicVolume;
guildConfig.ExclusiveSelfAssignedRoles = data.ExclusiveSelfAssignedRoles;
guildConfig.GenerateCurrencyChannelIds = new HashSet<GCChannelId>(data.GenerateCurrencyChannels.Select(gc => new GCChannelId() { ChannelId = gc.Key }));
selfAssRoles.AddRange(data.ListOfSelfAssignableRoles.Select(r => new SelfAssignedRole() { GuildId = guildConfig.GuildId, RoleId = r }).ToArray());
var logSetting = guildConfig.LogSetting;
guildConfig.LogSetting.IsLogging = data.LogChannel != null;
guildConfig.LogSetting.ChannelId = data.LogChannel ?? 0;
guildConfig.LogSetting.IgnoredChannels = new HashSet<IgnoredLogChannel>(data.LogserverIgnoreChannels.Select(id => new IgnoredLogChannel() { ChannelId = id }));
guildConfig.LogSetting.LogUserPresence = data.LogPresenceChannel != null;
guildConfig.LogSetting.UserPresenceChannelId = data.LogPresenceChannel ?? 0;
guildConfig.FollowedStreams = new HashSet<FollowedStream>(data.ObservingStreams.Select(x =>
{
FollowedStream.FollowedStreamType type = FollowedStream.FollowedStreamType.Twitch;
switch (x.Type)
{
case StreamNotificationConfig0_9.StreamType.Twitch:
type = FollowedStream.FollowedStreamType.Twitch;
break;
case StreamNotificationConfig0_9.StreamType.Beam:
type = FollowedStream.FollowedStreamType.Beam;
break;
case StreamNotificationConfig0_9.StreamType.Hitbox:
type = FollowedStream.FollowedStreamType.Hitbox;
break;
default:
break;
}
return new FollowedStream()
{
ChannelId = x.ChannelId,
GuildId = guildConfig.GuildId,
Username = x.Username.ToLowerInvariant(),
Type = type
};
}));
guildConfig.VoicePlusTextEnabled = data.VoicePlusTextEnabled;
_log.Info("Migrating SpecificConfig for {0} done ({1})", guildConfig.GuildId, ++i);
}
catch (Exception ex)
{
_log.Error(ex);
}
});
uow.SelfAssignedRoles.AddRange(selfAssRoles.ToArray());
try { File.Move("data/ServerSpecificConfigs.json", "data/DELETE_ME_ServerSpecificCOnfigs.json"); } catch { }
}
/// <summary>
/// 获取加载成功的服务列表
/// </summary>
/// <returns></returns>
public override List <string> GetLoadedList()
{
return(container.Select(n => n.Key).ToList());
}
public BermudaResult GetData(string domain, IEnumerable<string> blobs, string query, string mapreduce, string merge, DateTime minDate, DateTime maxDate, int remdepth, object[] parameters, string command)
{
var args = ParseCommand(command);
if (remdepth > 0)
{
//map
var blobInterfaces = blobs == null ? AzureInterface.Instance.ListBlobs(domain, minDate.Ticks, maxDate.Ticks) : AzureInterface.Instance.GetBlobInterfacesByNames(domain, blobs);
var blobSetKey = GetQueryChecksum(domain, string.Join(",", blobInterfaces.Select(x => x.Name)), query, mapreduce, minDate, maxDate, parameters, null);
//reduce
BermudaResult cachedDatapoints;
if (CachedData.TryGetValue(blobSetKey, out cachedDatapoints) && (DateTime.Now.Ticks - cachedDatapoints.CreatedOn) < CacheLifetime)
{
if (CacheTraceMessageLevel < 3) Trace.WriteLine("returned CACHED BLOBS DATAPOINTS results FOR ENTIRE BLOB SET [REMDEPTH:" + remdepth + "]");
return new BermudaResult { DataType = cachedDatapoints.DataType, Data = cachedDatapoints.Data, MetadataObject = new BermudaNodeStatistic { Notes = "Cache_Hit_1" } };
}
else
{
var assignments = PartitionBlobs(domain, blobInterfaces, minDate, maxDate, false, true);
if (!assignments.Any()) throw new Exception("Specified dataset not loaded: " + domain);
ConcurrentDictionary<IPEndPoint, BermudaResult> results = new ConcurrentDictionary<IPEndPoint, BermudaResult>();
Stopwatch sw = new Stopwatch();
sw.Start();
List<Task> tasks = new List<Task>();
foreach (var ass in assignments)
{
Task t = new Task((assObj) =>
{
ZipMetadata assignment = assObj as ZipMetadata;
var initiated = DateTime.Now;
var blobSubsetKey = GetQueryChecksum(domain, string.Join(",", assignment.Blobs.Select(x => x.Name)), query, mapreduce, minDate, maxDate, parameters, assignment.PeerEndpoint.ToString());
Stopwatch sw3 = new Stopwatch();
sw3.Start();
//see if the cache contains a matching result and return it if it's not outdated
BermudaResult cachedDatapoints2;
if (CachedData.TryGetValue(blobSubsetKey, out cachedDatapoints2) && (DateTime.Now.Ticks - cachedDatapoints2.CreatedOn) < CacheLifetime)
{
if (CacheTraceMessageLevel < 2) Trace.WriteLine("returned CACHED BLOB DATAPOINT results FOR BLOB SUBSET [REMDEPTH:" + remdepth + "]");
results[assignment.PeerEndpoint] = new BermudaResult { DataType = cachedDatapoints2.DataType, Data = cachedDatapoints2.Data, MetadataObject = new BermudaNodeStatistic { Notes = "Cache_Hit_2" } };
}
else
{
try
{
Stopwatch sw2 = new Stopwatch();
sw2.Start();
BermudaResult subresult = null;
if (assignment.PeerEndpoint.Equals(Endpoint))
{
subresult = GetData(domain, assignment.Blobs.Select(x => x.Name), query, mapreduce, merge, minDate, maxDate, remdepth - 1, parameters, command);
}
else
{
using (var client = AzureInterface.Instance.GetServiceClient(assignment.PeerEndpoint))
{
subresult = client.GetData(domain, query, mapreduce, merge, minDate, maxDate, remdepth - 1, parameters, command);
}
}
sw2.Stop();
subresult.CreatedOn = DateTime.Now.Ticks;
subresult.MetadataObject.Initiated = initiated;
subresult.MetadataObject.Completed = DateTime.Now;
subresult.MetadataObject.OperationTime = sw2.Elapsed;
results[assignment.PeerEndpoint] = CachedData[blobSubsetKey] = subresult;
}
catch (Exception ex)
{
results[assignment.PeerEndpoint] = new BermudaResult { Error = "[Failed Node] " + ex };
}
}
}, ass, TaskCreationOptions.LongRunning);
tasks.Add(t);
t.Start();
}
Task.WaitAll(tasks.ToArray());
sw.Stop();
Trace.WriteLine("Join Time:" + sw.Elapsed);
if (results.All(x => x.Value.Error != null)) throw new Exception("All nodes failed:\r\n" + string.Join("\r\n", results.Select(x => x.Value.Error)));
//if all results are not the same time throw an error
if (results.GroupBy(x => x.Value.DataType).Count() > 1) throw new Exception("Subresults must all return the same type");
var dataTypeDescriptor = results.Select(x => x.Value.DataType).FirstOrDefault(x => x != null);
if (dataTypeDescriptor == null) return new BermudaResult { Error = "Could not determine the merge type, none of the nodes provided type info" };
//use the passed combine espression to make multiple datapoint sets into one
var dataType = LinqRuntimeTypeBuilder.GetTypeFromTypeKey(dataTypeDescriptor);
//allItems = results.Values.SelectMany(x => x.DataObject)
var totalJson = "[" + string.Join(",", results.Values.Select(x => x.Data.Trim('[', ']'))) + "]";
var allItems = LinqRuntimeTypeBuilder.DeserializeJson(totalJson, dataTypeDescriptor, true);
//var aaa = new JavaScriptSerializer().Deserialize<Datapoint[]>(totalJson);
//var ggc = aaa.GroupBy(x => new { x.Id, x.Id2 }).Count();
//InvokeSelectManyViaReflectionTheKilla(results.Values.Select(x => x.DataObject), dataType);
var mergeFunc = GetMergeFunc(merge, mapreduce, dataType);
if (mergeFunc != null)
{
//var dataType = "kdsajkdsa";
var mergeInvokeMethod = mergeFunc.GetType().GetMethod("Invoke");
allItems = mergeInvokeMethod.Invoke(mergeFunc, new object[] { allItems }); // MergeDatapoints(results.Values.Where(x => x.Data != null).SelectMany(x => x.Data), mergeFunc);
}
//figure out the metadata
var finalMetadata = new BermudaNodeStatistic { Notes = "Merged Datapoints in " + sw.Elapsed, NodeId = AzureInterface.Instance.CurrentInstanceId, ChildNodes = results.Values.Select(x => x.MetadataObject ).ToArray() };
var finalResult = new BermudaResult { DataType = dataTypeDescriptor, DataObject = allItems, CreatedOn = DateTime.Now.Ticks, MetadataObject = finalMetadata };
CachedData[blobSetKey] = finalResult;
return finalResult;
}
}
else
{
ConcurrentDictionary<string, BermudaResult> results = new ConcurrentDictionary<string, BermudaResult>();
BermudaNodeStatistic stats = new BermudaNodeStatistic();
var blobInterfaces = AzureInterface.Instance.GetBlobInterfacesByNames(domain, blobs);
var blobSetKey = GetQueryChecksum(domain, string.Join(",", blobInterfaces.Select(x => x.Name)), query, mapreduce, minDate, maxDate, parameters, Endpoint.ToString());
BermudaResult cachedDatapoints;
if (CachedData.TryGetValue(blobSetKey, out cachedDatapoints) && (DateTime.Now.Ticks - cachedDatapoints.CreatedOn) < CacheLifetime)
{
if (CacheTraceMessageLevel < 2) Trace.WriteLine("returned CACHED BLOB SET DATAPOINT results [REMDEPTH:" + remdepth + "]");
return new BermudaResult { DataType = cachedDatapoints.DataType, Data = cachedDatapoints.Data, MetadataObject = new BermudaNodeStatistic { Notes = "Cache_Hit_3" } };
}
else
{
//Chad: short circuiting to test WCF response time in Azure
//return new DatapointResult() { Datapoints = new List<Datapoint>(), CreatedOn = DateTime.Now.Ticks, Metadata = new BermudaNodeStatistic() };
//IEnumerable<Datapoint> datapoints = null;
object datapoints = null;
Stopwatch sw = new Stopwatch();
sw.Start();
Type itemType = null;
Type resultType = null;
foreach (var blobInterface in blobInterfaces)
{
var blobKey = GetQueryChecksum(domain, blobInterface.Name, query, mapreduce, minDate, maxDate, parameters, Endpoint.ToString());
//see if the cache contains a matching result and return it if it's not outdated
BermudaResult cachedDatapoints2;
if (CachedData.TryGetValue(blobKey, out cachedDatapoints2) && (DateTime.Now.Ticks - cachedDatapoints2.CreatedOn) < CacheLifetime)
{
if (CacheTraceMessageLevel < 1) Trace.WriteLine("returned CACHED BLOB DATAPOINT results [REMDEPTH:" + remdepth + "]");
results[blobInterface.Name] = new BermudaResult { DataType = cachedDatapoints2.DataType, Data = cachedDatapoints2.Data, MetadataObject = new BermudaNodeStatistic { Notes = "Cache_Hit_4" } };
datapoints = cachedDatapoints2.DataObject;
}
else
{
//get mentions
var raw = blobInterface.GetData();
var rawType = raw.GetType();
itemType = ReduceExpressionGeneration.GetTypeOfEnumerable(rawType);
var mapreduceFunc = GetMapReduceFunc(mapreduce, itemType, out resultType);
var queryFunc = GetFilterFunc(query, itemType);
var minDateTicks = minDate.Ticks;
var maxDateTicks = maxDate.Ticks;
object subresult = raw.AsParallel();
//queryFunc == null ?
// raw.AsParallel() :
//minDate == DateTime.MinValue && maxDate == DateTime.MaxValue ?
// raw.AsParallel().Where(x => queryFunc) :
// raw.AsParallel().Where(x => x.OccurredOnTicks >= minDateTicks && x.OccurredOnTicks <= maxDateTicks && queryFunc(x, parameters));
if (queryFunc != null)
{
var queryFuncInvoke = queryFunc.GetType().GetMethod("Invoke");
subresult = queryFuncInvoke.Invoke(queryFunc, new object[] { subresult });
}
//reduce them using the passed expression
if (mapreduceFunc != null)
{
var mapReduceFuncInvoke = mapreduceFunc.GetType().GetMethod("Invoke");
subresult = mapReduceFuncInvoke.Invoke(mapreduceFunc, new object[] { subresult });
}
datapoints = subresult;
//format a metada string
if (!args.Contains("-nocount"))
{
//stats.TotalItems = raw.Count();
//stats.FilteredItems = filtered.Count();
//stats.ReducedItems = subresult.Count();
}
//cache the result
//results[blobInterface.Name] = new DatapointResult { Datapoints = subresult, CreatedOn = DateTime.UtcNow.Ticks, Metadata = stats.Serialize() };
//CachedDatapoints[blobKey] = new DatapointResult { Datapoints = subresult.ToList(), CreatedOn = DateTime.UtcNow.Ticks, Metadata = stats.Serialize() };
}
}
//figure out the metadata
//var finalMetadata = " [@" + AzureInterface.Instance.CurrentInstanceId + "] Calculated Datapoints:\r\n" + string.Join("\r\n", results.Values.Select(x => x.Metadata));
stats.NodeId = AzureInterface.Instance.CurrentInstanceId;
stats.Notes = "Computed Datapoints";
//Trace.WriteLine("total mentions processed: " + mentionCount);
//var datapoints = results.Values.SelectMany(x => x.Datapoints);
if (datapoints == null) return new BermudaResult() { MetadataObject = new BermudaNodeStatistic { Notes = "No Results" } };
//foreach (var p in datapoints) if (p.IsCount) p.Value = p.Count;
var mergeFunc = resultType == null ? null : GetMergeFunc(merge, mapreduce, resultType);
if (mergeFunc != null)
{
var mergeFuncInvoke = mergeFunc.GetType().GetMethod("Invoke");
datapoints = mergeFuncInvoke.Invoke(mergeFunc, new object[] { datapoints });
}
sw.Stop();
stats.LinqExecutionTime = sw.Elapsed;
var result = CachedData[blobSetKey] = new BermudaResult { DataType = LinqRuntimeTypeBuilder.GetTypeKey(resultType), DataObject = datapoints, CreatedOn = DateTime.Now.Ticks, MetadataObject = stats };
return result;
}
}
}
private async void Process(List <PackageVersionPair> packageVersions, string pathToSolution, bool isIncremental = false, bool incrementalRefresh = false)
{
if (!packageVersions.Any())
{
_logger.LogInformation("No package version compatibilities to process.");
return;
}
var packageVersionsGroupedByPackageIdDict = packageVersions.ToDictionary(t => t.ToString(), t => t);
ConcurrentDictionary <string, PackageVersionPair> packageVersionsGroupedByPackageIdConcurrent = new ConcurrentDictionary <string, PackageVersionPair>(packageVersionsGroupedByPackageIdDict);
var distinctPackageVersions = packageVersions.Distinct().ToList();
var exceptions = new ConcurrentDictionary <PackageVersionPair, Exception>();
foreach (var compatibilityChecker in _compatibilityCheckers)
{
try
{
var compatibilityResults = compatibilityChecker.Check(distinctPackageVersions, pathToSolution, isIncremental, incrementalRefresh);
await Task.WhenAll(compatibilityResults.Select(result =>
{
return(result.Value.ContinueWith(task =>
{
if (task.IsCompletedSuccessfully)
{
packageVersionsGroupedByPackageIdConcurrent.TryRemove(result.Key.ToString(), out _);
if (_compatibilityTaskCompletionSources.TryGetValue(result.Key, out var packageVersionPairResult))
{
packageVersionPairResult.TrySetResult(task.Result);
}
else
{
throw new ArgumentNullException($"Package version {result.Key} not found in compatibility tasks.");
}
}
else
{
exceptions.TryAdd(result.Key, task.Exception);
}
}));
}).ToList());
}
catch (Exception ex)
{
_logger.LogError("Package compatibility processing failed with error: {0}", ex);
}
}
foreach (var packageVersion in packageVersionsGroupedByPackageIdConcurrent.Select(packageVersionGroup => packageVersionGroup.Value))
{
if (packageVersion != null && _compatibilityTaskCompletionSources.TryGetValue(packageVersion, out var packageVersionPairResult))
{
_logger.LogError($"Cound not find package {packageVersion} in all sources");
var defaultErrorMessage = $"Could not find package {packageVersion}. Compatibility task status: {packageVersionPairResult.Task.Status}.";
var defaultException = new PortingAssistantClientException(ExceptionMessage.PackageNotFound(packageVersion), new PackageNotFoundException(defaultErrorMessage));
var exception = exceptions.GetValueOrDefault(packageVersion, defaultException);
packageVersionPairResult.TrySetException(exception);
}
else
{
_logger.LogInformation($"Attempted to get package {packageVersion} from compatibility tasks but it was not found.");
}
}
}
private IEnumerable <AppliedItem> GetAppliedItems()
{
return(InventoryItems.Select(i => i.Value.InventoryItemData?.AppliedItems)
.Where(aItems => aItems?.Item != null)
.SelectMany(aItems => aItems.Item));
}
IEnumerator <KeyValuePair <TKey, TValue> > IEnumerable <KeyValuePair <TKey, TValue> > .GetEnumerator() => _map .Select(v => new KeyValuePair <TKey, TValue>(v.Key, v.Value.Value)) .GetEnumerator();
public BermudaResult GetData(string domain, string query, string mapreduce, string merge, string paging, int remdepth, string command, string cursor, string paging2)
{
var args = ParseCommand(command);
bool noCache = args.Contains("-nocache");
bool makeCursor = cursor == MakeCursorToken;
bool useCursor = !makeCursor && !string.IsNullOrWhiteSpace(cursor);
DateTime minDate = DateTime.MinValue;
DateTime maxDate = DateTime.MaxValue;
if (remdepth > 0)
{
//map
var queryHash = cursor ?? GetQueryHash(domain, query, mapreduce, merge, paging, null);
//reduce
BermudaResult cachedDatapoints;
if (!noCache && CachedData.TryGetValue(queryHash, out cachedDatapoints) && (DateTime.Now.Ticks - cachedDatapoints.CreatedOn) < CacheLifetime)
{
#if DEBUG
if (CacheTraceMessageLevel < 3) Trace.WriteLine("returned CACHED BLOBS DATAPOINTS results FOR ENTIRE BLOB SET [REMDEPTH:" + remdepth + "]");
#endif
if (useCursor)
{
var dataType = LinqRuntimeTypeBuilder.GetTypeFromTypeKey(cachedDatapoints.DataType);
return GetCursorData(paging2, cachedDatapoints, dataType);
}
else
{
return new BermudaResult { DataType = cachedDatapoints.DataType, Data = cachedDatapoints.Data, Metadata = new BermudaNodeStatistic { Notes = "Cache_Hit_1" }, CacheKey = cachedDatapoints.CacheKey };
}
}
else
{
if (useCursor) throw new Exception("Cursor " + cursor + " not found");
//var assignments = PartitionBlobs(domain, blobInterfaces, minDate, maxDate, false, true);
var reducers = HostEnvironment.Instance.GetAvailablePeerConnections();
if (!reducers.Any()) throw new Exception("Specified dataset not loaded: " + domain);
ConcurrentDictionary<PeerInfo, BermudaResult> results = new ConcurrentDictionary<PeerInfo, BermudaResult>();
Stopwatch sw = new Stopwatch();
sw.Start();
List<Task> tasks = new List<Task>();
foreach (var reducer in reducers)
{
Task t = new Task((peerObj) =>
{
var peerInfo = peerObj as PeerInfo;
var initiated = DateTime.Now;
var subqueryHash = GetQueryHash(domain, query, mapreduce, merge, paging, peerInfo.ToString());
Stopwatch sw3 = new Stopwatch();
sw3.Start();
//see if the cache contains a matching result and return it if it's not outdated
BermudaResult cachedDatapoints2;
if (!noCache && CachedData.TryGetValue(subqueryHash, out cachedDatapoints2) && (DateTime.Now.Ticks - cachedDatapoints2.CreatedOn) < CacheLifetime)
{
if (CacheTraceMessageLevel < 2) Trace.WriteLine("returned CACHED BLOB DATAPOINT results FOR BLOB SUBSET [REMDEPTH:" + remdepth + "]");
BermudaResult res = null;
if (useCursor)
{
var dataType2 = LinqRuntimeTypeBuilder.GetTypeFromTypeKey(cachedDatapoints2.DataType);
res = GetCursorData(paging2, cachedDatapoints2, dataType2);
}
else
{
res = new BermudaResult { DataType = cachedDatapoints2.DataType, Data = cachedDatapoints2.Data, Metadata = new BermudaNodeStatistic { Notes = "Cache_Hit_2" } };
}
results[peerInfo] = res;
}
else
{
try
{
Stopwatch sw2 = new Stopwatch();
sw2.Start();
BermudaResult subresult = null;
if (peerInfo.Equals(Endpoint))
{
subresult = GetData(domain, query, mapreduce, merge, paging, remdepth - 1, command, cursor, paging2);
}
else
{
using (var client = HostEnvironment.GetServiceClient(peerInfo))
{
subresult = client.GetData(domain, query, mapreduce, merge, paging, remdepth - 1, command, cursor, paging2);
}
//subresult = GetDataFromPeer(domain, query, mapreduce, merge, minDate, maxDate, remdepth - 1, command, assignment.PeerEndpoint.Endpoint);
}
sw2.Stop();
subresult.CreatedOn = DateTime.Now.Ticks;
subresult.Metadata.Initiated = initiated;
subresult.Metadata.Completed = DateTime.Now;
subresult.Metadata.OperationTime = sw2.Elapsed;
results[peerInfo] = CachedData[subqueryHash] = subresult;
}
catch (Exception ex)
{
results[peerInfo] = new BermudaResult { Error = "[Failed Node] " + ex };
}
}
}, reducer, TaskCreationOptions.LongRunning);
tasks.Add(t);
t.Start();
}
Task.WaitAll(tasks.ToArray());
sw.Stop();
#if DEBUG
Trace.WriteLine("Join Time:" + sw.Elapsed);
#endif
if (results.Any(x => x.Value.Error != null)) throw new BermudaException("Some nodes failed:\r\n" + string.Join("\r\n", results.Select(x => x.Value.Error)));
if (results.All(x => x.Value.Data == null)) return new BermudaResult { Metadata = new BermudaNodeStatistic { Notes = "No Data" } };
//if all results are not the same time throw an error
if (results.GroupBy(x => x.Value.DataType).Count() > 1) throw new BermudaException("Subresults must all return the same type");
var dataTypeDescriptor = results.Select(x => x.Value.DataType).FirstOrDefault(x => x != null);
if (dataTypeDescriptor == null) return new BermudaResult { Error = "Could not determine the merge type, none of the nodes provided type info" };
//use the passed combine espression to make multiple datapoint sets into one
var dataType = LinqRuntimeTypeBuilder.GetTypeFromTypeKey(dataTypeDescriptor);
//allItems = results.Values.SelectMany(x => x.DataObject)
var totalJson = "[" + string.Join(",", results.Values.Where(x => !string.IsNullOrWhiteSpace(x.Data)).Select(x => x.Data.Trim('[', ']')).Where(x => !string.IsNullOrWhiteSpace(x))) + "]";
var allItems = LinqRuntimeTypeBuilder.DeserializeJson(totalJson, dataTypeDescriptor, true);
//var aaa = new JavaScriptSerializer().Deserialize<Datapoint[]>(totalJson);
//var ggc = aaa.GroupBy(x => new { x.Id, x.Id2 }).Count();
//InvokeSelectManyViaReflectionTheKilla(results.Values.Select(x => x.DataObject), dataType);
var mergeFunc = GetMergeFunc(merge, mapreduce, dataType, dataType);
if (mergeFunc != null)
{
//var dataType = "kdsajkdsa";
var mergeInvokeMethod = mergeFunc.GetType().GetMethod("Invoke");
allItems = mergeInvokeMethod.Invoke(mergeFunc, new object[] { allItems }); // MergeDatapoints(results.Values.Where(x => x.Data != null).SelectMany(x => x.Data), mergeFunc);
}
var pagingFunc = GetPagingFunc(paging, dataType);
if (pagingFunc != null)
{
var pagingInvokeMethod = pagingFunc.GetType().GetMethod("Invoke");
allItems = pagingInvokeMethod.Invoke(pagingFunc, new object[] { allItems });
}
//figure out the metadata
var finalMetadata = new BermudaNodeStatistic { Notes = "Merged Datapoints in " + sw.Elapsed, NodeId = HostEnvironment.Instance.CurrentInstanceId, ChildNodes = results.Values.Select(x => x.Metadata).ToArray() };
var arraylol = ToArrayCollection(allItems, dataType);
var json = JsonConvert.SerializeObject(arraylol);
//var json = JsonConvert.SerializeObject(allItems);
var originalData = makeCursor ? arraylol : null;
var finalResult = new BermudaResult { DataType = dataTypeDescriptor, OriginalData = originalData, Data = json, CreatedOn = DateTime.Now.Ticks, Metadata = finalMetadata, CacheKey = queryHash };
CachedData[queryHash] = finalResult;
return finalResult;
}
}
else
{
ConcurrentDictionary<string, BermudaResult> results = new ConcurrentDictionary<string, BermudaResult>();
BermudaNodeStatistic stats = new BermudaNodeStatistic();
var bucketInterfaces = HostEnvironment.Instance.GetBucketInterfacesForDomain(domain);
if (!bucketInterfaces.Any()) throw new BermudaException("Data not loaded for: " + domain);
if (bucketInterfaces.Count() > 1) throw new BermudaException("Multiple buckets not supported by BermudaMapReduce");
var queryHash = GetQueryHash(domain, query, mapreduce, merge, paging, Endpoint.ToString());
BermudaResult cachedDatapoints;
if (!noCache && CachedData.TryGetValue(queryHash, out cachedDatapoints) && (DateTime.Now.Ticks - cachedDatapoints.CreatedOn) < CacheLifetime)
{
if (CacheTraceMessageLevel < 2) Trace.WriteLine("returned CACHED BLOB SET DATAPOINT results [REMDEPTH:" + remdepth + "]");
if (useCursor)
{
var dataType = LinqRuntimeTypeBuilder.GetTypeFromTypeKey(cachedDatapoints.DataType);
return GetCursorData(paging2, cachedDatapoints, dataType);
}
else
{
return new BermudaResult { DataType = cachedDatapoints.DataType, Data = cachedDatapoints.Data, Metadata = new BermudaNodeStatistic { Notes = "Cache_Hit_3" }, CacheKey = queryHash };
}
}
else
{
//Chad: short circuiting to test WCF response time in Azure
//return new DatapointResult() { Datapoints = new List<Datapoint>(), CreatedOn = DateTime.Now.Ticks, Metadata = new BermudaNodeStatistic() };
//IEnumerable<Datapoint> datapoints = null;
object datapoints = null;
Stopwatch sw = new Stopwatch();
sw.Start();
Type itemType = null;
Type resultType = null;
string json = null;
foreach (var bucketInterface in bucketInterfaces)
{
var bucketKey = GetQueryHash(domain, query, mapreduce, merge, paging, Endpoint.ToString());
//see if the cache contains a matching result and return it if it's not outdated
BermudaResult cachedDatapoints2;
if (!noCache && CachedData.TryGetValue(bucketKey, out cachedDatapoints2) && (DateTime.Now.Ticks - cachedDatapoints2.CreatedOn) < CacheLifetime)
{
if (CacheTraceMessageLevel < 1) Trace.WriteLine("returned CACHED BLOB DATAPOINT results [REMDEPTH:" + remdepth + "]");
if (useCursor)
{
if (cachedDatapoints2.OriginalData == null) throw new Exception("Cursor " + cursor + " contains null data");
var dataType = LinqRuntimeTypeBuilder.GetTypeFromTypeKey(cachedDatapoints2.DataType);
results[bucketInterface.Name] = GetCursorData(paging2, cachedDatapoints2, dataType);
}
else
{
results[bucketInterface.Name] = new BermudaResult { DataType = cachedDatapoints2.DataType, Data = cachedDatapoints2.Data, Metadata = new BermudaNodeStatistic { Notes = "Cache_Hit_4" } };
json = cachedDatapoints2.Data;
}
}
else
{
//get mentions
var collections = GetCollections(query, mapreduce);
if (collections.Count() > 1) throw new BermudaException("More than one collection specified: " + string.Join(",", collections));
var table = collections.FirstOrDefault();
var tableName = table == null ? null : table.Source;
var raw = bucketInterface.GetData(tableName);
//var rawType = raw.GetType();
//itemType = ReduceExpressionGeneration.GetTypeOfEnumerable(rawType);
itemType = bucketInterface.GetDataType(tableName);
var mapreduceFunc = GetMapReduceFunc(mapreduce, itemType, out resultType);
var queryFunc = GetFilterFunc(query, itemType);
var pagingFunc = GetPagingFunc(paging, resultType);
var minDateTicks = minDate.Ticks;
var maxDateTicks = maxDate.Ticks;
object subresult = raw;
//queryFunc == null ?
// raw.AsParallel() :
//minDate == DateTime.MinValue && maxDate == DateTime.MaxValue ?
// raw.AsParallel().Where(x => queryFunc) :
// raw.AsParallel().Where(x => x.OccurredOnTicks >= minDateTicks && x.OccurredOnTicks <= maxDateTicks && queryFunc(x, parameters));
if (json == null)
{
if (queryFunc != null)
{
var queryFuncInvoke = queryFunc.GetType().GetMethod("Invoke");
subresult = queryFuncInvoke.Invoke(queryFunc, new object[] { subresult });
}
//reduce them using the passed expression
if (mapreduceFunc != null)
{
var mapReduceFuncInvoke = mapreduceFunc.GetType().GetMethod("Invoke");
subresult = mapReduceFuncInvoke.Invoke(mapreduceFunc, new object[] { subresult });
}
if (pagingFunc != null)
{
var pagingInvokeMethod = pagingFunc.GetType().GetMethod("Invoke");
subresult = pagingInvokeMethod.Invoke(pagingFunc, new object[] { subresult });
}
datapoints = subresult;
}
//format a metada string
if (!args.Contains("-nocount"))
{
stats.TotalItems = bucketInterface.GetCount(tableName);
//stats.FilteredItems = filtered.Count();
//stats.ReducedItems = subresult.Count();
}
//cache the result
//results[blobInterface.Name] = new DatapointResult { Datapoints = subresult, CreatedOn = DateTime.UtcNow.Ticks, Metadata = stats.Serialize() };
//CachedDatapoints[blobKey] = new DatapointResult { Datapoints = subresult.ToList(), CreatedOn = DateTime.UtcNow.Ticks, Metadata = stats.Serialize() };
}
}
//figure out the metadata
//var finalMetadata = " [@" + AzureInterface.Instance.CurrentInstanceId + "] Calculated Datapoints:\r\n" + string.Join("\r\n", results.Values.Select(x => x.Metadata));
stats.NodeId = HostEnvironment.Instance.CurrentInstanceId;
stats.Notes = "Computed Datapoints";
//Trace.WriteLine("total mentions processed: " + mentionCount);
//var datapoints = results.Values.SelectMany(x => x.Datapoints);
if (datapoints == null) return new BermudaResult() { Metadata = new BermudaNodeStatistic { Notes = "No Results" } };
//foreach (var p in datapoints) if (p.IsCount) p.Value = p.Count;
var mergeFunc = GetMergeFunc(merge, mapreduce, itemType, resultType);
if (mergeFunc != null)
{
var mergeFuncInvoke = mergeFunc.GetType().GetMethod("Invoke");
datapoints = mergeFuncInvoke.Invoke(mergeFunc, new object[] { datapoints });
}
stats.LinqExecutionTime = sw.Elapsed;
var arraylol = ToArrayCollection(datapoints, resultType);
if (json == null && datapoints != null)
{
json = JsonConvert.SerializeObject(arraylol);
}
//var json = JsonConvert.SerializeObject(datapoints);
var originalData = makeCursor ? arraylol : null;
var result = CachedData[queryHash] = new BermudaResult { DataType = LinqRuntimeTypeBuilder.GetTypeKey(resultType), OriginalData = originalData, Data = json, CreatedOn = DateTime.Now.Ticks, Metadata = stats };
sw.Stop();
return result;
}
}
}
public IList <Room> GetAllRooms()
{
return(_rooms.Select(p => p.Value).ToList());
}
public static List <TKey> GetKeys <TKey, TValue>(this ConcurrentDictionary <TKey, TValue> @this)
{
return(@this?.Select(i => i.Key).ToList());
}
public IReadOnlyList <int> GetSessionIds()
{
return(_sessionHandles.Select(h => h.Key).ToList());
}
public async Task <IEnumerable <TTenantInfo> > GetAllAsync()
{
return(await Task.FromResult(tenantMap?.Select(x => x.Value).ToList() ?? new List <TTenantInfo>()));
}
private void SaveTeachedValue()
{
try
{
ConcurrentDictionary<string, string> capthaList = new ConcurrentDictionary<string, string>(
File.ReadLines(labelTeachedFile.Text)
.Select(line => line.Split(','))
.ToDictionary(line => line[0], line => line[1]));
if (!String.IsNullOrWhiteSpace(textBoxAllDigits.Text) && !String.IsNullOrWhiteSpace(imagePath.Text))
capthaList.AddOrUpdate(imagePath.Text, textBoxAllDigits.Text);
File.WriteAllText(labelTeachedFile.Text,
String.Join(Environment.NewLine, capthaList.Select(d => d.Key + "," + d.Value).OrderBy(key => key)));
}
catch (Exception ex)
{
MessageBox.Show(ex.Message);
}
}
