public override bool OnNext()
{
bool compressData = compressCheckBox.IsChecked == true;
string path;
if (!AskForFolder(out path))
{
return(false);
}
if (compressData)
{
path += ".zip";
}
Storage storage = StorageFactory.Create(path, compressData);
OutputType outputType = controlToOutputTypeCache.First(
entry => entry.Key.IsChecked == true).Value;
Context.OutputWriter = OutputWriterFactory.Create(
outputType, storage);
Context.GroupingStrategy = CreateGroupingStrategy();
Context.UseNicknames = useNicknamesCheckBox.IsChecked == true;
Context.UseDateFilter = dateFilterCheckBox.IsChecked == true;
if (Context.UseDateFilter)
{
if (!SetDateFilter())
{
Logger.Info("Invalid date filter: filter1 is {0}, filter2 is {1}",
dateFilter1.SelectedDate, dateFilter2.SelectedDate);
return(false);
}
}
return(true);
}
protected override async Task <bool> OnProcessBatch(CollectorHttpClient client, IEnumerable <JToken> items, JToken context, DateTime commitTimeStamp)
{
foreach (JToken item in items)
{
string id = item["nuget:id"].ToString().ToLowerInvariant();
string version = item["nuget:version"].ToString().ToLowerInvariant();
Storage storage = _storageFactory.Create(id);
string nuspec = await LoadNuspec(id, version);
if (nuspec != null)
{
await SaveNuspec(storage, id, version, nuspec);
await CopyNupkg(storage, id, version);
await UpdateMetadata(storage, version);
Trace.TraceInformation("commit: {0}/{1}", id, version);
}
else
{
Trace.TraceInformation("no nuspec available for {0}/{1} skipping", id, version);
}
}
return(true);
}
public static async Task ProcessGraphs(
string id,
IDictionary <string, IGraph> sortedGraphs,
StorageFactory storageFactory,
Uri contentBaseAddress,
int partitionSize,
int packageCountThreshold)
{
int versionAlreadyExistsCount = 0;
existingVersionsWithID = new List <string>();
try
{
Storage storage = storageFactory.Create(id.ToLowerInvariant());
Uri resourceUri = storage.ResolveUri("index.json");
string json = await storage.LoadString(resourceUri);
int count = Utils.CountItems(json);
//Determine if there are any versions that are existing already
CollectorHttpClient httpClient = new CollectorHttpClient();
foreach (var graph in sortedGraphs)
{
JObject jsonContent = await httpClient.GetJObjectAsync(new Uri(graph.Key));
string existingId = jsonContent["@id"].ToString();
string existingVersionWithId = existingId.Substring(existingId.LastIndexOf("/") + 1);
string existingVersion = jsonContent["version"].ToString() + ".json";
//Determine if the version is actually available
//In Registration blobs, the format is /packageID/packageVersion.json
//So to check the existence of version we need to know only the version.json
if (storage.Exists(existingVersion))
{
//When we compare later in AddExistingItems, we need the "packageId.packageversion.json" for comparison so store it with Id
existingVersionsWithID.Add(existingVersionWithId);
versionAlreadyExistsCount++;
}
}
int total = count + sortedGraphs.Count - versionAlreadyExistsCount;
if (total < packageCountThreshold)
{
await SaveSmallRegistration(storage, storageFactory.BaseAddress, sortedGraphs, contentBaseAddress, partitionSize);
}
else
{
await SaveLargeRegistration(storage, storageFactory.BaseAddress, sortedGraphs, json, contentBaseAddress, partitionSize);
}
}
catch (Exception e)
{
throw new Exception(string.Format("Process id = {0}", id), e);
}
}
public RegistrationPersistence(StorageFactory storageFactory, RegistrationKey registrationKey, int partitionSize, int packageCountThreshold)
{
_storage = new RecordingStorage(storageFactory.Create(registrationKey.ToString()));
_registrationUri = _storage.ResolveUri("index.json");
_packageCountThreshold = packageCountThreshold;
_partitionSize = partitionSize;
_registrationBaseAddress = storageFactory.BaseAddress;
}
protected void Application_Start()
{
AreaRegistration.RegisterAllAreas();
FilterConfig.RegisterGlobalFilters(GlobalFilters.Filters);
RouteConfig.RegisterRoutes(RouteTable.Routes);
BundleConfig.RegisterBundles(BundleTable.Bundles);
StorageFactory.Create(RoleEnvironment.GetConfigurationSettingValue("StorageConnectionString"));
}
public static void CreateNewCursor(string[] args)
{
IDictionary <string, string> arguments = GetArguments(args, 0);
StorageFactory storageFactory = CreateStorageFactory(arguments, verbose: true);
Storage storage = storageFactory.Create();
DurableCursor cursor = new DurableCursor(storage.ResolveUri("cursor.json"), storage, GetDefaultValue(arguments));
cursor.Load().Wait();
cursor.Save().Wait();
}
/// <summary>
/// Fetches <see cref="DeletionAuditEntry"/>s.
/// </summary>
/// <param name="auditingStorageFactory">The <see cref="StorageFactory"/> to fetch audit records from.</param>
/// <param name="cancellationToken">A <see cref="CancellationToken"/> that can be used to cancel the task.</param>
/// <param name="package">If specified, will only fetch <see cref="DeletionAuditEntry"/>s that represent operations on this package.</param>
/// <param name="minTime">If specified, will only fetch <see cref="DeletionAuditEntry"/>s that are newer than this <see cref="DateTime"/> (non-inclusive).</param>
/// <param name="maxTime">If specified, will only fetch <see cref="DeletionAuditEntry"/>s that are older than this <see cref="DateTime"/> (non-inclusive).</param>
/// <param name="logger">An <see cref="ILogger"/> to log messages to.</param>
/// <returns>An <see cref="IEnumerable{DeletionAuditEntry}"/> containing the relevant <see cref="DeletionAuditEntry"/>s.</returns>
public static Task <IEnumerable <DeletionAuditEntry> > GetAsync(
StorageFactory auditingStorageFactory,
CancellationToken cancellationToken,
PackageIdentity package = null,
DateTime?minTime = null,
DateTime?maxTime = null,
ILogger logger = null)
{
Storage storage = auditingStorageFactory.Create(package != null ? GetAuditRecordPrefixFromPackage(package) : null);
return(GetAsync(storage, cancellationToken, minTime, maxTime, logger));
}
protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken)
{
var source = arguments.GetOrThrow <string>(Arguments.Source);
var verbose = arguments.GetOrDefault(Arguments.Verbose, false);
var contentBaseAddress = arguments.GetOrDefault <string>(Arguments.ContentBaseAddress);
var storageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose);
var httpClientTimeoutInSeconds = arguments.GetOrDefault <int?>(Arguments.HttpClientTimeoutInSeconds);
var httpClientTimeout = httpClientTimeoutInSeconds.HasValue ? (TimeSpan?)TimeSpan.FromSeconds(httpClientTimeoutInSeconds.Value) : null;
StorageFactory preferredPackageSourceStorageFactory = null;
IAzureStorage preferredPackageSourceStorage = null;
var preferAlternatePackageSourceStorage = arguments.GetOrDefault(Arguments.PreferAlternatePackageSourceStorage, defaultValue: false);
if (preferAlternatePackageSourceStorage)
{
preferredPackageSourceStorageFactory = CommandHelpers.CreateSuffixedStorageFactory("PreferredPackageSourceStorage", arguments, verbose);
preferredPackageSourceStorage = preferredPackageSourceStorageFactory.Create() as IAzureStorage;
}
Logger.LogInformation("CONFIG source: \"{ConfigSource}\" storage: \"{Storage}\" preferred package source storage: \"{PreferredPackageSourceStorage}\"",
source,
storageFactory,
preferredPackageSourceStorageFactory);
Logger.LogInformation("HTTP client timeout: {Timeout}", httpClientTimeout);
MaxDegreeOfParallelism = 256;
_collector = new DnxCatalogCollector(
new Uri(source),
storageFactory,
preferredPackageSourceStorage,
contentBaseAddress == null ? null : new Uri(contentBaseAddress),
TelemetryService,
Logger,
MaxDegreeOfParallelism,
httpClient => new CatalogClient(new SimpleHttpClient(httpClient, LoggerFactory.CreateLogger <SimpleHttpClient>()), LoggerFactory.CreateLogger <CatalogClient>()),
CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose),
httpClientTimeout);
var storage = storageFactory.Create();
_front = new DurableCursor(storage.ResolveUri("cursor.json"), storage, MemoryCursor.MinValue);
_back = MemoryCursor.CreateMax();
_destination = storageFactory.BaseAddress;
TelemetryService.GlobalDimensions[TelemetryConstants.Destination] = _destination.AbsoluteUri;
}
public void Initialize()
{
this.Storage = StorageFactory <T, Cell <T> > .Create(this.Key, this.Config.Storage);
this.NameSpace = Storage.NameSpace;
this.DataSources = new DataSourceCollection(this.Config);
this.Cells = new CellCollection <T>(this.Storage);
this.Schema = new DataSchema <T>(
this.Config,
this.DataSources,
this.Storage.Dimensions,
this.Storage.Measures,
this.Storage.Metrics);
_rowHelper = new DataRowHelper <T>(this.Schema, this.Config.Source);
Initialized = true;
}
private async Task InitializeConfigurationAsync(string repository, CancellationToken token)
{
var storage = StorageFactory.Create(repository);
Container.RegisterInstance(storage);
IConfigurationRoot configuration;
using (var settings = await storage.GetOrCreateAppSettingsAsync(token))
{
configuration = new ConfigurationBuilder()
.AddJsonStream(settings)
.AddUserSecrets(UserSecretsId)
.AddEnvironmentVariables(prefix: EnvironmentVariablePrefix)
.Build();
}
Container.RegisterInstance <IConfigurationManager>(new ConfigurationManager(configuration));
}
/// <summary>
/// 语音传输服务器
/// </summary>
/// <param name="endPoint"></param>
public TransferServer(IPEndPoint endPoint)
{
_cache = new ConcurrentDictionary <string, IUserToken>();
_invitedMapping = new ConcurrentDictionary <string, string>();
_udpServer = SocketFactory.CreateServerSocket(SocketOptionBuilder.Instance.SetSocket(SAEASocketType.Udp)
.SetIPEndPoint(endPoint)
.UseIocp()
.SetReadBufferSize(SocketOption.UDPMaxLength)
.SetWriteBufferSize(SocketOption.UDPMaxLength)
.SetTimeOut(60 * 1000)
.Build());
_udpServer.OnAccepted += _udpServer_OnAccepted;
_udpServer.OnDisconnected += _udpServer_OnDisconnected;
_udpServer.OnReceive += _udpServer_OnReceive;
_storage = StorageFactory.Create <T>();
}
public RegistrationPersistence(
StorageFactory storageFactory,
RegistrationMakerCatalogItem.PostProcessGraph postProcessGraph,
RegistrationKey registrationKey,
int partitionSize,
int packageCountThreshold,
Uri contentBaseAddress,
Uri galleryBaseAddress,
bool forcePackagePathProviderForIcons)
{
_storage = new RecordingStorage(storageFactory.Create(registrationKey.ToString()));
_postProcessGraph = postProcessGraph;
_registrationUri = _storage.ResolveUri("index.json");
_packageCountThreshold = packageCountThreshold;
_partitionSize = partitionSize;
_registrationBaseAddress = storageFactory.BaseAddress;
_contentBaseAddress = contentBaseAddress;
_galleryBaseAddress = galleryBaseAddress;
_forcePackagePathProviderForIcons = forcePackagePathProviderForIcons;
}
protected override async Task <bool> OnProcessBatch(CollectorHttpClient client, IEnumerable <JToken> items, JToken context, DateTime commitTimeStamp, CancellationToken cancellationToken)
{
foreach (JToken item in items)
{
string id = item["nuget:id"].ToString().ToLowerInvariant();
string version = item["nuget:version"].ToString().ToLowerInvariant();
string type = item["@type"].ToString().Replace("nuget:", Schema.Prefixes.NuGet);
Storage storage = _storageFactory.Create(id);
if (type == Schema.DataTypes.PackageDetails.ToString())
{
// Add/update package
string nuspec = await LoadNuspec(client, id, version, cancellationToken);
if (nuspec != null)
{
await SaveNuspec(storage, id, version, nuspec, cancellationToken);
await CopyNupkg(client, storage, id, version, cancellationToken);
await UpdateMetadata(storage, versions => versions.Add(NuGetVersion.Parse(version)), cancellationToken);
Trace.TraceInformation("commit: {0}/{1}", id, version);
}
else
{
Trace.TraceWarning("no nuspec available for {0}/{1} skipping", id, version);
}
}
else if (type == Schema.DataTypes.PackageDelete.ToString())
{
// Delete package
await UpdateMetadata(storage, versions => versions.Remove(NuGetVersion.Parse(version)), cancellationToken);
await DeleteNuspec(storage, id, version, cancellationToken);
await DeleteNupkg(storage, id, version, cancellationToken);
Trace.TraceInformation("commit delete: {0}/{1}", id, version);
}
}
return(true);
}
static async Task Loop(string source, StorageFactory storageFactory, string contentBaseAddress, bool verbose, int interval)
{
CommitCollector collector = new DnxCatalogCollector(new Uri(source), storageFactory, CommandHelpers.GetHttpMessageHandlerFactory(verbose))
{
ContentBaseAddress = contentBaseAddress == null ? null : new Uri(contentBaseAddress)
};
Storage storage = storageFactory.Create();
ReadWriteCursor front = new DurableCursor(storage.ResolveUri("cursor.json"), storage, MemoryCursor.Min.Value);
ReadCursor back = MemoryCursor.Max;
while (true)
{
bool run = false;
do
{
run = await collector.Run(front, back);
}while (run);
Thread.Sleep(interval * 1000);
}
}
protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken)
{
Gallery = arguments.GetOrThrow <string>(Arguments.Gallery);
Verbose = arguments.GetOrDefault(Arguments.Verbose, false);
StartDate = arguments.GetOrDefault(Arguments.StartDate, Constants.DateTimeMinValueUtc);
SkipCreatedPackagesProcessing = arguments.GetOrDefault(Arguments.SkipCreatedPackagesProcessing, false);
StorageFactory preferredPackageSourceStorageFactory = null;
var preferAlternatePackageSourceStorage = arguments.GetOrDefault(Arguments.PreferAlternatePackageSourceStorage, false);
if (preferAlternatePackageSourceStorage)
{
preferredPackageSourceStorageFactory = CommandHelpers.CreateSuffixedStorageFactory("PreferredPackageSourceStorage", arguments, Verbose);
}
var catalogStorageFactory = CommandHelpers.CreateStorageFactory(arguments, Verbose);
var auditingStorageFactory = CommandHelpers.CreateSuffixedStorageFactory("Auditing", arguments, Verbose);
Logger.LogInformation("CONFIG source: \"{ConfigSource}\" storage: \"{Storage}\" preferred package source storage: \"{PreferredPackageSourceStorage}\"",
Gallery,
catalogStorageFactory,
preferredPackageSourceStorageFactory);
CatalogStorage = catalogStorageFactory.Create();
AuditingStorage = auditingStorageFactory.Create();
if (preferAlternatePackageSourceStorage)
{
PreferredPackageSourceStorage = preferredPackageSourceStorageFactory.Create();
}
Destination = catalogStorageFactory.BaseAddress;
TelemetryService.GlobalDimensions[TelemetryConstants.Destination] = Destination.AbsoluteUri;
Top = 20;
Timeout = TimeSpan.FromSeconds(300);
}
static async Task ProcessPackages(string gallery, StorageFactory storageFactory, string id, string version, bool verbose)
{
int timeout = 300;
Func <HttpMessageHandler> handlerFunc = CommandHelpers.GetHttpMessageHandlerFactory(verbose);
HttpMessageHandler handler = (handlerFunc != null) ? handlerFunc() : new WebRequestHandler {
AllowPipelining = true
};
using (HttpClient client = new HttpClient(handler))
{
client.Timeout = TimeSpan.FromSeconds(timeout);
// if teh version is specified a single package is processed otherwise all the packages corresponding to that id are processed
Uri uri = (version == null) ? MakePackageUri(gallery, id) : MakePackageUri(gallery, id, version);
SortedList <DateTime, IList <Tuple <Uri, PackageDates> > > packages = await GetPackages(client, uri, "Created");
Trace.TraceInformation("downloading {0} packages", packages.Select(t => t.Value.Count).Sum());
Storage storage = storageFactory.Create();
// the idea here is to leave the lastCreated and lastEdited values exactly as they were
const string LastCreated = "nuget:lastCreated";
const string LastEdited = "nuget:lastEdited";
DateTime lastCreated = await GetCatalogProperty(storage, LastCreated) ?? DateTime.MinValue.ToUniversalTime();
DateTime lastEdited = await GetCatalogProperty(storage, LastEdited) ?? DateTime.MinValue.ToUniversalTime();
DateTime d = await DownloadMetadata2Catalog(client, packages, storage, lastCreated, lastEdited);
}
}
protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken)
{
Verbose = arguments.GetOrDefault(Arguments.Verbose, false);
StartDate = arguments.GetOrDefault(Arguments.StartDate, Constants.DateTimeMinValueUtc);
Top = arguments.GetOrDefault(Arguments.CursorSize, 20);
SkipCreatedPackagesProcessing = arguments.GetOrDefault(Arguments.SkipCreatedPackagesProcessing, false);
StorageFactory preferredPackageSourceStorageFactory = null;
var preferAlternatePackageSourceStorage = arguments.GetOrDefault(Arguments.PreferAlternatePackageSourceStorage, false);
if (preferAlternatePackageSourceStorage)
{
preferredPackageSourceStorageFactory = CommandHelpers.CreateSuffixedStorageFactory(
"PreferredPackageSourceStorage",
arguments,
Verbose,
new SemaphoreSlimThrottle(new SemaphoreSlim(ServicePointManager.DefaultConnectionLimit)));
}
var catalogStorageFactory = CommandHelpers.CreateStorageFactory(
arguments,
Verbose,
new SemaphoreSlimThrottle(new SemaphoreSlim(ServicePointManager.DefaultConnectionLimit)));
var auditingStorageFactory = CommandHelpers.CreateSuffixedStorageFactory(
"Auditing",
arguments,
Verbose,
new SemaphoreSlimThrottle(new SemaphoreSlim(ServicePointManager.DefaultConnectionLimit)));
Logger.LogInformation("CONFIG source: \"{ConfigSource}\" storage: \"{Storage}\" preferred package source storage: \"{PreferredPackageSourceStorage}\"",
GalleryDbConnection,
catalogStorageFactory,
preferredPackageSourceStorageFactory);
CatalogStorage = catalogStorageFactory.Create();
AuditingStorage = auditingStorageFactory.Create();
if (preferAlternatePackageSourceStorage)
{
PreferredPackageSourceStorage = preferredPackageSourceStorageFactory.Create();
}
Destination = catalogStorageFactory.BaseAddress;
TelemetryService.GlobalDimensions[TelemetryConstants.Destination] = Destination.AbsoluteUri;
// Setup gallery database access
PackageContentUriBuilder = new PackageContentUriBuilder(
arguments.GetOrThrow <string>(Arguments.PackageContentUrlFormat));
var connectionString = arguments.GetOrThrow <string>(Arguments.ConnectionString);
GalleryDbConnection = new AzureSqlConnectionFactory(
connectionString,
SecretInjector,
LoggerFactory.CreateLogger <AzureSqlConnectionFactory>());
var timeoutInSeconds = arguments.GetOrDefault(Arguments.SqlCommandTimeoutInSeconds, 300);
Timeout = TimeSpan.FromSeconds(timeoutInSeconds);
GalleryDatabaseQueryService = new GalleryDatabaseQueryService(
GalleryDbConnection,
PackageContentUriBuilder,
TelemetryService,
timeoutInSeconds);
}
public StorageRegistration(StorageFactory storageFactory)
{
_storage = storageFactory.Create();
}
public EntityManyRelationship()
{
Known = StorageFactory.Create <S, E, C>();
}
static async Task Loop(string gallery, StorageFactory storageFactory, bool verbose, int interval, DateTime?startDate)
{
Storage storage = storageFactory.Create();
const string LastCreated = "nuget:lastCreated";
const string LastEdited = "nuget:lastEdited";
int top = 20;
int timeout = 300;
while (true)
{
Func <HttpMessageHandler> handlerFunc = CommandHelpers.GetHttpMessageHandlerFactory(verbose);
HttpMessageHandler handler = (handlerFunc != null) ? handlerFunc() : new WebRequestHandler {
AllowPipelining = true
};
using (HttpClient client = new HttpClient(handler))
{
client.Timeout = TimeSpan.FromSeconds(timeout);
// fetch and add all newly CREATED packages - in order
DateTime lastCreated = await GetCatalogProperty(storage, LastCreated) ?? (startDate ?? DateTime.MinValue.ToUniversalTime());
DateTime lastEdited = await GetCatalogProperty(storage, LastEdited) ?? lastCreated;
SortedList <DateTime, IList <Tuple <Uri, PackageDates> > > createdPackages;
DateTime previousLastCreated = DateTime.MinValue;
do
{
Trace.TraceInformation("CATALOG LastCreated: {0}", lastCreated.ToString("O"));
createdPackages = await GetCreatedPackages(client, gallery, lastCreated, top);
Trace.TraceInformation("FEED CreatedPackages: {0}", createdPackages.Count);
lastCreated = await DownloadMetadata2Catalog(client, createdPackages, storage, lastCreated, lastEdited, createdPackages : true);
if (previousLastCreated == lastCreated)
{
break;
}
previousLastCreated = lastCreated;
}while (createdPackages.Count > 0);
// THEN fetch and add all EDITED packages - in order
SortedList <DateTime, IList <Tuple <Uri, PackageDates> > > editedPackages;
DateTime previousLastEdited = DateTime.MinValue;
do
{
Trace.TraceInformation("CATALOG LastEdited: {0}", lastEdited.ToString("O"));
editedPackages = await GetEditedPackages(client, gallery, lastEdited, top);
Trace.TraceInformation("FEED EditedPackages: {0}", editedPackages.Count);
lastEdited = await DownloadMetadata2Catalog(client, editedPackages, storage, lastCreated, lastEdited, createdPackages : false);
if (previousLastEdited == lastEdited)
{
break;
}
previousLastEdited = lastEdited;
}while (editedPackages.Count > 0);
}
Thread.Sleep(interval * 1000);
}
}
