private void CopyPackageData(IPackage package, LucenePackage lucenePackage) { lucenePackage.Id = package.Id; lucenePackage.Version = new StrictSemanticVersion(package.Version.ToString()); lucenePackage.MinClientVersion = package.MinClientVersion; lucenePackage.Title = package.Title; lucenePackage.Authors = (package.Authors ?? Enumerable.Empty <string>()).Select(i => i.Trim()).ToArray(); lucenePackage.Owners = (package.Owners ?? Enumerable.Empty <string>()).Select(i => i.Trim()).ToArray(); lucenePackage.IconUrl = FilterPlaceholderUri(package.IconUrl); lucenePackage.LicenseUrl = FilterPlaceholderUri(package.LicenseUrl); lucenePackage.ProjectUrl = FilterPlaceholderUri(package.ProjectUrl); lucenePackage.RequireLicenseAcceptance = package.RequireLicenseAcceptance; lucenePackage.Description = package.Description; lucenePackage.Summary = package.Summary; lucenePackage.ReleaseNotes = package.ReleaseNotes; lucenePackage.Language = package.Language; lucenePackage.Tags = package.Tags; lucenePackage.Copyright = package.Copyright; lucenePackage.FrameworkAssemblies = package.FrameworkAssemblies; lucenePackage.DependencySets = package.DependencySets; lucenePackage.ReportAbuseUrl = package.ReportAbuseUrl; lucenePackage.DownloadCount = package.DownloadCount; lucenePackage.IsAbsoluteLatestVersion = package.IsAbsoluteLatestVersion; lucenePackage.IsLatestVersion = package.IsLatestVersion; lucenePackage.Listed = package.Listed; lucenePackage.Published = package.Published; lucenePackage.AssemblyReferences = package.AssemblyReferences; lucenePackage.PackageAssemblyReferences = package.PackageAssemblyReferences; lucenePackage.DevelopmentDependency = package.DevelopmentDependency; }
public Task AddPackageAsync(LucenePackage package, CancellationToken cancellationToken) { var update = new Update(package, UpdateType.Add); pendingUpdates.Add(update, cancellationToken); return(update.Task); }
private DateTimeOffset GetLastModified(LucenePackage package, string path) { var lastModified = FileSystem.GetLastModified(path); if (lastModified.Year <= 1700) { lastModified = package.Created; } return(lastModified); }
private LucenePackage Convert(IPackage package, LucenePackage lucenePackage) { CopyPackageData(package, lucenePackage); if (string.IsNullOrWhiteSpace(lucenePackage.Path)) { lucenePackage.Path = GetPackageFilePath(lucenePackage); } CalculateDerivedData(package, lucenePackage, lucenePackage.Path, lucenePackage.GetStream); return(lucenePackage); }
internal async Task SynchronizeIndexWithFileSystemAsync(IndexDifferences diff, CancellationToken cancellationToken) { if (diff.IsEmpty) { return; } var tasks = new ConcurrentQueue <Task>(); Log.Info(string.Format("Updates to process: {0} packages added, {1} packages updated, {2} packages removed.", diff.NewPackages.Count(), diff.ModifiedPackages.Count(), diff.MissingPackages.Count())); foreach (var path in diff.MissingPackages) { cancellationToken.ThrowIfCancellationRequested(); var package = new LucenePackage(FileSystem) { Path = path }; var update = new Update(package, UpdateType.RemoveByPath); pendingUpdates.Add(update, cancellationToken); tasks.Enqueue(update.Task); } var pathsToIndex = diff.NewPackages.Union(diff.ModifiedPackages).OrderBy(p => p).ToArray(); var packagesToIndex = pathsToIndex.Length; var i = 0; Parallel.ForEach(pathsToIndex, new ParallelOptions { MaxDegreeOfParallelism = 4, CancellationToken = cancellationToken }, (p, s) => { UpdateSynchronizationStatus(SynchronizationState.Indexing, Interlocked.Increment(ref i), packagesToIndex); tasks.Enqueue(SynchronizePackage(p, cancellationToken)); }); var task = TaskEx.WhenAll(tasks.ToArray()); try { await task; } finally { if (task.IsFaulted && task.Exception.InnerExceptions.Count > 1) { throw new AggregateException(task.Exception.InnerExceptions); } } }
private async Task <LucenePackage> DownloadOrMoveOrAddPackageToFileSystemAsync(IPackage package, CancellationToken cancellationToken) { string temporaryLocation = null; try { if (PackageOverwriteMode == PackageOverwriteMode.Deny && FindPackage(package.Id, package.Version) != null) { throw new PackageOverwriteDeniedException(package); } LucenePackage lucenePackage = null; var fastZipPackage = package as IFastZipPackage; var dataPackage = package as DataServicePackage; if (dataPackage != null) { fastZipPackage = await DownloadDataServicePackage(dataPackage, cancellationToken); lucenePackage = Convert(fastZipPackage); lucenePackage.OriginUrl = dataPackage.DownloadUrl; lucenePackage.IsMirrored = true; } temporaryLocation = fastZipPackage != null?fastZipPackage.GetFileLocation() : null; if (!string.IsNullOrEmpty(temporaryLocation)) { MoveFileWithOverwrite(temporaryLocation, base.GetPackageFilePath(fastZipPackage)); if (lucenePackage == null) { lucenePackage = Convert(fastZipPackage); } return(lucenePackage); } else { return(await AddPackageToFileSystemAsync(package)); } } finally { if (!string.IsNullOrEmpty(temporaryLocation)) { FileSystem.DeleteFile(temporaryLocation); } } }
public LucenePackage Convert(IPackage package) { var lucenePackage = package as LucenePackage; if (lucenePackage == null) { lucenePackage = new LucenePackage(FileSystem); Convert(package, lucenePackage); } frameworkCompatibilityTool.AddKnownFrameworkShortNames(lucenePackage.SupportedFrameworks); return(lucenePackage); }
private void CalculateDerivedDataFromStream(LucenePackage package, Func <Stream> openStream) { using (var stream = openStream()) { if (!stream.CanSeek) { throw new InvalidOperationException("Package stream must support CanSeek."); } package.Created = FastZipPackageBase.GetPackageCreatedDateTime(stream); package.PackageSize = stream.Length; stream.Seek(0, SeekOrigin.Begin); package.PackageHash = System.Convert.ToBase64String(HashProvider.CalculateHash(stream)); } }
protected virtual void CalculateDerivedData(IPackage sourcePackage, LucenePackage package, string path, Func <Stream> openStream) { var fastPackage = sourcePackage as FastZipPackage; if (fastPackage == null) { CalculateDerivedDataFromStream(package, openStream); } else { package.PackageSize = fastPackage.Size; package.PackageHash = System.Convert.ToBase64String(fastPackage.Hash); package.Created = fastPackage.Created; } package.PackageHashAlgorithm = HashAlgorithmName; package.LastUpdated = GetLastModified(package, path); package.Published = package.LastUpdated; package.Path = path; package.SupportedFrameworks = sourcePackage.GetSupportedFrameworks().Select(VersionUtility.GetShortFrameworkName); package.Files = sourcePackage.GetFiles().Select(f => f.Path).ToArray(); }
public Update(LucenePackage package, UpdateType updateType) { this.package = package; this.updateType = updateType; }
internal static bool TimestampsMismatch(LucenePackage lucenePackage, DateTimeOffset fileLastModified) { var diff = fileLastModified - lucenePackage.Published.GetValueOrDefault(DateTimeOffset.MinValue); return(Math.Abs(diff.TotalSeconds) > 1); }
public LucenePackage Convert(IPackage package) { if (package is LucenePackage) return (LucenePackage)package; var lucenePackage = new LucenePackage(FileSystem); return Convert(package, lucenePackage); }
private void CopyPackageData(IPackage package, LucenePackage lucenePackage) { lucenePackage.Id = package.Id; lucenePackage.Version = new StrictSemanticVersion(package.Version.ToString()); lucenePackage.Title = package.Title; lucenePackage.Authors = package.Authors; lucenePackage.Owners = package.Owners; lucenePackage.IconUrl = package.IconUrl; lucenePackage.LicenseUrl = package.LicenseUrl; lucenePackage.ProjectUrl = package.ProjectUrl; lucenePackage.RequireLicenseAcceptance = package.RequireLicenseAcceptance; lucenePackage.Description = package.Description; lucenePackage.Summary = package.Summary; lucenePackage.ReleaseNotes = package.ReleaseNotes; lucenePackage.Language = package.Language; lucenePackage.Tags = package.Tags; lucenePackage.Copyright = package.Copyright; lucenePackage.FrameworkAssemblies = package.FrameworkAssemblies; lucenePackage.DependencySets = package.DependencySets; lucenePackage.ReportAbuseUrl = package.ReportAbuseUrl; lucenePackage.DownloadCount = package.DownloadCount; lucenePackage.IsAbsoluteLatestVersion = package.IsAbsoluteLatestVersion; lucenePackage.IsLatestVersion = package.IsLatestVersion; lucenePackage.Listed = package.Listed; lucenePackage.Published = package.Published; lucenePackage.AssemblyReferences = package.AssemblyReferences; }
protected virtual void CalculateDerivedData(IPackage sourcePackage, LucenePackage package, string path, Stream stream) { byte[] fileBytes; using (stream) { fileBytes = stream.ReadAllBytes(); } package.PackageSize = fileBytes.Length; package.PackageHash = System.Convert.ToBase64String(HashProvider.CalculateHash(fileBytes)); package.PackageHashAlgorithm = HashAlgorithm; package.LastUpdated = FileSystem.GetLastModified(path); package.Published = package.LastUpdated; package.Created = GetZipArchiveCreateDate(new MemoryStream(fileBytes)); package.Path = path; package.SupportedFrameworks = sourcePackage.GetSupportedFrameworks().Select(VersionUtility.GetShortFrameworkName); var localPackage = sourcePackage as LocalPackage; if (localPackage != null) { package.Files = localPackage.GetFiles().Select(f => f.Path); } }
internal static bool TimestampsMismatch(LucenePackage lucenePackage, DateTimeOffset fileLastModified) { var diff = fileLastModified - lucenePackage.Published.GetValueOrDefault(DateTimeOffset.MinValue); return Math.Abs(diff.TotalSeconds) > 1; }
private LucenePackage Convert(IPackage package, LucenePackage lucenePackage) { CopyPackageData(package, lucenePackage); var path = GetPackageFilePath(lucenePackage); lucenePackage.Path = path; CalculateDerivedData(package, lucenePackage, path, () => lucenePackage.GetStream()); return lucenePackage; }
private DateTimeOffset GetLastModified(LucenePackage package, string path) { var lastModified = FileSystem.GetLastModified(path); if (lastModified.Year <= 1700) { lastModified = package.Created; } return lastModified; }
private void CopyPackageData(IPackage package, LucenePackage lucenePackage) { lucenePackage.Id = package.Id; lucenePackage.Version = new StrictSemanticVersion(package.Version.ToString()); lucenePackage.MinClientVersion = package.MinClientVersion; lucenePackage.Title = package.Title; lucenePackage.Authors = (package.Authors ?? Enumerable.Empty<string>()).Select(i => i.Trim()).ToArray(); lucenePackage.Owners = (package.Owners ?? Enumerable.Empty<string>()).Select(i => i.Trim()).ToArray(); lucenePackage.IconUrl = FilterPlaceholderUri(package.IconUrl); lucenePackage.LicenseUrl = FilterPlaceholderUri(package.LicenseUrl); lucenePackage.ProjectUrl = FilterPlaceholderUri(package.ProjectUrl); lucenePackage.RequireLicenseAcceptance = package.RequireLicenseAcceptance; lucenePackage.Description = package.Description; lucenePackage.Summary = package.Summary; lucenePackage.ReleaseNotes = package.ReleaseNotes; lucenePackage.Language = package.Language; lucenePackage.Tags = package.Tags; lucenePackage.Copyright = package.Copyright; lucenePackage.FrameworkAssemblies = package.FrameworkAssemblies; lucenePackage.DependencySets = package.DependencySets; lucenePackage.ReportAbuseUrl = package.ReportAbuseUrl; lucenePackage.DownloadCount = package.DownloadCount; lucenePackage.IsAbsoluteLatestVersion = package.IsAbsoluteLatestVersion; lucenePackage.IsLatestVersion = package.IsLatestVersion; lucenePackage.Listed = package.Listed; lucenePackage.Published = package.Published; lucenePackage.AssemblyReferences = package.AssemblyReferences; lucenePackage.PackageAssemblyReferences = package.PackageAssemblyReferences; lucenePackage.DevelopmentDependency = package.DevelopmentDependency; }
public LucenePackage Convert(IPackage package) { var lucenePackage = package as LucenePackage; if (lucenePackage == null) { lucenePackage = new LucenePackage(FileSystem); Convert(package, lucenePackage); } frameworkCompatibilityTool.AddKnownFrameworkShortNames(lucenePackage.SupportedFrameworks); return lucenePackage; }
internal void SynchronizeIndexWithFileSystem(IndexDifferences diff, CancellationToken cancellationToken) { if (diff.IsEmpty) return; var tasks = new ConcurrentQueue<Task>(); Log.Info(string.Format("Updates to process: {0} packages added, {1} packages updated, {2} packages removed.", diff.NewPackages.Count(), diff.ModifiedPackages.Count(), diff.MissingPackages.Count())); foreach (var path in diff.MissingPackages) { cancellationToken.ThrowIfCancellationRequested(); var package = new LucenePackage(FileSystem) { Path = path }; var update = new Update(package, UpdateType.RemoveByPath); pendingUpdates.Add(update); tasks.Enqueue(update.Task); } var pathsToIndex = diff.NewPackages.Union(diff.ModifiedPackages).OrderBy(p => p).ToArray(); var packagesToIndex = pathsToIndex.Length; var i = 0; Parallel.ForEach(pathsToIndex, new ParallelOptions {MaxDegreeOfParallelism = 4, CancellationToken = cancellationToken}, (p, s) => { UpdateSynchronizationStatus(SynchronizationState.Indexing, Interlocked.Increment(ref i), packagesToIndex); tasks.Enqueue(SynchronizePackage(p)); }); Task.WaitAll(tasks.ToArray(), cancellationToken); }
public Task AddPackage(LucenePackage package) { var update = new Update(package, UpdateType.Add); pendingUpdates.Add(update); return update.Task; }
private LucenePackage Convert(IPackage package, LucenePackage lucenePackage) { CopyPackageData(package, lucenePackage); if (string.IsNullOrWhiteSpace(lucenePackage.Path)) { lucenePackage.Path = GetPackageFilePath(lucenePackage); } CalculateDerivedData(package, lucenePackage, lucenePackage.Path, lucenePackage.GetStream); return lucenePackage; }
public LucenePackage Convert(IPackage package) { var lucenePackage = package as LucenePackage; if (lucenePackage != null) return lucenePackage; lucenePackage = new LucenePackage(FileSystem); return Convert(package, lucenePackage); }
protected virtual void CalculateDerivedData(IPackage sourcePackage, LucenePackage package, string path, Func<Stream> openStream) { var fastPackage = sourcePackage as FastZipPackage; if (fastPackage == null) { CalculateDerivedDataFromStream(package, openStream); } else { package.PackageSize = fastPackage.Size; package.PackageHash = System.Convert.ToBase64String(fastPackage.Hash); package.Created = fastPackage.Created; } package.PackageHashAlgorithm = HashAlgorithmName; package.LastUpdated = GetLastModified(package, path); package.Published = package.LastUpdated; package.Path = path; package.SupportedFrameworks = sourcePackage.GetSupportedFrameworks().Select(VersionUtility.GetShortFrameworkName); var files = IgnorePackageFiles ? Enumerable.Empty<string>() : sourcePackage.GetFiles().Select(f => f.Path).ToArray(); package.Files = files; }
protected virtual void CalculateDerivedData(IPackage sourcePackage, LucenePackage package, string path, Func<Stream> openStream) { var fastPackage = sourcePackage as FastZipPackage; if (fastPackage == null) { CalculateDerivedDataSlowlyConsumingLotsOfMemory(package, openStream); } else { package.PackageSize = fastPackage.Size; package.PackageHash = System.Convert.ToBase64String(fastPackage.Hash); package.Created = fastPackage.Created; } package.PackageHashAlgorithm = HashAlgorithm; package.LastUpdated = GetLastModified(package, path); package.Published = package.LastUpdated; package.Path = path; package.SupportedFrameworks = sourcePackage.GetSupportedFrameworks().Select(VersionUtility.GetShortFrameworkName); var localPackage = sourcePackage as LocalPackage; if (localPackage != null) { package.Files = localPackage.GetFiles().Select(f => f.Path); } }
private void CalculateDerivedDataFromStream(LucenePackage package, Func<Stream> openStream) { using (var stream = openStream()) { if (!stream.CanSeek) { throw new InvalidOperationException("Package stream must support CanSeek."); } package.Created = FastZipPackageBase.GetPackageCreatedDateTime(stream); package.PackageSize = stream.Length; stream.Seek(0, SeekOrigin.Begin); package.PackageHash = System.Convert.ToBase64String(HashProvider.CalculateHash(stream)); } }
private void CalculateDerivedDataSlowlyConsumingLotsOfMemory(LucenePackage package, Func<Stream> openStream) { byte[] fileBytes; using (var stream = openStream()) { fileBytes = stream.ReadAllBytes(); } package.PackageSize = fileBytes.Length; package.PackageHash = System.Convert.ToBase64String(HashProvider.CalculateHash(fileBytes)); package.Created = GetZipArchiveCreateDate(new MemoryStream(fileBytes)); }