private void CopyPackageData(IPackage package, LucenePackage lucenePackage)
{
lucenePackage.Id = package.Id;
lucenePackage.Version = new StrictSemanticVersion(package.Version.ToString());
lucenePackage.MinClientVersion = package.MinClientVersion;
lucenePackage.Title = package.Title;
lucenePackage.Authors = (package.Authors ?? Enumerable.Empty <string>()).Select(i => i.Trim()).ToArray();
lucenePackage.Owners = (package.Owners ?? Enumerable.Empty <string>()).Select(i => i.Trim()).ToArray();
lucenePackage.IconUrl = FilterPlaceholderUri(package.IconUrl);
lucenePackage.LicenseUrl = FilterPlaceholderUri(package.LicenseUrl);
lucenePackage.ProjectUrl = FilterPlaceholderUri(package.ProjectUrl);
lucenePackage.RequireLicenseAcceptance = package.RequireLicenseAcceptance;
lucenePackage.Description = package.Description;
lucenePackage.Summary = package.Summary;
lucenePackage.ReleaseNotes = package.ReleaseNotes;
lucenePackage.Language = package.Language;
lucenePackage.Tags = package.Tags;
lucenePackage.Copyright = package.Copyright;
lucenePackage.FrameworkAssemblies = package.FrameworkAssemblies;
lucenePackage.DependencySets = package.DependencySets;
lucenePackage.ReportAbuseUrl = package.ReportAbuseUrl;
lucenePackage.DownloadCount = package.DownloadCount;
lucenePackage.IsAbsoluteLatestVersion = package.IsAbsoluteLatestVersion;
lucenePackage.IsLatestVersion = package.IsLatestVersion;
lucenePackage.Listed = package.Listed;
lucenePackage.Published = package.Published;
lucenePackage.AssemblyReferences = package.AssemblyReferences;
lucenePackage.PackageAssemblyReferences = package.PackageAssemblyReferences;
lucenePackage.DevelopmentDependency = package.DevelopmentDependency;
}
public Task AddPackageAsync(LucenePackage package, CancellationToken cancellationToken)
{
var update = new Update(package, UpdateType.Add);
pendingUpdates.Add(update, cancellationToken);
return(update.Task);
}
private DateTimeOffset GetLastModified(LucenePackage package, string path)
{
var lastModified = FileSystem.GetLastModified(path);
if (lastModified.Year <= 1700)
{
lastModified = package.Created;
}
return(lastModified);
}
private LucenePackage Convert(IPackage package, LucenePackage lucenePackage)
{
CopyPackageData(package, lucenePackage);
if (string.IsNullOrWhiteSpace(lucenePackage.Path))
{
lucenePackage.Path = GetPackageFilePath(lucenePackage);
}
CalculateDerivedData(package, lucenePackage, lucenePackage.Path, lucenePackage.GetStream);
return(lucenePackage);
}
internal async Task SynchronizeIndexWithFileSystemAsync(IndexDifferences diff, CancellationToken cancellationToken)
{
if (diff.IsEmpty)
{
return;
}
var tasks = new ConcurrentQueue <Task>();
Log.Info(string.Format("Updates to process: {0} packages added, {1} packages updated, {2} packages removed.", diff.NewPackages.Count(), diff.ModifiedPackages.Count(), diff.MissingPackages.Count()));
foreach (var path in diff.MissingPackages)
{
cancellationToken.ThrowIfCancellationRequested();
var package = new LucenePackage(FileSystem)
{
Path = path
};
var update = new Update(package, UpdateType.RemoveByPath);
pendingUpdates.Add(update, cancellationToken);
tasks.Enqueue(update.Task);
}
var pathsToIndex = diff.NewPackages.Union(diff.ModifiedPackages).OrderBy(p => p).ToArray();
var packagesToIndex = pathsToIndex.Length;
var i = 0;
Parallel.ForEach(pathsToIndex, new ParallelOptions {
MaxDegreeOfParallelism = 4, CancellationToken = cancellationToken
}, (p, s) =>
{
UpdateSynchronizationStatus(SynchronizationState.Indexing, Interlocked.Increment(ref i),
packagesToIndex);
tasks.Enqueue(SynchronizePackage(p, cancellationToken));
});
var task = TaskEx.WhenAll(tasks.ToArray());
try
{
await task;
}
finally
{
if (task.IsFaulted && task.Exception.InnerExceptions.Count > 1)
{
throw new AggregateException(task.Exception.InnerExceptions);
}
}
}
private async Task <LucenePackage> DownloadOrMoveOrAddPackageToFileSystemAsync(IPackage package, CancellationToken cancellationToken)
{
string temporaryLocation = null;
try
{
if (PackageOverwriteMode == PackageOverwriteMode.Deny && FindPackage(package.Id, package.Version) != null)
{
throw new PackageOverwriteDeniedException(package);
}
LucenePackage lucenePackage = null;
var fastZipPackage = package as IFastZipPackage;
var dataPackage = package as DataServicePackage;
if (dataPackage != null)
{
fastZipPackage = await DownloadDataServicePackage(dataPackage, cancellationToken);
lucenePackage = Convert(fastZipPackage);
lucenePackage.OriginUrl = dataPackage.DownloadUrl;
lucenePackage.IsMirrored = true;
}
temporaryLocation = fastZipPackage != null?fastZipPackage.GetFileLocation() : null;
if (!string.IsNullOrEmpty(temporaryLocation))
{
MoveFileWithOverwrite(temporaryLocation, base.GetPackageFilePath(fastZipPackage));
if (lucenePackage == null)
{
lucenePackage = Convert(fastZipPackage);
}
return(lucenePackage);
}
else
{
return(await AddPackageToFileSystemAsync(package));
}
}
finally
{
if (!string.IsNullOrEmpty(temporaryLocation))
{
FileSystem.DeleteFile(temporaryLocation);
}
}
}
public LucenePackage Convert(IPackage package)
{
var lucenePackage = package as LucenePackage;
if (lucenePackage == null)
{
lucenePackage = new LucenePackage(FileSystem);
Convert(package, lucenePackage);
}
frameworkCompatibilityTool.AddKnownFrameworkShortNames(lucenePackage.SupportedFrameworks);
return(lucenePackage);
}
private void CalculateDerivedDataFromStream(LucenePackage package, Func <Stream> openStream)
{
using (var stream = openStream())
{
if (!stream.CanSeek)
{
throw new InvalidOperationException("Package stream must support CanSeek.");
}
package.Created = FastZipPackageBase.GetPackageCreatedDateTime(stream);
package.PackageSize = stream.Length;
stream.Seek(0, SeekOrigin.Begin);
package.PackageHash = System.Convert.ToBase64String(HashProvider.CalculateHash(stream));
}
}
protected virtual void CalculateDerivedData(IPackage sourcePackage, LucenePackage package, string path, Func <Stream> openStream)
{
var fastPackage = sourcePackage as FastZipPackage;
if (fastPackage == null)
{
CalculateDerivedDataFromStream(package, openStream);
}
else
{
package.PackageSize = fastPackage.Size;
package.PackageHash = System.Convert.ToBase64String(fastPackage.Hash);
package.Created = fastPackage.Created;
}
package.PackageHashAlgorithm = HashAlgorithmName;
package.LastUpdated = GetLastModified(package, path);
package.Published = package.LastUpdated;
package.Path = path;
package.SupportedFrameworks = sourcePackage.GetSupportedFrameworks().Select(VersionUtility.GetShortFrameworkName);
package.Files = sourcePackage.GetFiles().Select(f => f.Path).ToArray();
}
public Update(LucenePackage package, UpdateType updateType)
{
this.package = package;
this.updateType = updateType;
}
internal static bool TimestampsMismatch(LucenePackage lucenePackage, DateTimeOffset fileLastModified)
{
var diff = fileLastModified - lucenePackage.Published.GetValueOrDefault(DateTimeOffset.MinValue);
return(Math.Abs(diff.TotalSeconds) > 1);
}
public LucenePackage Convert(IPackage package)
{
if (package is LucenePackage) return (LucenePackage)package;
var lucenePackage = new LucenePackage(FileSystem);
return Convert(package, lucenePackage);
}
private void CopyPackageData(IPackage package, LucenePackage lucenePackage)
{
lucenePackage.Id = package.Id;
lucenePackage.Version = new StrictSemanticVersion(package.Version.ToString());
lucenePackage.Title = package.Title;
lucenePackage.Authors = package.Authors;
lucenePackage.Owners = package.Owners;
lucenePackage.IconUrl = package.IconUrl;
lucenePackage.LicenseUrl = package.LicenseUrl;
lucenePackage.ProjectUrl = package.ProjectUrl;
lucenePackage.RequireLicenseAcceptance = package.RequireLicenseAcceptance;
lucenePackage.Description = package.Description;
lucenePackage.Summary = package.Summary;
lucenePackage.ReleaseNotes = package.ReleaseNotes;
lucenePackage.Language = package.Language;
lucenePackage.Tags = package.Tags;
lucenePackage.Copyright = package.Copyright;
lucenePackage.FrameworkAssemblies = package.FrameworkAssemblies;
lucenePackage.DependencySets = package.DependencySets;
lucenePackage.ReportAbuseUrl = package.ReportAbuseUrl;
lucenePackage.DownloadCount = package.DownloadCount;
lucenePackage.IsAbsoluteLatestVersion = package.IsAbsoluteLatestVersion;
lucenePackage.IsLatestVersion = package.IsLatestVersion;
lucenePackage.Listed = package.Listed;
lucenePackage.Published = package.Published;
lucenePackage.AssemblyReferences = package.AssemblyReferences;
}
protected virtual void CalculateDerivedData(IPackage sourcePackage, LucenePackage package, string path, Stream stream)
{
byte[] fileBytes;
using (stream)
{
fileBytes = stream.ReadAllBytes();
}
package.PackageSize = fileBytes.Length;
package.PackageHash = System.Convert.ToBase64String(HashProvider.CalculateHash(fileBytes));
package.PackageHashAlgorithm = HashAlgorithm;
package.LastUpdated = FileSystem.GetLastModified(path);
package.Published = package.LastUpdated;
package.Created = GetZipArchiveCreateDate(new MemoryStream(fileBytes));
package.Path = path;
package.SupportedFrameworks = sourcePackage.GetSupportedFrameworks().Select(VersionUtility.GetShortFrameworkName);
var localPackage = sourcePackage as LocalPackage;
if (localPackage != null)
{
package.Files = localPackage.GetFiles().Select(f => f.Path);
}
}
public Update(LucenePackage package, UpdateType updateType)
{
this.package = package;
this.updateType = updateType;
}
internal static bool TimestampsMismatch(LucenePackage lucenePackage, DateTimeOffset fileLastModified)
{
var diff = fileLastModified - lucenePackage.Published.GetValueOrDefault(DateTimeOffset.MinValue);
return Math.Abs(diff.TotalSeconds) > 1;
}
private LucenePackage Convert(IPackage package, LucenePackage lucenePackage)
{
CopyPackageData(package, lucenePackage);
var path = GetPackageFilePath(lucenePackage);
lucenePackage.Path = path;
CalculateDerivedData(package, lucenePackage, path, () => lucenePackage.GetStream());
return lucenePackage;
}
private DateTimeOffset GetLastModified(LucenePackage package, string path)
{
var lastModified = FileSystem.GetLastModified(path);
if (lastModified.Year <= 1700)
{
lastModified = package.Created;
}
return lastModified;
}
private void CopyPackageData(IPackage package, LucenePackage lucenePackage)
{
lucenePackage.Id = package.Id;
lucenePackage.Version = new StrictSemanticVersion(package.Version.ToString());
lucenePackage.MinClientVersion = package.MinClientVersion;
lucenePackage.Title = package.Title;
lucenePackage.Authors = (package.Authors ?? Enumerable.Empty<string>()).Select(i => i.Trim()).ToArray();
lucenePackage.Owners = (package.Owners ?? Enumerable.Empty<string>()).Select(i => i.Trim()).ToArray();
lucenePackage.IconUrl = FilterPlaceholderUri(package.IconUrl);
lucenePackage.LicenseUrl = FilterPlaceholderUri(package.LicenseUrl);
lucenePackage.ProjectUrl = FilterPlaceholderUri(package.ProjectUrl);
lucenePackage.RequireLicenseAcceptance = package.RequireLicenseAcceptance;
lucenePackage.Description = package.Description;
lucenePackage.Summary = package.Summary;
lucenePackage.ReleaseNotes = package.ReleaseNotes;
lucenePackage.Language = package.Language;
lucenePackage.Tags = package.Tags;
lucenePackage.Copyright = package.Copyright;
lucenePackage.FrameworkAssemblies = package.FrameworkAssemblies;
lucenePackage.DependencySets = package.DependencySets;
lucenePackage.ReportAbuseUrl = package.ReportAbuseUrl;
lucenePackage.DownloadCount = package.DownloadCount;
lucenePackage.IsAbsoluteLatestVersion = package.IsAbsoluteLatestVersion;
lucenePackage.IsLatestVersion = package.IsLatestVersion;
lucenePackage.Listed = package.Listed;
lucenePackage.Published = package.Published;
lucenePackage.AssemblyReferences = package.AssemblyReferences;
lucenePackage.PackageAssemblyReferences = package.PackageAssemblyReferences;
lucenePackage.DevelopmentDependency = package.DevelopmentDependency;
}
public LucenePackage Convert(IPackage package)
{
var lucenePackage = package as LucenePackage;
if (lucenePackage == null)
{
lucenePackage = new LucenePackage(FileSystem);
Convert(package, lucenePackage);
}
frameworkCompatibilityTool.AddKnownFrameworkShortNames(lucenePackage.SupportedFrameworks);
return lucenePackage;
}
internal void SynchronizeIndexWithFileSystem(IndexDifferences diff, CancellationToken cancellationToken)
{
if (diff.IsEmpty) return;
var tasks = new ConcurrentQueue<Task>();
Log.Info(string.Format("Updates to process: {0} packages added, {1} packages updated, {2} packages removed.", diff.NewPackages.Count(), diff.ModifiedPackages.Count(), diff.MissingPackages.Count()));
foreach (var path in diff.MissingPackages)
{
cancellationToken.ThrowIfCancellationRequested();
var package = new LucenePackage(FileSystem) { Path = path };
var update = new Update(package, UpdateType.RemoveByPath);
pendingUpdates.Add(update);
tasks.Enqueue(update.Task);
}
var pathsToIndex = diff.NewPackages.Union(diff.ModifiedPackages).OrderBy(p => p).ToArray();
var packagesToIndex = pathsToIndex.Length;
var i = 0;
Parallel.ForEach(pathsToIndex, new ParallelOptions {MaxDegreeOfParallelism = 4, CancellationToken = cancellationToken}, (p, s) =>
{
UpdateSynchronizationStatus(SynchronizationState.Indexing, Interlocked.Increment(ref i),
packagesToIndex);
tasks.Enqueue(SynchronizePackage(p));
});
Task.WaitAll(tasks.ToArray(), cancellationToken);
}
public Task AddPackage(LucenePackage package)
{
var update = new Update(package, UpdateType.Add);
pendingUpdates.Add(update);
return update.Task;
}
private LucenePackage Convert(IPackage package, LucenePackage lucenePackage)
{
CopyPackageData(package, lucenePackage);
if (string.IsNullOrWhiteSpace(lucenePackage.Path))
{
lucenePackage.Path = GetPackageFilePath(lucenePackage);
}
CalculateDerivedData(package, lucenePackage, lucenePackage.Path, lucenePackage.GetStream);
return lucenePackage;
}
public LucenePackage Convert(IPackage package)
{
var lucenePackage = package as LucenePackage;
if (lucenePackage != null) return lucenePackage;
lucenePackage = new LucenePackage(FileSystem);
return Convert(package, lucenePackage);
}
protected virtual void CalculateDerivedData(IPackage sourcePackage, LucenePackage package, string path, Func<Stream> openStream)
{
var fastPackage = sourcePackage as FastZipPackage;
if (fastPackage == null)
{
CalculateDerivedDataFromStream(package, openStream);
}
else
{
package.PackageSize = fastPackage.Size;
package.PackageHash = System.Convert.ToBase64String(fastPackage.Hash);
package.Created = fastPackage.Created;
}
package.PackageHashAlgorithm = HashAlgorithmName;
package.LastUpdated = GetLastModified(package, path);
package.Published = package.LastUpdated;
package.Path = path;
package.SupportedFrameworks = sourcePackage.GetSupportedFrameworks().Select(VersionUtility.GetShortFrameworkName);
var files = IgnorePackageFiles
? Enumerable.Empty<string>()
: sourcePackage.GetFiles().Select(f => f.Path).ToArray();
package.Files = files;
}
protected virtual void CalculateDerivedData(IPackage sourcePackage, LucenePackage package, string path, Func<Stream> openStream)
{
var fastPackage = sourcePackage as FastZipPackage;
if (fastPackage == null)
{
CalculateDerivedDataSlowlyConsumingLotsOfMemory(package, openStream);
}
else
{
package.PackageSize = fastPackage.Size;
package.PackageHash = System.Convert.ToBase64String(fastPackage.Hash);
package.Created = fastPackage.Created;
}
package.PackageHashAlgorithm = HashAlgorithm;
package.LastUpdated = GetLastModified(package, path);
package.Published = package.LastUpdated;
package.Path = path;
package.SupportedFrameworks = sourcePackage.GetSupportedFrameworks().Select(VersionUtility.GetShortFrameworkName);
var localPackage = sourcePackage as LocalPackage;
if (localPackage != null)
{
package.Files = localPackage.GetFiles().Select(f => f.Path);
}
}
private void CalculateDerivedDataFromStream(LucenePackage package, Func<Stream> openStream)
{
using (var stream = openStream())
{
if (!stream.CanSeek)
{
throw new InvalidOperationException("Package stream must support CanSeek.");
}
package.Created = FastZipPackageBase.GetPackageCreatedDateTime(stream);
package.PackageSize = stream.Length;
stream.Seek(0, SeekOrigin.Begin);
package.PackageHash = System.Convert.ToBase64String(HashProvider.CalculateHash(stream));
}
}
private void CalculateDerivedDataSlowlyConsumingLotsOfMemory(LucenePackage package, Func<Stream> openStream)
{
byte[] fileBytes;
using (var stream = openStream())
{
fileBytes = stream.ReadAllBytes();
}
package.PackageSize = fileBytes.Length;
package.PackageHash = System.Convert.ToBase64String(HashProvider.CalculateHash(fileBytes));
package.Created = GetZipArchiveCreateDate(new MemoryStream(fileBytes));
}
