private async Task <SourceStatus> IndexSource(IStorageFeed feed, PackageName packageName, ISourceInfo sourceInfo) { var sourceStatus = new SourceStatus(new SourceName(Path.GetFileName(sourceInfo.OriginalPath), sourceInfo.Hash)); try { Debug.WriteLine("Indexing source {0}", sourceStatus.SourceName); await RequestOrSkip(string.Format("src/{0}", sourceStatus.SourceName), async() => { using (var stream = sourceInfo.File.GetStream()) using (var convertedStream = SourceConverter.Convert(stream)) await feed.GetSource(packageName, sourceStatus.SourceName).Put(convertedStream); }); sourceStatus.Stored = true; Debug.WriteLine("Stored source {0}", sourceStatus.SourceName); } catch (Exception e) { support.TrackException(e, new { packageName }); sourceStatus.Exception = new ExceptionStatus(e); } return(sourceStatus); }
private async Task <SourceStatus> DeleteSource(IStorageFeed feed, PackageName packageName, SourceStatus sourceStatus) { var newSourceStatus = new SourceStatus(sourceStatus.SourceName); if (sourceStatus.Stored) { try { Debug.WriteLine("Deleting source {0}", sourceStatus.SourceName); newSourceStatus.Stored = true; await RequestOrSkip(string.Format("src/{0}", sourceStatus.SourceName), async() => { var source = feed.GetSource(packageName, sourceStatus.SourceName); await source.Delete(); }); newSourceStatus.Stored = false; Debug.WriteLine("Deleted source {0}", sourceStatus.SourceName); } catch (Exception e) { support.TrackException(e, new { packageName }); newSourceStatus.Exception = new ExceptionStatus(e); } } return(newSourceStatus); }
public async Task <PackageName> ReadName(UserInfo userInfo, ZipPackage package) { var suffix = "-at-" + DateTime.Now.ToEncodedSeconds(); var newVersion = package.Version.ToString().AddSuffix(suffix, MaxSemanticVersionLength, '.'); var newPackageName = new PackageName(package.Id, newVersion); return(newPackageName); }
public async Task IndexPackage(UserInfo userInfo, IStorageFeed feed, PackageName packageName, IPackageStorageItem packageItem, ZipPackage package) { var statusPackageBuilder = CreatePackage(packageName, package); var statusPackageState = PackageState.Partial; try { var statusFiles = await ProcessThrottled( 2, addInfoBuilder.Build(new NugetPackage(package)), async i => await IndexImage(feed, packageName, i)); foreach (var statusFile in statusFiles) { statusPackageBuilder.Files.Add(CreateFile(statusFile.FilePath, statusFile.ImageStatus)); } if (statusFiles.All(s => s.ImageStatus.Check(true))) { statusPackageState = PackageState.Succeded; } Trace.TraceInformation("Marking package {0} as {1}", packageName, statusPackageState); } catch (Exception e) { support.TrackException(e, new { packageName }); Trace.TraceError("Error while indexing package {0}:\n{1}", packageName, e); statusPackageBuilder.Files.Add(CreateFile("error.txt", e.ToString())); } if (statusPackageBuilder.Files.Count == 0) { statusPackageBuilder.Files.Add(CreateFile("empty.txt", string.Empty)); } Debug.WriteLine("Saving package processing status {0}", packageName); var statusPackageItem = feed.GetPackage(await packageItem.GetUserName(), statusPackageState, packageName); using (var statusStream = await statusPackageItem.Put()) statusPackageBuilder.SaveBuffered(statusStream); switch (statusPackageState) { case PackageState.Partial: await notifier.PartiallyIndexed(userInfo, packageName); break; case PackageState.Succeded: await notifier.Indexed(userInfo, packageName); break; default: // ReSharper disable once NotResolvedInText throw new ArgumentOutOfRangeException("statusPackageState", statusPackageState, null); } }
private async Task <bool> ProcessPackage(UserInfo userInfo, string feedName, PackageName packageName, PackageState packageState, PackageState damagedPackageState, Func <IPackageStorageItem, Task <IPackageStorageItem> > tryWithPackageItemBefore, Func <IPackageStorageItem, ZipPackage, Task> tryWithPackage, Func <IPackageStorageItem, Task> tryWithPackageItemAfter) { var feed = storage.GetFeed(feedName); var packageItem = feed.GetPackage(null, packageState, packageName); var traceName = packageItem.ToString(); try { packageItem = await tryWithPackageItemBefore(packageItem); if (packageItem == null) { throw new Exception(string.Format("Missing package {0}", traceName)); } using (var packageStream = await packageItem.Get()) { if (packageStream == null) { throw new Exception(string.Format("Missing package data {0}", traceName)); } var package = new ZipPackage(packageStream); await tryWithPackage(packageItem, package); } await tryWithPackageItemAfter(packageItem); return(true); } catch (Exception e) { support.TrackException(e, new { packageName }); Trace.TraceError("Unexpected error while processing package {0}:\n{1}", traceName, e); notifier.Damaged(userInfo, packageName).Wait(); if (packageItem != null) { packageItem.Move(damagedPackageState, packageName).Wait(); } return(false); } }
private async Task <SymbolStatus> IndexSymbol(IStorageFeed feed, PackageName packageName, IBinaryInfo binaryInfo) { var symbolStatus = new SymbolStatus(new SymbolName(binaryInfo.Name, binaryInfo.SymbolInfo.Hash)); try { Debug.WriteLine("Indexing symbol {0}", symbolStatus.SymbolName); await RequestOrSkip(string.Format("pdb/{0}", symbolStatus.SymbolName), async() => { Debug.WriteLine("Storing symbol {0}", symbolStatus.SymbolName); var symbolItem = feed.GetSymbol(packageName, symbolStatus.SymbolName); var pdbstrSection = CreatePdbStrSection(binaryInfo); using (var inputStream = binaryInfo.SymbolInfo.File.GetStream()) using (var tempStream = new MemoryStream()) using (var outputStream = await symbolItem.Put()) { pdbStoreManager.WriteSrcSrv(inputStream, tempStream, pdbstrSection); tempStream.Position = 0; var symbolFileName = Path.GetFileName(binaryInfo.SymbolInfo.File.FullPath); fileCompressor.Compress(symbolFileName, tempStream, outputStream); } }); symbolStatus.Stored = true; Debug.WriteLine("Stored symbol {0}", symbolStatus.SymbolName); } catch (Exception e) { support.TrackException(e, new { packageName }); symbolStatus.Exception = new ExceptionStatus(e); } if (binaryInfo.SymbolInfo.SourceInfos != null) { symbolStatus.SourceStatuses = await ProcessThrottled( 5, binaryInfo.SymbolInfo.SourceInfos, async s => await IndexSource(feed, packageName, s)); } return(symbolStatus); }
public async Task <IHttpActionResult> Delete(string packageId, string packageVersion) { var principal = (FeedPrincipal)User; var packageName = new PackageName(packageId, packageVersion); Trace.TraceInformation("{0} - deleting package {1}", principal, packageName); if (!security.Authorize(principal, new[] { packageName })) { Trace.TraceInformation("{0} - not authorizated to delete package {1}", principal, packageName); return(Forbidden()); } if (!principal.Identity.IsAuthenticated) { throw new InvalidOperationException("Identity is required to delete packages"); } var package = storage.GetFeed(principal.FeedName).GetPackage(null, principal.PackageState, packageName); //if (principal.AuthenticatedArea == AuthenticatedArea.DeletingOwn && package.UserName != identity.Name) // return NotFound(); if (principal.PackageState == PackageState.Succeded || principal.PackageState == PackageState.Partial) { if (await package.Move(PackageState.DeletingQueued, packageName) == null) { return(NotFound()); } await scheduler.Signal(principal, PackageState.DeletingQueued, packageName); } else { if (!await package.Delete()) { return(NotFound()); } } return(Ok()); }
private async Task Queue(PackageMessage message) { var task = new QueuePackageTask(); PackageName newPackageName = null; // ReSharper disable RedundantArgumentName await ProcessPackage( message.UserInfo, message.FeedName, message.PackageName, PackageState.New, PackageState.DamagedNew, tryWithPackageItemBefore : async pi => { Trace.TraceInformation("Reading package {0}", pi); return(pi); }, tryWithPackage : async(pi, p) => { newPackageName = await task.ReadName(message.UserInfo, p); await notifier.Starting(message.UserInfo, newPackageName); }, tryWithPackageItemAfter : async pi => { Trace.TraceInformation("Queueing package {0} as {1}", pi, newPackageName); if (await pi.Copy(PackageState.Original, newPackageName) == null) { throw new Exception(string.Format("Failed to copy package {0} to {1}", pi, newPackageName)); } if (await pi.Move(PackageState.IndexingQueued, newPackageName) == null) { throw new Exception(string.Format("Failed to move package {0} to {1}", pi, newPackageName)); } message.PackageState = PackageState.IndexingQueued; message.PackageName = newPackageName; await scheduler.Signal(message); Trace.TraceInformation("Finished queueing package {0} as {1}", pi, newPackageName); }); // ReSharper restore RedundantArgumentName }
public async Task <IHttpActionResult> Download(string packageId, string packageVersion) { var principal = (FeedPrincipal)User; var packageName = new PackageName(packageId, packageVersion); Trace.TraceInformation("{0} - downloading package {1}", principal, packageName); if (!security.Authorize(principal, new[] { packageName })) { Trace.TraceInformation("{0} - not authorizated to download package {1}", principal, packageName); return(Unauthorized()); } if (principal.PackageState == PackageState.New) { return(BadRequest("Downloading new packages is not supported. Wait until they are at least queued.")); } var package = storage.GetFeed(principal.FeedName).GetPackage(principal.Identity.Name, principal.PackageState, packageName); return(await Storage(package)); }
private async Task <ImageStatusFile> IndexImage(IStorageFeed feed, PackageName packageName, IBinaryInfo binaryInfo) { var statusFile = new ImageStatusFile(binaryInfo.File.FullPath, new ImageStatus(new ImageName(binaryInfo.Name, binaryInfo.Hash))); //try //{ // //Debug.WriteLine("Indexing image {0}", imageStatus.ImageName); // //imageStatus.Stored = true; // //Debug.WriteLine("Stored image {0}", imageStatus.ImageName); //} //catch (Exception e) //{ // support.TrackException(e); // statusFile.ImageStatus.Exception = new ExceptionStatus(e); //} if (binaryInfo.SymbolInfo != null) { statusFile.ImageStatus.SymbolStatus = await IndexSymbol(feed, packageName, binaryInfo); } return(statusFile); }
private async Task <SymbolStatus> DeleteSymbol(IStorageFeed feed, PackageName packageName, SymbolStatus symbolStatus) { var newSymbolStatus = new SymbolStatus(symbolStatus.SymbolName); if (symbolStatus.Stored) { try { Debug.WriteLine("Deleting symbol {0}", symbolStatus.SymbolName); newSymbolStatus.Stored = true; await RequestOrSkip(string.Format("pdb/{0}", symbolStatus.SymbolName), async() => { var symbolItem = feed.GetSymbol(packageName, symbolStatus.SymbolName); await symbolItem.Delete(); }); newSymbolStatus.Stored = false; Debug.WriteLine("Deleted symbol {0}", symbolStatus.SymbolName); } catch (Exception e) { support.TrackException(e, new { packageName }); newSymbolStatus.Exception = new ExceptionStatus(e); } } if (symbolStatus.SourceStatuses != null) { newSymbolStatus.SourceStatuses = await ProcessThrottled( 5, symbolStatus.SourceStatuses, async s => await DeleteSource(feed, packageName, s)); } return(newSymbolStatus); }
private async Task <ImageStatusFile> DeleteImage(IStorageFeed feed, PackageName packageName, ImageStatusFile statusFile) { var newStatusFile = new ImageStatusFile(statusFile.FilePath, new ImageStatus(statusFile.ImageStatus.ImageName)); //try //{ // //Debug.WriteLine("Deleting image {0}", statusFile.ImageStatus.ImageName); // //statusFile.ImageStatus.Stored = false; // statusFile.ImageStatus.Exception = null; // //Debug.WriteLine("Deleted image {0}", statusFile.ImageStatus.ImageName); //} //catch (Exception e) //{ // support.TrackException(e); // newStatusFile.ImageStatus.Exception = new ExceptionStatus(e); //} if (statusFile.ImageStatus.SymbolStatus != null) { newStatusFile.ImageStatus.SymbolStatus = await DeleteSymbol(feed, packageName, statusFile.ImageStatus.SymbolStatus); } return(newStatusFile); }
public async Task <IHttpActionResult> Put() { var principal = (FeedPrincipal)User; Trace.TraceInformation("{0} - {1} byte(s)", principal, Request.Content.Headers.ContentLength); if (!principal.Identity.IsAuthenticated) { throw new InvalidOperationException("Identity is required to put packages"); } var provider = new StorageMultipartStreamProvider(storage, scheduler, principal); await Request.Content.ReadAsMultipartAsync(provider); var forbidden = 0; var corrupted = 0; foreach (var packageInfo in provider.Packages) { packageInfo.Value.Position = 0; PackageName packageName = null; try { var package = new ZipPackage(packageInfo.Value); packageName = new PackageName(package.Id, package.Version.ToString()); } catch (Exception e) { Trace.TraceInformation("{0} - corrupted package", principal); support.TrackException(e, null); corrupted++; } if (packageName == null) { await packageInfo.Key.Delete(); continue; } if (!security.Authorize(principal, new[] { packageName })) { Trace.TraceInformation("{0} - not authorizated to push {1}", principal, packageName); forbidden++; await packageInfo.Key.Delete(); continue; } Trace.TraceInformation("Signaling package {0} ({1})", packageInfo.Key.Name, packageName); await scheduler.Signal(principal, PackageState.New, packageInfo.Key.Name); } if (corrupted > 0) { return(BadRequest()); } if (forbidden > 0) { return(Forbidden()); } return(Ok()); }
public async Task DeletePackage(UserInfo userInfo, IStorageFeed feed, PackageName packageName, IPackageStorageItem packageItem, IPackage package) { var statusPackageBuilder = CreatePackage(packageName, package); var statusPackageState = PackageState.Partial; try { var allRead = true; var statusFiles = new List <ImageStatusFile>(); foreach (var statusPackageFile in package.GetFiles().Where(f => f.Path.EndsWith(".smbsrc"))) { try { var imageStatus = ReadFile <ImageStatus>(statusPackageFile); statusFiles.Add(new ImageStatusFile(statusPackageFile.Path, imageStatus)); } catch (Exception e) { support.TrackException(e, new { packageName }); Trace.TraceError("Error while deserializing status {0}:\n{1}", packageName, e); allRead = false; statusPackageBuilder.Files.Add(statusPackageFile); } } var newStatusFiles = await ProcessThrottled( 2, statusFiles, async s => await DeleteImage(feed, packageName, s)); foreach (var statusFile in newStatusFiles) { statusPackageBuilder.Files.Add(CreateFile(statusFile.FilePath, statusFile.ImageStatus)); } if (allRead && newStatusFiles.All(s => s.ImageStatus.Check(false))) { statusPackageState = PackageState.Deleted; } Trace.TraceInformation("Marking package {0} as {1}", packageName, statusPackageState); } catch (Exception e) { support.TrackException(e, new { packageName }); Trace.TraceError("Error while deleting package {0}:\n{1}", packageName, e); statusPackageBuilder.Files.Add(CreateFile("error.txt", e.ToString())); } if (statusPackageBuilder.Files.Count == 0) { statusPackageBuilder.Files.Add(CreateFile("empty.txt", string.Empty)); } Debug.WriteLine("Saving package processing status {0}", packageName); var statusPackageItem = feed.GetPackage(await packageItem.GetUserName(), statusPackageState, packageName); using (var statusStream = await statusPackageItem.Put()) statusPackageBuilder.SaveBuffered(statusStream); switch (statusPackageState) { case PackageState.Partial: await notifier.PartiallyDeleted(userInfo, packageName); break; case PackageState.Deleted: await notifier.Deleted(userInfo, packageName); break; default: // ReSharper disable once NotResolvedInText throw new ArgumentOutOfRangeException("statusPackageState", statusPackageState, null); } }