private async Task <SourceStatus> IndexSource(IStorageFeed feed, PackageName packageName, ISourceInfo sourceInfo)
{
var sourceStatus = new SourceStatus(new SourceName(Path.GetFileName(sourceInfo.OriginalPath), sourceInfo.Hash));
try
{
Debug.WriteLine("Indexing source {0}", sourceStatus.SourceName);
await RequestOrSkip(string.Format("src/{0}", sourceStatus.SourceName),
async() =>
{
using (var stream = sourceInfo.File.GetStream())
using (var convertedStream = SourceConverter.Convert(stream))
await feed.GetSource(packageName, sourceStatus.SourceName).Put(convertedStream);
});
sourceStatus.Stored = true;
Debug.WriteLine("Stored source {0}", sourceStatus.SourceName);
}
catch (Exception e)
{
support.TrackException(e, new { packageName });
sourceStatus.Exception = new ExceptionStatus(e);
}
return(sourceStatus);
}
private async Task <SourceStatus> DeleteSource(IStorageFeed feed, PackageName packageName, SourceStatus sourceStatus)
{
var newSourceStatus = new SourceStatus(sourceStatus.SourceName);
if (sourceStatus.Stored)
{
try
{
Debug.WriteLine("Deleting source {0}", sourceStatus.SourceName);
newSourceStatus.Stored = true;
await RequestOrSkip(string.Format("src/{0}", sourceStatus.SourceName),
async() =>
{
var source = feed.GetSource(packageName, sourceStatus.SourceName);
await source.Delete();
});
newSourceStatus.Stored = false;
Debug.WriteLine("Deleted source {0}", sourceStatus.SourceName);
}
catch (Exception e)
{
support.TrackException(e, new { packageName });
newSourceStatus.Exception = new ExceptionStatus(e);
}
}
return(newSourceStatus);
}
public async Task <PackageName> ReadName(UserInfo userInfo, ZipPackage package)
{
var suffix = "-at-" + DateTime.Now.ToEncodedSeconds();
var newVersion = package.Version.ToString().AddSuffix(suffix, MaxSemanticVersionLength, '.');
var newPackageName = new PackageName(package.Id, newVersion);
return(newPackageName);
}
public async Task IndexPackage(UserInfo userInfo, IStorageFeed feed, PackageName packageName, IPackageStorageItem packageItem, ZipPackage package)
{
var statusPackageBuilder = CreatePackage(packageName, package);
var statusPackageState = PackageState.Partial;
try
{
var statusFiles = await ProcessThrottled(
2, addInfoBuilder.Build(new NugetPackage(package)),
async i => await IndexImage(feed, packageName, i));
foreach (var statusFile in statusFiles)
{
statusPackageBuilder.Files.Add(CreateFile(statusFile.FilePath, statusFile.ImageStatus));
}
if (statusFiles.All(s => s.ImageStatus.Check(true)))
{
statusPackageState = PackageState.Succeded;
}
Trace.TraceInformation("Marking package {0} as {1}", packageName, statusPackageState);
}
catch (Exception e)
{
support.TrackException(e, new { packageName });
Trace.TraceError("Error while indexing package {0}:\n{1}", packageName, e);
statusPackageBuilder.Files.Add(CreateFile("error.txt", e.ToString()));
}
if (statusPackageBuilder.Files.Count == 0)
{
statusPackageBuilder.Files.Add(CreateFile("empty.txt", string.Empty));
}
Debug.WriteLine("Saving package processing status {0}", packageName);
var statusPackageItem = feed.GetPackage(await packageItem.GetUserName(), statusPackageState, packageName);
using (var statusStream = await statusPackageItem.Put())
statusPackageBuilder.SaveBuffered(statusStream);
switch (statusPackageState)
{
case PackageState.Partial:
await notifier.PartiallyIndexed(userInfo, packageName);
break;
case PackageState.Succeded:
await notifier.Indexed(userInfo, packageName);
break;
default:
// ReSharper disable once NotResolvedInText
throw new ArgumentOutOfRangeException("statusPackageState", statusPackageState, null);
}
}
private async Task <bool> ProcessPackage(UserInfo userInfo, string feedName, PackageName packageName, PackageState packageState, PackageState damagedPackageState, Func <IPackageStorageItem, Task <IPackageStorageItem> > tryWithPackageItemBefore, Func <IPackageStorageItem, ZipPackage, Task> tryWithPackage, Func <IPackageStorageItem, Task> tryWithPackageItemAfter)
{
var feed = storage.GetFeed(feedName);
var packageItem = feed.GetPackage(null, packageState, packageName);
var traceName = packageItem.ToString();
try
{
packageItem = await tryWithPackageItemBefore(packageItem);
if (packageItem == null)
{
throw new Exception(string.Format("Missing package {0}", traceName));
}
using (var packageStream = await packageItem.Get())
{
if (packageStream == null)
{
throw new Exception(string.Format("Missing package data {0}", traceName));
}
var package = new ZipPackage(packageStream);
await tryWithPackage(packageItem, package);
}
await tryWithPackageItemAfter(packageItem);
return(true);
}
catch (Exception e)
{
support.TrackException(e, new { packageName });
Trace.TraceError("Unexpected error while processing package {0}:\n{1}", traceName, e);
notifier.Damaged(userInfo, packageName).Wait();
if (packageItem != null)
{
packageItem.Move(damagedPackageState, packageName).Wait();
}
return(false);
}
}
private async Task <SymbolStatus> IndexSymbol(IStorageFeed feed, PackageName packageName, IBinaryInfo binaryInfo)
{
var symbolStatus = new SymbolStatus(new SymbolName(binaryInfo.Name, binaryInfo.SymbolInfo.Hash));
try
{
Debug.WriteLine("Indexing symbol {0}", symbolStatus.SymbolName);
await RequestOrSkip(string.Format("pdb/{0}", symbolStatus.SymbolName),
async() =>
{
Debug.WriteLine("Storing symbol {0}", symbolStatus.SymbolName);
var symbolItem = feed.GetSymbol(packageName, symbolStatus.SymbolName);
var pdbstrSection = CreatePdbStrSection(binaryInfo);
using (var inputStream = binaryInfo.SymbolInfo.File.GetStream())
using (var tempStream = new MemoryStream())
using (var outputStream = await symbolItem.Put())
{
pdbStoreManager.WriteSrcSrv(inputStream, tempStream, pdbstrSection);
tempStream.Position = 0;
var symbolFileName = Path.GetFileName(binaryInfo.SymbolInfo.File.FullPath);
fileCompressor.Compress(symbolFileName, tempStream, outputStream);
}
});
symbolStatus.Stored = true;
Debug.WriteLine("Stored symbol {0}", symbolStatus.SymbolName);
}
catch (Exception e)
{
support.TrackException(e, new { packageName });
symbolStatus.Exception = new ExceptionStatus(e);
}
if (binaryInfo.SymbolInfo.SourceInfos != null)
{
symbolStatus.SourceStatuses = await ProcessThrottled(
5, binaryInfo.SymbolInfo.SourceInfos,
async s => await IndexSource(feed, packageName, s));
}
return(symbolStatus);
}
public async Task <IHttpActionResult> Delete(string packageId, string packageVersion)
{
var principal = (FeedPrincipal)User;
var packageName = new PackageName(packageId, packageVersion);
Trace.TraceInformation("{0} - deleting package {1}", principal, packageName);
if (!security.Authorize(principal, new[] { packageName }))
{
Trace.TraceInformation("{0} - not authorizated to delete package {1}", principal, packageName);
return(Forbidden());
}
if (!principal.Identity.IsAuthenticated)
{
throw new InvalidOperationException("Identity is required to delete packages");
}
var package = storage.GetFeed(principal.FeedName).GetPackage(null, principal.PackageState, packageName);
//if (principal.AuthenticatedArea == AuthenticatedArea.DeletingOwn && package.UserName != identity.Name)
// return NotFound();
if (principal.PackageState == PackageState.Succeded || principal.PackageState == PackageState.Partial)
{
if (await package.Move(PackageState.DeletingQueued, packageName) == null)
{
return(NotFound());
}
await scheduler.Signal(principal, PackageState.DeletingQueued, packageName);
}
else
{
if (!await package.Delete())
{
return(NotFound());
}
}
return(Ok());
}
private async Task Queue(PackageMessage message)
{
var task = new QueuePackageTask();
PackageName newPackageName = null;
// ReSharper disable RedundantArgumentName
await ProcessPackage(
message.UserInfo, message.FeedName, message.PackageName,
PackageState.New, PackageState.DamagedNew,
tryWithPackageItemBefore : async pi =>
{
Trace.TraceInformation("Reading package {0}", pi);
return(pi);
},
tryWithPackage : async(pi, p) =>
{
newPackageName = await task.ReadName(message.UserInfo, p);
await notifier.Starting(message.UserInfo, newPackageName);
},
tryWithPackageItemAfter : async pi =>
{
Trace.TraceInformation("Queueing package {0} as {1}", pi, newPackageName);
if (await pi.Copy(PackageState.Original, newPackageName) == null)
{
throw new Exception(string.Format("Failed to copy package {0} to {1}", pi, newPackageName));
}
if (await pi.Move(PackageState.IndexingQueued, newPackageName) == null)
{
throw new Exception(string.Format("Failed to move package {0} to {1}", pi, newPackageName));
}
message.PackageState = PackageState.IndexingQueued;
message.PackageName = newPackageName;
await scheduler.Signal(message);
Trace.TraceInformation("Finished queueing package {0} as {1}", pi, newPackageName);
});
// ReSharper restore RedundantArgumentName
}
public async Task <IHttpActionResult> Download(string packageId, string packageVersion)
{
var principal = (FeedPrincipal)User;
var packageName = new PackageName(packageId, packageVersion);
Trace.TraceInformation("{0} - downloading package {1}", principal, packageName);
if (!security.Authorize(principal, new[] { packageName }))
{
Trace.TraceInformation("{0} - not authorizated to download package {1}", principal, packageName);
return(Unauthorized());
}
if (principal.PackageState == PackageState.New)
{
return(BadRequest("Downloading new packages is not supported. Wait until they are at least queued."));
}
var package = storage.GetFeed(principal.FeedName).GetPackage(principal.Identity.Name, principal.PackageState, packageName);
return(await Storage(package));
}
private async Task <ImageStatusFile> IndexImage(IStorageFeed feed, PackageName packageName, IBinaryInfo binaryInfo)
{
var statusFile = new ImageStatusFile(binaryInfo.File.FullPath, new ImageStatus(new ImageName(binaryInfo.Name, binaryInfo.Hash)));
//try
//{
// //Debug.WriteLine("Indexing image {0}", imageStatus.ImageName);
// //imageStatus.Stored = true;
// //Debug.WriteLine("Stored image {0}", imageStatus.ImageName);
//}
//catch (Exception e)
//{
// support.TrackException(e);
// statusFile.ImageStatus.Exception = new ExceptionStatus(e);
//}
if (binaryInfo.SymbolInfo != null)
{
statusFile.ImageStatus.SymbolStatus = await IndexSymbol(feed, packageName, binaryInfo);
}
return(statusFile);
}
private async Task <SymbolStatus> DeleteSymbol(IStorageFeed feed, PackageName packageName, SymbolStatus symbolStatus)
{
var newSymbolStatus = new SymbolStatus(symbolStatus.SymbolName);
if (symbolStatus.Stored)
{
try
{
Debug.WriteLine("Deleting symbol {0}", symbolStatus.SymbolName);
newSymbolStatus.Stored = true;
await RequestOrSkip(string.Format("pdb/{0}", symbolStatus.SymbolName),
async() =>
{
var symbolItem = feed.GetSymbol(packageName, symbolStatus.SymbolName);
await symbolItem.Delete();
});
newSymbolStatus.Stored = false;
Debug.WriteLine("Deleted symbol {0}", symbolStatus.SymbolName);
}
catch (Exception e)
{
support.TrackException(e, new { packageName });
newSymbolStatus.Exception = new ExceptionStatus(e);
}
}
if (symbolStatus.SourceStatuses != null)
{
newSymbolStatus.SourceStatuses = await ProcessThrottled(
5, symbolStatus.SourceStatuses,
async s => await DeleteSource(feed, packageName, s));
}
return(newSymbolStatus);
}
private async Task <ImageStatusFile> DeleteImage(IStorageFeed feed, PackageName packageName, ImageStatusFile statusFile)
{
var newStatusFile = new ImageStatusFile(statusFile.FilePath, new ImageStatus(statusFile.ImageStatus.ImageName));
//try
//{
// //Debug.WriteLine("Deleting image {0}", statusFile.ImageStatus.ImageName);
// //statusFile.ImageStatus.Stored = false;
// statusFile.ImageStatus.Exception = null;
// //Debug.WriteLine("Deleted image {0}", statusFile.ImageStatus.ImageName);
//}
//catch (Exception e)
//{
// support.TrackException(e);
// newStatusFile.ImageStatus.Exception = new ExceptionStatus(e);
//}
if (statusFile.ImageStatus.SymbolStatus != null)
{
newStatusFile.ImageStatus.SymbolStatus = await DeleteSymbol(feed, packageName, statusFile.ImageStatus.SymbolStatus);
}
return(newStatusFile);
}
public async Task <IHttpActionResult> Put()
{
var principal = (FeedPrincipal)User;
Trace.TraceInformation("{0} - {1} byte(s)", principal, Request.Content.Headers.ContentLength);
if (!principal.Identity.IsAuthenticated)
{
throw new InvalidOperationException("Identity is required to put packages");
}
var provider = new StorageMultipartStreamProvider(storage, scheduler, principal);
await Request.Content.ReadAsMultipartAsync(provider);
var forbidden = 0;
var corrupted = 0;
foreach (var packageInfo in provider.Packages)
{
packageInfo.Value.Position = 0;
PackageName packageName = null;
try
{
var package = new ZipPackage(packageInfo.Value);
packageName = new PackageName(package.Id, package.Version.ToString());
}
catch (Exception e)
{
Trace.TraceInformation("{0} - corrupted package", principal);
support.TrackException(e, null);
corrupted++;
}
if (packageName == null)
{
await packageInfo.Key.Delete();
continue;
}
if (!security.Authorize(principal, new[] { packageName }))
{
Trace.TraceInformation("{0} - not authorizated to push {1}", principal, packageName);
forbidden++;
await packageInfo.Key.Delete();
continue;
}
Trace.TraceInformation("Signaling package {0} ({1})", packageInfo.Key.Name, packageName);
await scheduler.Signal(principal, PackageState.New, packageInfo.Key.Name);
}
if (corrupted > 0)
{
return(BadRequest());
}
if (forbidden > 0)
{
return(Forbidden());
}
return(Ok());
}
public async Task DeletePackage(UserInfo userInfo, IStorageFeed feed, PackageName packageName, IPackageStorageItem packageItem, IPackage package)
{
var statusPackageBuilder = CreatePackage(packageName, package);
var statusPackageState = PackageState.Partial;
try
{
var allRead = true;
var statusFiles = new List <ImageStatusFile>();
foreach (var statusPackageFile in package.GetFiles().Where(f => f.Path.EndsWith(".smbsrc")))
{
try
{
var imageStatus = ReadFile <ImageStatus>(statusPackageFile);
statusFiles.Add(new ImageStatusFile(statusPackageFile.Path, imageStatus));
}
catch (Exception e)
{
support.TrackException(e, new { packageName });
Trace.TraceError("Error while deserializing status {0}:\n{1}", packageName, e);
allRead = false;
statusPackageBuilder.Files.Add(statusPackageFile);
}
}
var newStatusFiles = await ProcessThrottled(
2, statusFiles,
async s => await DeleteImage(feed, packageName, s));
foreach (var statusFile in newStatusFiles)
{
statusPackageBuilder.Files.Add(CreateFile(statusFile.FilePath, statusFile.ImageStatus));
}
if (allRead && newStatusFiles.All(s => s.ImageStatus.Check(false)))
{
statusPackageState = PackageState.Deleted;
}
Trace.TraceInformation("Marking package {0} as {1}", packageName, statusPackageState);
}
catch (Exception e)
{
support.TrackException(e, new { packageName });
Trace.TraceError("Error while deleting package {0}:\n{1}", packageName, e);
statusPackageBuilder.Files.Add(CreateFile("error.txt", e.ToString()));
}
if (statusPackageBuilder.Files.Count == 0)
{
statusPackageBuilder.Files.Add(CreateFile("empty.txt", string.Empty));
}
Debug.WriteLine("Saving package processing status {0}", packageName);
var statusPackageItem = feed.GetPackage(await packageItem.GetUserName(), statusPackageState, packageName);
using (var statusStream = await statusPackageItem.Put())
statusPackageBuilder.SaveBuffered(statusStream);
switch (statusPackageState)
{
case PackageState.Partial:
await notifier.PartiallyDeleted(userInfo, packageName);
break;
case PackageState.Deleted:
await notifier.Deleted(userInfo, packageName);
break;
default:
// ReSharper disable once NotResolvedInText
throw new ArgumentOutOfRangeException("statusPackageState", statusPackageState, null);
}
}
