/// <exception cref="System.IO.IOException"></exception>
private void VerifyOpenPack(bool thin)
{
byte[] packData = os.ToByteArray();
if (thin)
{
PackParser p = Index(packData);
try
{
p.Parse(NullProgressMonitor.INSTANCE);
NUnit.Framework.Assert.Fail("indexer should grumble about missing object");
}
catch (IOException)
{
}
}
// expected
ObjectDirectoryPackParser p_1 = (ObjectDirectoryPackParser)Index(packData);
p_1.SetKeepEmpty(true);
p_1.SetAllowThin(thin);
p_1.SetIndexVersion(2);
p_1.Parse(NullProgressMonitor.INSTANCE);
pack = p_1.GetPackFile();
NUnit.Framework.Assert.IsNotNull(pack, "have PackFile after parsing");
}
private void InsertPack(PackFile pf)
{
ObjectDirectory.PackList o;
ObjectDirectory.PackList n;
do
{
o = packList.Get();
// If the pack in question is already present in the list
// (picked up by a concurrent thread that did a scan?) we
// do not want to insert it a second time.
//
PackFile[] oldList = o.packs;
string name = pf.GetPackFile().GetName();
foreach (PackFile p in oldList)
{
if (PackFile.SORT.Compare(pf, p) < 0)
{
break;
}
if (name.Equals(p.GetPackFile().GetName()))
{
return;
}
}
PackFile[] newList = new PackFile[1 + oldList.Length];
newList[0] = pf;
System.Array.Copy(oldList, 0, newList, 1, oldList.Length);
n = new ObjectDirectory.PackList(o.snapshot, newList);
}while (!packList.CompareAndSet(o, n));
}
/// <exception cref="System.IO.IOException"></exception>
private ByteWindow Load(PackFile pack, long offset)
{
if (pack.BeginWindowCache())
{
openFiles.IncrementAndGet();
}
try
{
if (mmap)
{
return(pack.Mmap(offset, windowSize));
}
return(pack.Read(offset, windowSize));
}
catch (IOException e)
{
Close(pack);
throw;
}
catch (RuntimeException e)
{
Close(pack);
throw;
}
catch (Error e)
{
Close(pack);
throw;
}
}
private void Close(PackFile pack)
{
if (pack.EndWindowCache())
{
openFiles.DecrementAndGet();
}
}
/// <summary>
/// Inflate a region of the pack starting at
/// <code>position</code>
/// .
/// </summary>
/// <param name="pack">the file the desired window is stored within.</param>
/// <param name="position">position within the file to read from.</param>
/// <param name="dstbuf">
/// destination buffer the inflater should output decompressed
/// data to.
/// </param>
/// <param name="dstoff">current offset within <code>dstbuf</code> to inflate into.</param>
/// <returns>
/// updated <code>dstoff</code> based on the number of bytes
/// successfully inflated into <code>dstbuf</code>.
/// </returns>
/// <exception cref="System.IO.IOException">
/// this cursor does not match the provider or id and the proper
/// window could not be acquired through the provider's cache.
/// </exception>
/// <exception cref="ICSharpCode.SharpZipLib.SharpZipBaseException">
/// the inflater encountered an invalid chunk of data. Data
/// stream corruption is likely.
/// </exception>
internal int Inflate(PackFile pack, long position, byte[] dstbuf, int dstoff)
{
PrepareInflater();
Pin(pack, position);
position += window.SetInput(position, inf);
do
{
int n = inf.Inflate(dstbuf, dstoff, dstbuf.Length - dstoff);
if (n == 0)
{
if (inf.IsNeedingInput)
{
Pin(pack, position);
position += window.SetInput(position, inf);
}
else
{
if (inf.IsFinished || (dstbuf.Length - dstoff) == 0)
{
return(dstoff);
}
else
{
throw new SharpZipBaseException();
}
}
}
dstoff += n;
}while (dstoff < dstbuf.Length);
return(dstoff);
}
/// <summary>Clear all entries related to a single file.</summary>
/// <remarks>
/// Clear all entries related to a single file.
/// <p>
/// Typically this method is invoked during
/// <see cref="PackFile.Close()">PackFile.Close()</see>
/// , when we
/// know the pack is never going to be useful to us again (for example, it no
/// longer exists on disk). A concurrent reader loading an entry from this
/// same pack may cause the pack to become stuck in the cache anyway.
/// </remarks>
/// <param name="pack">the file to purge all entries of.</param>
private void RemoveAll(PackFile pack)
{
for (int s = 0; s < tableSize; s++)
{
WindowCache.Entry e1 = table.Get(s);
bool hasDead = false;
for (WindowCache.Entry e = e1; e != null; e = e.next)
{
if ([email protected] == pack)
{
e.Kill();
hasDead = true;
}
else
{
if (e.dead)
{
hasDead = true;
}
}
}
if (hasDead)
{
table.CompareAndSet(s, e1, Clean(e1));
}
}
Gc();
}
internal virtual void Store(PackFile pack, long position, byte[] data, int objectType
)
{
if (data.Length > maxByteCount)
{
return;
}
// Too large to cache.
DeltaBaseCache.Slot e = cache[Hash(position)];
if (e == null)
{
e = new DeltaBaseCache.Slot();
cache[Hash(position)] = e;
}
else
{
ClearEntry(e);
}
openByteCount += data.Length;
ReleaseMemory();
e.provider = pack;
e.position = position;
e.sz = data.Length;
e.data = new SoftReference <DeltaBaseCache.Entry>(new DeltaBaseCache.Entry(data, objectType
));
MoveToHead(e);
}
private static IDictionary <string, PackFile> ReuseMap(ObjectDirectory.PackList old
)
{
IDictionary <string, PackFile> forReuse = new Dictionary <string, PackFile>();
foreach (PackFile p in old.packs)
{
if (p.Invalid())
{
// The pack instance is corrupted, and cannot be safely used
// again. Do not include it in our reuse map.
//
p.Close();
continue;
}
PackFile prior = forReuse.Put(p.GetPackFile().GetName(), p);
if (prior != null)
{
// This should never occur. It should be impossible for us
// to have two pack files with the same name, as all of them
// came out of the same directory. If it does, we promised to
// close any PackFiles we did not reuse, so close the second,
// readers are likely to be actively using the first.
//
forReuse.Put(prior.GetPackFile().GetName(), prior);
p.Close();
}
}
return(forReuse);
}
protected internal Ref(PackFile pack, long position, ByteWindow v, ReferenceQueue
<ByteWindow> queue) : base(v, queue)
{
this.pack = pack;
this.position = position;
this.size = v.Size();
}
private ObjectDirectory.PackList ScanPacksImpl(ObjectDirectory.PackList old)
{
IDictionary <string, PackFile> forReuse = ReuseMap(old);
FileSnapshot snapshot = FileSnapshot.Save(packDirectory);
ICollection <string> names = ListPackDirectory();
IList <PackFile> list = new AList <PackFile>(names.Count >> 2);
bool foundNew = false;
foreach (string indexName in names)
{
// Must match "pack-[0-9a-f]{40}.idx" to be an index.
//
if (indexName.Length != 49 || !indexName.EndsWith(".idx"))
{
continue;
}
string @base = Sharpen.Runtime.Substring(indexName, 0, indexName.Length - 4);
string packName = @base + ".pack";
if (!names.Contains(packName))
{
// Sometimes C Git's HTTP fetch transport leaves a
// .idx file behind and does not download the .pack.
// We have to skip over such useless indexes.
//
continue;
}
PackFile oldPack = Sharpen.Collections.Remove(forReuse, packName);
if (oldPack != null)
{
list.AddItem(oldPack);
continue;
}
FilePath packFile = new FilePath(packDirectory, packName);
FilePath idxFile = new FilePath(packDirectory, indexName);
list.AddItem(new PackFile(idxFile, packFile));
foundNew = true;
}
// If we did not discover any new files, the modification time was not
// changed, and we did not remove any files, then the set of files is
// the same as the set we were given. Instead of building a new object
// return the same collection.
//
if (!foundNew && forReuse.IsEmpty() && snapshot.Equals(old.snapshot))
{
old.snapshot.SetClean(snapshot);
return(old);
}
foreach (PackFile p in forReuse.Values)
{
p.Close();
}
if (list.IsEmpty())
{
return(new ObjectDirectory.PackList(snapshot, NO_PACKS.packs));
}
PackFile[] r = Sharpen.Collections.ToArray(list, new PackFile[list.Count]);
Arrays.Sort(r, PackFile.SORT);
return(new ObjectDirectory.PackList(snapshot, r));
}
public override void Select(StoredObjectRepresentation @ref)
{
LocalObjectRepresentation ptr = (LocalObjectRepresentation)@ref;
this.pack = ptr.pack;
this.offset = ptr.offset;
this.length = ptr.length;
}
/// <exception cref="System.IO.IOException"></exception>
internal PackInputStream(PackFile pack, long pos, WindowCursor wc)
{
this.pack = pack;
this.pos = pos;
this.wc = wc;
// Pin the first window, to ensure the pack is open and valid.
//
wc.Pin(pack, pos);
}
/// <exception cref="System.IO.IOException"></exception>
internal ByteArrayWindow QuickCopy(PackFile p, long pos, long cnt)
{
Pin(p, pos);
if (window is ByteArrayWindow && window.Contains(p, pos + (cnt - 1)))
{
return((ByteArrayWindow)window);
}
return(null);
}
internal virtual ObjectLoader Large(PackFile pack, WindowCursor wc)
{
PackFile.Delta d = this;
while (d.next != null)
{
d = d.next;
}
return(d.NewLargeLoader(pack, wc));
}
internal LargePackedWholeObject(int type, long size, long objectOffset, int headerLength
, PackFile pack, FileObjectDatabase db)
{
this.type = type;
this.size = size;
this.objectOffset = objectOffset;
this.headerLength = headerLength;
this.pack = pack;
this.db = db;
}
internal static LocalObjectRepresentation NewWhole(PackFile f, long p, long length
)
{
LocalObjectRepresentation r = new _LocalObjectRepresentation_53();
r.pack = f;
r.offset = p;
r.length = length;
return(r);
}
internal LargePackedDeltaObject(long objectOffset, long baseOffset, int headerLength
, PackFile pack, FileObjectDatabase db)
{
this.type = Constants.OBJ_BAD;
this.size = SIZE_UNKNOWN;
this.objectOffset = objectOffset;
this.baseOffset = baseOffset;
this.headerLength = headerLength;
this.pack = pack;
this.db = db;
}
internal static LocalObjectRepresentation NewDelta(PackFile f, long p, long n, long
@base)
{
LocalObjectRepresentation r = new LocalObjectRepresentation.Delta();
r.pack = f;
r.offset = p;
r.length = n;
r.baseOffset = @base;
return(r);
}
private static int IndexOf(PackFile[] list, PackFile pack)
{
for (int i = 0; i < list.Length; i++)
{
if (list[i] == pack)
{
return(i);
}
}
return(-1);
}
/// <exception cref="System.IO.FileNotFoundException"></exception>
private PackFile[] GetPacks()
{
if (packs == null)
{
PackFile[] p = new PackFile[packNames.Length];
for (int i = 0; i < packNames.Length; i++)
{
p[i] = GetPackFile(packNames[i]);
}
packs = p;
}
return(packs);
}
public virtual void TestKeepFiles()
{
BranchBuilder bb = tr.Branch("refs/heads/master");
bb.Commit().Add("A", "A").Add("B", "B").Create();
stats = gc.GetStatistics();
NUnit.Framework.Assert.AreEqual(4, stats.numberOfLooseObjects);
NUnit.Framework.Assert.AreEqual(0, stats.numberOfPackedObjects);
NUnit.Framework.Assert.AreEqual(0, stats.numberOfPackFiles);
gc.Gc();
stats = gc.GetStatistics();
NUnit.Framework.Assert.AreEqual(0, stats.numberOfLooseObjects);
NUnit.Framework.Assert.AreEqual(4, stats.numberOfPackedObjects);
NUnit.Framework.Assert.AreEqual(1, stats.numberOfPackFiles);
Iterator <PackFile> packIt = ((ObjectDirectory)repo.ObjectDatabase).GetPacks().Iterator
();
PackFile singlePack = packIt.Next();
NUnit.Framework.Assert.IsFalse(packIt.HasNext());
FilePath keepFile = new FilePath(singlePack.GetPackFile().GetPath() + ".keep");
NUnit.Framework.Assert.IsFalse(keepFile.Exists());
NUnit.Framework.Assert.IsTrue(keepFile.CreateNewFile());
bb.Commit().Add("A", "A2").Add("B", "B2").Create();
stats = gc.GetStatistics();
NUnit.Framework.Assert.AreEqual(4, stats.numberOfLooseObjects);
NUnit.Framework.Assert.AreEqual(4, stats.numberOfPackedObjects);
NUnit.Framework.Assert.AreEqual(1, stats.numberOfPackFiles);
gc.Gc();
stats = gc.GetStatistics();
NUnit.Framework.Assert.AreEqual(0, stats.numberOfLooseObjects);
NUnit.Framework.Assert.AreEqual(8, stats.numberOfPackedObjects);
NUnit.Framework.Assert.AreEqual(2, stats.numberOfPackFiles);
// check that no object is packed twice
Iterator <PackFile> packs = ((ObjectDirectory)repo.ObjectDatabase).GetPacks().Iterator
();
PackIndex ind1 = packs.Next().GetIndex();
NUnit.Framework.Assert.AreEqual(4, ind1.GetObjectCount());
PackIndex ind2 = packs.Next().GetIndex();
NUnit.Framework.Assert.AreEqual(4, ind2.GetObjectCount());
foreach (PackIndex.MutableEntry e in ind1)
{
if (ind2.HasObject(e.ToObjectId()))
{
NUnit.Framework.Assert.IsFalse(ind2.HasObject(e.ToObjectId()), "the following object is in both packfiles: "
+ e.ToObjectId());
}
}
}
internal static void Purge(PackFile file)
{
lock (typeof(DeltaBaseCache))
{
foreach (DeltaBaseCache.Slot e in cache)
{
if (e.provider == file)
{
ClearEntry(e);
Unlink(e);
}
}
}
}
public virtual void Test003_lookupCompressedObject()
{
PackFile pr;
ObjectId id;
ObjectLoader or;
id = ObjectId.FromString("902d5476fa249b7abc9d84c611577a81381f0327");
pr = new PackFile(TEST_IDX, TEST_PACK);
or = pr.Get(new WindowCursor(null), id);
NUnit.Framework.Assert.IsNotNull(or);
NUnit.Framework.Assert.AreEqual(Constants.OBJ_TREE, or.GetType());
NUnit.Framework.Assert.AreEqual(35, or.GetSize());
pr.Close();
}
/// <summary>Lookup a cached object, creating and loading it if it doesn't exist.</summary>
/// <remarks>Lookup a cached object, creating and loading it if it doesn't exist.</remarks>
/// <param name="pack">the pack that "contains" the cached object.</param>
/// <param name="position">offset within <code>pack</code> of the object.</param>
/// <returns>the object reference.</returns>
/// <exception cref="System.IO.IOException">
/// the object reference was not in the cache and could not be
/// obtained by
/// <see cref="Load(PackFile, long)">Load(PackFile, long)</see>
/// .
/// </exception>
private ByteWindow GetOrLoad(PackFile pack, long position)
{
int slot = Slot(pack, position);
WindowCache.Entry e1 = table.Get(slot);
ByteWindow v = Scan(e1, pack, position);
if (v != null)
{
return(v);
}
lock (LockCache(pack, position))
{
WindowCache.Entry e2 = table.Get(slot);
if (e2 != e1)
{
v = Scan(e2, pack, position);
if (v != null)
{
return(v);
}
}
v = Load(pack, position);
WindowCache.Ref @ref = CreateRef(pack, position, v);
Hit(@ref);
for (; ;)
{
WindowCache.Entry n = new WindowCache.Entry(Clean(e2), @ref);
if (table.CompareAndSet(slot, e2, n))
{
break;
}
e2 = table.Get(slot);
}
}
if (evictLock.TryLock())
{
try
{
Gc();
Evict();
}
finally
{
evictLock.Unlock();
}
}
return(v);
}
/// <exception cref="System.IO.IOException"></exception>
internal void Pin(PackFile pack, long position)
{
ByteWindow w = window;
if (w == null || !w.Contains(pack, position))
{
// If memory is low, we may need what is in our window field to
// be cleaned up by the GC during the get for the next window.
// So we always clear it, even though we are just going to set
// it again.
//
window = null;
window = WindowCache.Get(pack, position);
}
}
/// <summary>Copy bytes from the window to a caller supplied buffer.</summary>
/// <remarks>Copy bytes from the window to a caller supplied buffer.</remarks>
/// <param name="pack">the file the desired window is stored within.</param>
/// <param name="position">position within the file to read from.</param>
/// <param name="dstbuf">destination buffer to copy into.</param>
/// <param name="dstoff">offset within <code>dstbuf</code> to start copying into.</param>
/// <param name="cnt">
/// number of bytes to copy. This value may exceed the number of
/// bytes remaining in the window starting at offset
/// <code>pos</code>.
/// </param>
/// <returns>
/// number of bytes actually copied; this may be less than
/// <code>cnt</code> if <code>cnt</code> exceeded the number of bytes
/// available.
/// </returns>
/// <exception cref="System.IO.IOException">
/// this cursor does not match the provider or id and the proper
/// window could not be acquired through the provider's cache.
/// </exception>
internal int Copy(PackFile pack, long position, byte[] dstbuf, int dstoff, int cnt
)
{
long length = pack.length;
int need = cnt;
while (need > 0 && position < length)
{
Pin(pack, position);
int r = window.Copy(position, dstbuf, dstoff, need);
position += r;
dstoff += r;
need -= r;
}
return(cnt - need);
}
/// <exception cref="System.IO.IOException"></exception>
internal static ByteWindow Get(PackFile pack, long offset)
{
NGit.Storage.File.WindowCache c = cache;
ByteWindow r = c.GetOrLoad(pack, c.ToStart(offset));
if (c != cache)
{
// The cache was reconfigured while we were using the old one
// to load this window. The window is still valid, but our
// cache may think its still live. Ensure the window is removed
// from the old cache so resources can be released.
//
c.RemoveAll();
}
return(r);
}
/// <exception cref="System.IO.IOException"></exception>
internal void CopyPackAsIs(PackFile pack, long length, bool validate, PackOutputStream
@out)
{
MessageDigest md = null;
if (validate)
{
md = Constants.NewMessageDigest();
byte[] buf = @out.GetCopyBuffer();
Pin(pack, 0);
if (window.Copy(0, buf, 0, 12) != 12)
{
pack.SetInvalid();
throw new IOException(JGitText.Get().packfileIsTruncated);
}
md.Update(buf, 0, 12);
}
long position = 12;
long remaining = length - (12 + 20);
while (0 < remaining)
{
Pin(pack, position);
int ptr = (int)(position - window.start);
int n = (int)Math.Min(window.Size() - ptr, remaining);
window.Write(@out, position, n, md);
position += n;
remaining -= n;
}
if (md != null)
{
byte[] buf = new byte[20];
byte[] actHash = md.Digest();
Pin(pack, position);
if (window.Copy(position, buf, 0, 20) != 20)
{
pack.SetInvalid();
throw new IOException(JGitText.Get().packfileIsTruncated);
}
if (!Arrays.Equals(actHash, buf))
{
pack.SetInvalid();
throw new IOException(MessageFormat.Format(JGitText.Get().packfileCorruptionDetected
, pack.GetPackFile().GetPath()));
}
}
}
internal static DeltaBaseCache.Entry Get(PackFile pack, long position)
{
lock (typeof(DeltaBaseCache))
{
DeltaBaseCache.Slot e = cache[Hash(position)];
if (e.provider == pack && e.position == position)
{
DeltaBaseCache.Entry buf = e.data.Get();
if (buf != null)
{
MoveToHead(e);
return(buf);
}
}
return(null);
}
}
internal virtual DeltaBaseCache.Entry Get(PackFile pack, long position)
{
DeltaBaseCache.Slot e = cache[Hash(position)];
if (e == null)
{
return(null);
}
if (e.provider == pack && e.position == position)
{
DeltaBaseCache.Entry buf = e.data.Get();
if (buf != null)
{
MoveToHead(e);
return(buf);
}
}
return(null);
}
