Ejemplo n.º 1
0
        /// <exception cref="System.IO.IOException"></exception>
        private void VerifyOpenPack(bool thin)
        {
            byte[] packData = os.ToByteArray();
            if (thin)
            {
                PackParser p = Index(packData);
                try
                {
                    p.Parse(NullProgressMonitor.INSTANCE);
                    NUnit.Framework.Assert.Fail("indexer should grumble about missing object");
                }
                catch (IOException)
                {
                }
            }
            // expected
            ObjectDirectoryPackParser p_1 = (ObjectDirectoryPackParser)Index(packData);

            p_1.SetKeepEmpty(true);
            p_1.SetAllowThin(thin);
            p_1.SetIndexVersion(2);
            p_1.Parse(NullProgressMonitor.INSTANCE);
            pack = p_1.GetPackFile();
            NUnit.Framework.Assert.IsNotNull(pack, "have PackFile after parsing");
        }
Ejemplo n.º 2
0
 private void InsertPack(PackFile pf)
 {
     ObjectDirectory.PackList o;
     ObjectDirectory.PackList n;
     do
     {
         o = packList.Get();
         // If the pack in question is already present in the list
         // (picked up by a concurrent thread that did a scan?) we
         // do not want to insert it a second time.
         //
         PackFile[] oldList = o.packs;
         string     name    = pf.GetPackFile().GetName();
         foreach (PackFile p in oldList)
         {
             if (PackFile.SORT.Compare(pf, p) < 0)
             {
                 break;
             }
             if (name.Equals(p.GetPackFile().GetName()))
             {
                 return;
             }
         }
         PackFile[] newList = new PackFile[1 + oldList.Length];
         newList[0] = pf;
         System.Array.Copy(oldList, 0, newList, 1, oldList.Length);
         n = new ObjectDirectory.PackList(o.snapshot, newList);
     }while (!packList.CompareAndSet(o, n));
 }
Ejemplo n.º 3
0
 /// <exception cref="System.IO.IOException"></exception>
 private ByteWindow Load(PackFile pack, long offset)
 {
     if (pack.BeginWindowCache())
     {
         openFiles.IncrementAndGet();
     }
     try
     {
         if (mmap)
         {
             return(pack.Mmap(offset, windowSize));
         }
         return(pack.Read(offset, windowSize));
     }
     catch (IOException e)
     {
         Close(pack);
         throw;
     }
     catch (RuntimeException e)
     {
         Close(pack);
         throw;
     }
     catch (Error e)
     {
         Close(pack);
         throw;
     }
 }
Ejemplo n.º 4
0
 private void Close(PackFile pack)
 {
     if (pack.EndWindowCache())
     {
         openFiles.DecrementAndGet();
     }
 }
Ejemplo n.º 5
0
 /// <summary>
 /// Inflate a region of the pack starting at
 /// <code>position</code>
 /// .
 /// </summary>
 /// <param name="pack">the file the desired window is stored within.</param>
 /// <param name="position">position within the file to read from.</param>
 /// <param name="dstbuf">
 /// destination buffer the inflater should output decompressed
 /// data to.
 /// </param>
 /// <param name="dstoff">current offset within <code>dstbuf</code> to inflate into.</param>
 /// <returns>
 /// updated <code>dstoff</code> based on the number of bytes
 /// successfully inflated into <code>dstbuf</code>.
 /// </returns>
 /// <exception cref="System.IO.IOException">
 /// this cursor does not match the provider or id and the proper
 /// window could not be acquired through the provider's cache.
 /// </exception>
 /// <exception cref="ICSharpCode.SharpZipLib.SharpZipBaseException">
 /// the inflater encountered an invalid chunk of data. Data
 /// stream corruption is likely.
 /// </exception>
 internal int Inflate(PackFile pack, long position, byte[] dstbuf, int dstoff)
 {
     PrepareInflater();
     Pin(pack, position);
     position += window.SetInput(position, inf);
     do
     {
         int n = inf.Inflate(dstbuf, dstoff, dstbuf.Length - dstoff);
         if (n == 0)
         {
             if (inf.IsNeedingInput)
             {
                 Pin(pack, position);
                 position += window.SetInput(position, inf);
             }
             else
             {
                 if (inf.IsFinished || (dstbuf.Length - dstoff) == 0)
                 {
                     return(dstoff);
                 }
                 else
                 {
                     throw new SharpZipBaseException();
                 }
             }
         }
         dstoff += n;
     }while (dstoff < dstbuf.Length);
     return(dstoff);
 }
Ejemplo n.º 6
0
 /// <summary>Clear all entries related to a single file.</summary>
 /// <remarks>
 /// Clear all entries related to a single file.
 /// <p>
 /// Typically this method is invoked during
 /// <see cref="PackFile.Close()">PackFile.Close()</see>
 /// , when we
 /// know the pack is never going to be useful to us again (for example, it no
 /// longer exists on disk). A concurrent reader loading an entry from this
 /// same pack may cause the pack to become stuck in the cache anyway.
 /// </remarks>
 /// <param name="pack">the file to purge all entries of.</param>
 private void RemoveAll(PackFile pack)
 {
     for (int s = 0; s < tableSize; s++)
     {
         WindowCache.Entry e1 = table.Get(s);
         bool hasDead         = false;
         for (WindowCache.Entry e = e1; e != null; e = e.next)
         {
             if ([email protected] == pack)
             {
                 e.Kill();
                 hasDead = true;
             }
             else
             {
                 if (e.dead)
                 {
                     hasDead = true;
                 }
             }
         }
         if (hasDead)
         {
             table.CompareAndSet(s, e1, Clean(e1));
         }
     }
     Gc();
 }
 internal virtual void Store(PackFile pack, long position, byte[] data, int objectType
                             )
 {
     if (data.Length > maxByteCount)
     {
         return;
     }
     // Too large to cache.
     DeltaBaseCache.Slot e = cache[Hash(position)];
     if (e == null)
     {
         e = new DeltaBaseCache.Slot();
         cache[Hash(position)] = e;
     }
     else
     {
         ClearEntry(e);
     }
     openByteCount += data.Length;
     ReleaseMemory();
     e.provider = pack;
     e.position = position;
     e.sz       = data.Length;
     e.data     = new SoftReference <DeltaBaseCache.Entry>(new DeltaBaseCache.Entry(data, objectType
                                                                                    ));
     MoveToHead(e);
 }
Ejemplo n.º 8
0
        private static IDictionary <string, PackFile> ReuseMap(ObjectDirectory.PackList old
                                                               )
        {
            IDictionary <string, PackFile> forReuse = new Dictionary <string, PackFile>();

            foreach (PackFile p in old.packs)
            {
                if (p.Invalid())
                {
                    // The pack instance is corrupted, and cannot be safely used
                    // again. Do not include it in our reuse map.
                    //
                    p.Close();
                    continue;
                }
                PackFile prior = forReuse.Put(p.GetPackFile().GetName(), p);
                if (prior != null)
                {
                    // This should never occur. It should be impossible for us
                    // to have two pack files with the same name, as all of them
                    // came out of the same directory. If it does, we promised to
                    // close any PackFiles we did not reuse, so close the second,
                    // readers are likely to be actively using the first.
                    //
                    forReuse.Put(prior.GetPackFile().GetName(), prior);
                    p.Close();
                }
            }
            return(forReuse);
        }
Ejemplo n.º 9
0
 protected internal Ref(PackFile pack, long position, ByteWindow v, ReferenceQueue
                        <ByteWindow> queue) : base(v, queue)
 {
     this.pack     = pack;
     this.position = position;
     this.size     = v.Size();
 }
Ejemplo n.º 10
0
        private ObjectDirectory.PackList ScanPacksImpl(ObjectDirectory.PackList old)
        {
            IDictionary <string, PackFile> forReuse = ReuseMap(old);
            FileSnapshot         snapshot           = FileSnapshot.Save(packDirectory);
            ICollection <string> names = ListPackDirectory();
            IList <PackFile>     list  = new AList <PackFile>(names.Count >> 2);
            bool foundNew = false;

            foreach (string indexName in names)
            {
                // Must match "pack-[0-9a-f]{40}.idx" to be an index.
                //
                if (indexName.Length != 49 || !indexName.EndsWith(".idx"))
                {
                    continue;
                }
                string @base    = Sharpen.Runtime.Substring(indexName, 0, indexName.Length - 4);
                string packName = @base + ".pack";
                if (!names.Contains(packName))
                {
                    // Sometimes C Git's HTTP fetch transport leaves a
                    // .idx file behind and does not download the .pack.
                    // We have to skip over such useless indexes.
                    //
                    continue;
                }
                PackFile oldPack = Sharpen.Collections.Remove(forReuse, packName);
                if (oldPack != null)
                {
                    list.AddItem(oldPack);
                    continue;
                }
                FilePath packFile = new FilePath(packDirectory, packName);
                FilePath idxFile  = new FilePath(packDirectory, indexName);
                list.AddItem(new PackFile(idxFile, packFile));
                foundNew = true;
            }
            // If we did not discover any new files, the modification time was not
            // changed, and we did not remove any files, then the set of files is
            // the same as the set we were given. Instead of building a new object
            // return the same collection.
            //
            if (!foundNew && forReuse.IsEmpty() && snapshot.Equals(old.snapshot))
            {
                old.snapshot.SetClean(snapshot);
                return(old);
            }
            foreach (PackFile p in forReuse.Values)
            {
                p.Close();
            }
            if (list.IsEmpty())
            {
                return(new ObjectDirectory.PackList(snapshot, NO_PACKS.packs));
            }
            PackFile[] r = Sharpen.Collections.ToArray(list, new PackFile[list.Count]);
            Arrays.Sort(r, PackFile.SORT);
            return(new ObjectDirectory.PackList(snapshot, r));
        }
        public override void Select(StoredObjectRepresentation @ref)
        {
            LocalObjectRepresentation ptr = (LocalObjectRepresentation)@ref;

            this.pack   = ptr.pack;
            this.offset = ptr.offset;
            this.length = ptr.length;
        }
Ejemplo n.º 12
0
 /// <exception cref="System.IO.IOException"></exception>
 internal PackInputStream(PackFile pack, long pos, WindowCursor wc)
 {
     this.pack = pack;
     this.pos  = pos;
     this.wc   = wc;
     // Pin the first window, to ensure the pack is open and valid.
     //
     wc.Pin(pack, pos);
 }
Ejemplo n.º 13
0
 /// <exception cref="System.IO.IOException"></exception>
 internal ByteArrayWindow QuickCopy(PackFile p, long pos, long cnt)
 {
     Pin(p, pos);
     if (window is ByteArrayWindow && window.Contains(p, pos + (cnt - 1)))
     {
         return((ByteArrayWindow)window);
     }
     return(null);
 }
Ejemplo n.º 14
0
 internal virtual ObjectLoader Large(PackFile pack, WindowCursor wc)
 {
     PackFile.Delta d = this;
     while (d.next != null)
     {
         d = d.next;
     }
     return(d.NewLargeLoader(pack, wc));
 }
Ejemplo n.º 15
0
 internal LargePackedWholeObject(int type, long size, long objectOffset, int headerLength
                                 , PackFile pack, FileObjectDatabase db)
 {
     this.type         = type;
     this.size         = size;
     this.objectOffset = objectOffset;
     this.headerLength = headerLength;
     this.pack         = pack;
     this.db           = db;
 }
Ejemplo n.º 16
0
        internal static LocalObjectRepresentation NewWhole(PackFile f, long p, long length
                                                           )
        {
            LocalObjectRepresentation r = new _LocalObjectRepresentation_53();

            r.pack   = f;
            r.offset = p;
            r.length = length;
            return(r);
        }
Ejemplo n.º 17
0
 internal LargePackedDeltaObject(long objectOffset, long baseOffset, int headerLength
                                 , PackFile pack, FileObjectDatabase db)
 {
     this.type         = Constants.OBJ_BAD;
     this.size         = SIZE_UNKNOWN;
     this.objectOffset = objectOffset;
     this.baseOffset   = baseOffset;
     this.headerLength = headerLength;
     this.pack         = pack;
     this.db           = db;
 }
Ejemplo n.º 18
0
        internal static LocalObjectRepresentation NewDelta(PackFile f, long p, long n, long
                                                           @base)
        {
            LocalObjectRepresentation r = new LocalObjectRepresentation.Delta();

            r.pack       = f;
            r.offset     = p;
            r.length     = n;
            r.baseOffset = @base;
            return(r);
        }
Ejemplo n.º 19
0
 private static int IndexOf(PackFile[] list, PackFile pack)
 {
     for (int i = 0; i < list.Length; i++)
     {
         if (list[i] == pack)
         {
             return(i);
         }
     }
     return(-1);
 }
Ejemplo n.º 20
0
 /// <exception cref="System.IO.FileNotFoundException"></exception>
 private PackFile[] GetPacks()
 {
     if (packs == null)
     {
         PackFile[] p = new PackFile[packNames.Length];
         for (int i = 0; i < packNames.Length; i++)
         {
             p[i] = GetPackFile(packNames[i]);
         }
         packs = p;
     }
     return(packs);
 }
Ejemplo n.º 21
0
        public virtual void TestKeepFiles()
        {
            BranchBuilder bb = tr.Branch("refs/heads/master");

            bb.Commit().Add("A", "A").Add("B", "B").Create();
            stats = gc.GetStatistics();
            NUnit.Framework.Assert.AreEqual(4, stats.numberOfLooseObjects);
            NUnit.Framework.Assert.AreEqual(0, stats.numberOfPackedObjects);
            NUnit.Framework.Assert.AreEqual(0, stats.numberOfPackFiles);
            gc.Gc();
            stats = gc.GetStatistics();
            NUnit.Framework.Assert.AreEqual(0, stats.numberOfLooseObjects);
            NUnit.Framework.Assert.AreEqual(4, stats.numberOfPackedObjects);
            NUnit.Framework.Assert.AreEqual(1, stats.numberOfPackFiles);
            Iterator <PackFile> packIt = ((ObjectDirectory)repo.ObjectDatabase).GetPacks().Iterator
                                             ();
            PackFile singlePack = packIt.Next();

            NUnit.Framework.Assert.IsFalse(packIt.HasNext());
            FilePath keepFile = new FilePath(singlePack.GetPackFile().GetPath() + ".keep");

            NUnit.Framework.Assert.IsFalse(keepFile.Exists());
            NUnit.Framework.Assert.IsTrue(keepFile.CreateNewFile());
            bb.Commit().Add("A", "A2").Add("B", "B2").Create();
            stats = gc.GetStatistics();
            NUnit.Framework.Assert.AreEqual(4, stats.numberOfLooseObjects);
            NUnit.Framework.Assert.AreEqual(4, stats.numberOfPackedObjects);
            NUnit.Framework.Assert.AreEqual(1, stats.numberOfPackFiles);
            gc.Gc();
            stats = gc.GetStatistics();
            NUnit.Framework.Assert.AreEqual(0, stats.numberOfLooseObjects);
            NUnit.Framework.Assert.AreEqual(8, stats.numberOfPackedObjects);
            NUnit.Framework.Assert.AreEqual(2, stats.numberOfPackFiles);
            // check that no object is packed twice
            Iterator <PackFile> packs = ((ObjectDirectory)repo.ObjectDatabase).GetPacks().Iterator
                                            ();
            PackIndex ind1 = packs.Next().GetIndex();

            NUnit.Framework.Assert.AreEqual(4, ind1.GetObjectCount());
            PackIndex ind2 = packs.Next().GetIndex();

            NUnit.Framework.Assert.AreEqual(4, ind2.GetObjectCount());
            foreach (PackIndex.MutableEntry e in ind1)
            {
                if (ind2.HasObject(e.ToObjectId()))
                {
                    NUnit.Framework.Assert.IsFalse(ind2.HasObject(e.ToObjectId()), "the following object is in both packfiles: "
                                                   + e.ToObjectId());
                }
            }
        }
Ejemplo n.º 22
0
 internal static void Purge(PackFile file)
 {
     lock (typeof(DeltaBaseCache))
     {
         foreach (DeltaBaseCache.Slot e in cache)
         {
             if (e.provider == file)
             {
                 ClearEntry(e);
                 Unlink(e);
             }
         }
     }
 }
Ejemplo n.º 23
0
        public virtual void Test003_lookupCompressedObject()
        {
            PackFile     pr;
            ObjectId     id;
            ObjectLoader or;

            id = ObjectId.FromString("902d5476fa249b7abc9d84c611577a81381f0327");
            pr = new PackFile(TEST_IDX, TEST_PACK);
            or = pr.Get(new WindowCursor(null), id);
            NUnit.Framework.Assert.IsNotNull(or);
            NUnit.Framework.Assert.AreEqual(Constants.OBJ_TREE, or.GetType());
            NUnit.Framework.Assert.AreEqual(35, or.GetSize());
            pr.Close();
        }
Ejemplo n.º 24
0
        /// <summary>Lookup a cached object, creating and loading it if it doesn't exist.</summary>
        /// <remarks>Lookup a cached object, creating and loading it if it doesn't exist.</remarks>
        /// <param name="pack">the pack that "contains" the cached object.</param>
        /// <param name="position">offset within <code>pack</code> of the object.</param>
        /// <returns>the object reference.</returns>
        /// <exception cref="System.IO.IOException">
        /// the object reference was not in the cache and could not be
        /// obtained by
        /// <see cref="Load(PackFile, long)">Load(PackFile, long)</see>
        /// .
        /// </exception>
        private ByteWindow GetOrLoad(PackFile pack, long position)
        {
            int slot = Slot(pack, position);

            WindowCache.Entry e1 = table.Get(slot);
            ByteWindow        v  = Scan(e1, pack, position);

            if (v != null)
            {
                return(v);
            }
            lock (LockCache(pack, position))
            {
                WindowCache.Entry e2 = table.Get(slot);
                if (e2 != e1)
                {
                    v = Scan(e2, pack, position);
                    if (v != null)
                    {
                        return(v);
                    }
                }
                v = Load(pack, position);
                WindowCache.Ref @ref = CreateRef(pack, position, v);
                Hit(@ref);
                for (; ;)
                {
                    WindowCache.Entry n = new WindowCache.Entry(Clean(e2), @ref);
                    if (table.CompareAndSet(slot, e2, n))
                    {
                        break;
                    }
                    e2 = table.Get(slot);
                }
            }
            if (evictLock.TryLock())
            {
                try
                {
                    Gc();
                    Evict();
                }
                finally
                {
                    evictLock.Unlock();
                }
            }
            return(v);
        }
Ejemplo n.º 25
0
        /// <exception cref="System.IO.IOException"></exception>
        internal void Pin(PackFile pack, long position)
        {
            ByteWindow w = window;

            if (w == null || !w.Contains(pack, position))
            {
                // If memory is low, we may need what is in our window field to
                // be cleaned up by the GC during the get for the next window.
                // So we always clear it, even though we are just going to set
                // it again.
                //
                window = null;
                window = WindowCache.Get(pack, position);
            }
        }
Ejemplo n.º 26
0
        /// <summary>Copy bytes from the window to a caller supplied buffer.</summary>
        /// <remarks>Copy bytes from the window to a caller supplied buffer.</remarks>
        /// <param name="pack">the file the desired window is stored within.</param>
        /// <param name="position">position within the file to read from.</param>
        /// <param name="dstbuf">destination buffer to copy into.</param>
        /// <param name="dstoff">offset within <code>dstbuf</code> to start copying into.</param>
        /// <param name="cnt">
        /// number of bytes to copy. This value may exceed the number of
        /// bytes remaining in the window starting at offset
        /// <code>pos</code>.
        /// </param>
        /// <returns>
        /// number of bytes actually copied; this may be less than
        /// <code>cnt</code> if <code>cnt</code> exceeded the number of bytes
        /// available.
        /// </returns>
        /// <exception cref="System.IO.IOException">
        /// this cursor does not match the provider or id and the proper
        /// window could not be acquired through the provider's cache.
        /// </exception>
        internal int Copy(PackFile pack, long position, byte[] dstbuf, int dstoff, int cnt
                          )
        {
            long length = pack.length;
            int  need   = cnt;

            while (need > 0 && position < length)
            {
                Pin(pack, position);
                int r = window.Copy(position, dstbuf, dstoff, need);
                position += r;
                dstoff   += r;
                need     -= r;
            }
            return(cnt - need);
        }
Ejemplo n.º 27
0
        /// <exception cref="System.IO.IOException"></exception>
        internal static ByteWindow Get(PackFile pack, long offset)
        {
            NGit.Storage.File.WindowCache c = cache;
            ByteWindow r = c.GetOrLoad(pack, c.ToStart(offset));

            if (c != cache)
            {
                // The cache was reconfigured while we were using the old one
                // to load this window. The window is still valid, but our
                // cache may think its still live. Ensure the window is removed
                // from the old cache so resources can be released.
                //
                c.RemoveAll();
            }
            return(r);
        }
Ejemplo n.º 28
0
        /// <exception cref="System.IO.IOException"></exception>
        internal void CopyPackAsIs(PackFile pack, long length, bool validate, PackOutputStream
                                   @out)
        {
            MessageDigest md = null;

            if (validate)
            {
                md = Constants.NewMessageDigest();
                byte[] buf = @out.GetCopyBuffer();
                Pin(pack, 0);
                if (window.Copy(0, buf, 0, 12) != 12)
                {
                    pack.SetInvalid();
                    throw new IOException(JGitText.Get().packfileIsTruncated);
                }
                md.Update(buf, 0, 12);
            }
            long position  = 12;
            long remaining = length - (12 + 20);

            while (0 < remaining)
            {
                Pin(pack, position);
                int ptr = (int)(position - window.start);
                int n   = (int)Math.Min(window.Size() - ptr, remaining);
                window.Write(@out, position, n, md);
                position  += n;
                remaining -= n;
            }
            if (md != null)
            {
                byte[] buf     = new byte[20];
                byte[] actHash = md.Digest();
                Pin(pack, position);
                if (window.Copy(position, buf, 0, 20) != 20)
                {
                    pack.SetInvalid();
                    throw new IOException(JGitText.Get().packfileIsTruncated);
                }
                if (!Arrays.Equals(actHash, buf))
                {
                    pack.SetInvalid();
                    throw new IOException(MessageFormat.Format(JGitText.Get().packfileCorruptionDetected
                                                               , pack.GetPackFile().GetPath()));
                }
            }
        }
Ejemplo n.º 29
0
 internal static DeltaBaseCache.Entry Get(PackFile pack, long position)
 {
     lock (typeof(DeltaBaseCache))
     {
         DeltaBaseCache.Slot e = cache[Hash(position)];
         if (e.provider == pack && e.position == position)
         {
             DeltaBaseCache.Entry buf = e.data.Get();
             if (buf != null)
             {
                 MoveToHead(e);
                 return(buf);
             }
         }
         return(null);
     }
 }
 internal virtual DeltaBaseCache.Entry Get(PackFile pack, long position)
 {
     DeltaBaseCache.Slot e = cache[Hash(position)];
     if (e == null)
     {
         return(null);
     }
     if (e.provider == pack && e.position == position)
     {
         DeltaBaseCache.Entry buf = e.data.Get();
         if (buf != null)
         {
             MoveToHead(e);
             return(buf);
         }
     }
     return(null);
 }