상속: ICheckoutOrderable
예제 #1
0
        public AlterationVM(Alteration alteration, Area area, Version version)
        {
            _alteration = alteration;
            _area = area;
            _version = version;

            if (_alteration.PriorRecord.HasValue)
                _priorRecord = _area.GetRecord(_alteration.PriorRecord.Value);
            if (_alteration.NewRecord.HasValue)
                _newRecord = _area.GetRecord(_alteration.NewRecord.Value);

            DiffWithPreviousCommand = new DelegateCommand(DiffWithPrevious, CanDiffWithPrevious);
            DiffWithCurrentCommand = new DelegateCommand(DiffWithCurrent, CanDiffWithCurrent);
            LogCommand = new DelegateCommand(Log);
            SaveVersionAsCommand = new DelegateCommand(SaveVersionAs, CanSaveVersionAs);
        }
예제 #2
0
 public static void CheckRecord(SharedNetworkInfo sharedInfo, Record record)
 {
     if (record == null)
         return;
     sharedInfo.RemoteRecordMap[record.Id] = record;
     if (!sharedInfo.UnknownRecordSet.Contains(record.Id))
     {
         var localRecord = sharedInfo.Workspace.LocateRecord(record);
         if (localRecord == null)
         {
             Printer.PrintDiagnostics("Received version dependent on missing record {0}", record.UniqueIdentifier);
             sharedInfo.UnknownRecords.Add(record.Id);
             sharedInfo.UnknownRecordSet.Add(record.Id);
         }
         else
         {
             localRecord.CanonicalName = record.CanonicalName;
             sharedInfo.LocalRecordMap[record.Id] = localRecord;
         }
     }
 }
예제 #3
0
 public override void WriteRecordStream(Record record, System.IO.Stream outputStream)
 {
     string lookup = GetLookup(record);
     var storeData = ObjectDatabase.Find<FileObjectStoreData>(lookup);
     WriteRecordStream(storeData, outputStream);
 }
예제 #4
0
 private Stream GetDataStream(Record record)
 {
     string lookup = GetLookup(record);
     var storeData = ObjectDatabase.Find<FileObjectStoreData>(lookup);
     return OpenLegacyStream(storeData);
     throw new Exception();
 }
예제 #5
0
 public override bool TransmitRecordData(Record record, Func<byte[], int, bool, bool> sender, byte[] scratchBuffer, Action beginTransmission = null)
 {
     if (!record.HasData)
     {
         return true;
     }
     long dataSize = GetTransmissionLength(record);
     if (dataSize == -1)
         return false;
     using (System.IO.Stream dataStream = GetDataStream(record))
     {
         if (dataStream == null)
             return false;
         if (beginTransmission != null)
             beginTransmission();
         sender(BitConverter.GetBytes(dataSize), 8, false);
         while (true)
         {
             var readCount = dataStream.Read(scratchBuffer, 0, scratchBuffer.Length);
             if (readCount == 0)
                 break;
             sender(scratchBuffer, readCount, false);
         }
     }
     return true;
 }
예제 #6
0
 public override long GetTransmissionLength(Record record)
 {
     if (!record.HasData)
         return 0;
     string lookup = GetLookup(record);
     var storeData = ObjectDatabase.Find<FileObjectStoreData>(lookup);
     if (storeData == null)
         return -1;
     return GetTransmissionLengthInternal(storeData);
 }
예제 #7
0
 public abstract long GetTransmissionLength(Record record);
예제 #8
0
 private Record DeserializeRecord(BinaryReader br)
 {
     Record rec = new Record();
     rec.Id = br.ReadInt64();
     long parent = br.ReadInt64();
     if (parent != -1)
         rec.Parent = parent;
     rec.Size = br.ReadInt64();
     rec.Attributes = (Attributes)br.ReadUInt32();
     rec.Fingerprint = br.ReadString();
     rec.CanonicalNameId = br.ReadInt64();
     rec.ModificationTime = new DateTime(br.ReadInt64());
     rec.CanonicalName = br.ReadString();
     return rec;
 }
예제 #9
0
 private void ImportRecordFromFlatStore(Record x)
 {
     if (x.HasData)
     {
         if (!CheckFileForDataIDExists(x.DataIdentifier))
             return;
         var recordData = new FileObjectStoreData();
         recordData.FileSize = x.Size;
         recordData.HasSignatureData = false;
         recordData.Lookup = x.DataIdentifier;
         recordData.Mode = StorageMode.Legacy;
         recordData.Offset = 0;
         recordData.PackFileID = null;
         try
         {
             ObjectDatabase.InsertSafe(recordData);
         }
         catch (SQLite.SQLiteException e)
         {
             if (e.Result != SQLite.SQLite3.Result.Constraint)
                 throw;
         }
     }
 }
예제 #10
0
 internal override RecordInfo GetInfo(Record x)
 {
     if (!x.HasData)
     {
         return new RecordInfo()
         {
             AllocatedSize = 0,
             DeltaCompressed = false,
             ID = -1,
         };
     }
     string lookup = GetLookup(x);
     var storeData = ObjectDatabase.Find<FileObjectStoreData>(lookup);
     if (storeData == null)
         return null;
     return new RecordInfo()
     {
         AllocatedSize = GetTransmissionLengthInternal(storeData),
         DeltaCompressed = storeData.Mode == StorageMode.Delta,
         ID = storeData.ID
     };
 }
예제 #11
0
 private string GetLookup(Record record)
 {
     return record.Fingerprint + "-" + record.Size.ToString();
 }
예제 #12
0
        private List <Record> Consolidate(List <Record> baseList, List <Alteration> alterations, List <Record> deletions)
        {
            System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
            sw.Start();
            Dictionary <long, Record> records = new Dictionary <long, Record>();

            foreach (var x in baseList)
            {
                records[x.Id] = x;
            }

#if FULL_CONSOLIDATE_DEBUG
            Printer.PrintMessage("Initial Snapshot Contains: ");
            foreach (var x in baseList)
            {
                Printer.PrintMessage("Initial: ({2}) {0} - {1}", x.CanonicalName, x.UniqueIdentifier, x.Id);
            }
#endif

            List <long> pending = new List <long>();
            CacheRecords(alterations.Select(x => x.NewRecord).Where(x => x.HasValue).Select(x => x.Value));
            foreach (var x in alterations.Select(x => x).Reverse())
            {
                Objects.Record rec = null;
                switch (x.Type)
                {
                case AlterationType.Add:
                case AlterationType.Copy:
                {
                    var record = GetCachedRecord(x.NewRecord.Value);
#if FULL_CONSOLIDATE_DEBUG
                    Printer.PrintMessage("Add: ({2}) {0} - {1}", record.CanonicalName, record.UniqueIdentifier, record.Id);
                    if (records.ContainsKey(record.Id))
                    {
                        Printer.PrintMessage(" (error, already present)");
                    }
#endif
                    records[record.Id] = record;
                    break;
                }

                case AlterationType.Move:
                {
                    var record = GetCachedRecord(x.NewRecord.Value);
                    if (deletions != null)
                    {
                        rec = Get <Objects.Record>(x.PriorRecord);
                        rec.CanonicalName = Get <Objects.ObjectName>(rec.CanonicalNameId).CanonicalName;
                        deletions.Add(rec);
                    }
#if FULL_CONSOLIDATE_DEBUG
                    rec = Get <Objects.Record>(x.PriorRecord);
                    rec.CanonicalName = Get <Objects.ObjectName>(rec.CanonicalNameId).CanonicalName;
                    Printer.PrintMessage("Move (from): ({2}) {0} - {1}", rec.CanonicalName, rec.UniqueIdentifier, rec.Id);
                    if (!records.ContainsKey(rec.Id))
                    {
                        Printer.PrintMessage(" (error, not present)");
                    }
                    Printer.PrintMessage("Move (to): ({2}) {0} - {1}", record.CanonicalName, record.UniqueIdentifier, record.Id);
                    if (records.ContainsKey(record.Id))
                    {
                        Printer.PrintMessage(" (error, already present)");
                    }
#endif
                    if (!records.Remove(x.PriorRecord.Value))
                    {
                        throw new Exception("Consistency constraint invalid!");
                    }
                    records[record.Id] = record;
                    break;
                }

                case AlterationType.Update:
                {
                    var record = GetCachedRecord(x.NewRecord.Value);
#if FULL_CONSOLIDATE_DEBUG
                    rec = Get <Objects.Record>(x.PriorRecord);
                    rec.CanonicalName = Get <Objects.ObjectName>(rec.CanonicalNameId).CanonicalName;
                    Printer.PrintMessage("Update (from): ({2}) {0} - {1}", rec.CanonicalName, rec.UniqueIdentifier, rec.Id);
                    if (!records.ContainsKey(rec.Id))
                    {
                        Printer.PrintMessage(" (error, not present)");
                    }
                    Printer.PrintMessage("Update (to): ({2}) {0} - {1}", record.CanonicalName, record.UniqueIdentifier, record.Id);
                    if (records.ContainsKey(record.Id))
                    {
                        Printer.PrintMessage(" (error, already present)");
                    }
#endif
                    if (!records.Remove(x.PriorRecord.Value))
                    {
                        throw new Exception();
                    }
                    records[record.Id] = record;
                    break;
                }

                case AlterationType.Discard:
                    break;

                case AlterationType.Delete:
                    if (deletions != null)
                    {
                        rec = Get <Objects.Record>(x.PriorRecord);
                        rec.CanonicalName = Get <Objects.ObjectName>(rec.CanonicalNameId).CanonicalName;
                        deletions.Add(rec);
                    }
#if FULL_CONSOLIDATE_DEBUG
                    rec = Get <Objects.Record>(x.PriorRecord);
                    rec.CanonicalName = Get <Objects.ObjectName>(rec.CanonicalNameId).CanonicalName;
                    Printer.PrintMessage("Delete: ({2}) {0} - {1}", rec.CanonicalName, rec.UniqueIdentifier, rec.Id);
                    if (!records.ContainsKey(rec.Id))
                    {
                        Printer.PrintMessage(" (error, not present)");
                    }
#endif
                    if (!records.Remove(x.PriorRecord.Value))
                    {
                        throw new Exception("Consistency constraint invalid!");
                    }
                    break;

                default:
                    throw new Exception();
                }
            }
            var result = records.Select(x => x.Value).ToList();
#if DEBUG
            HashSet <string> namecheck = new HashSet <string>();
            foreach (var x in result)
            {
                if (namecheck.Contains(x.CanonicalName))
                {
                    throw new Exception("Inconsistency in internal state!");
                }
                namecheck.Add(x.CanonicalName);
            }
#endif
            return(result);
        }
예제 #13
0
 public virtual bool HasData(Record recordInfo)
 {
     List<string> ignored;
     return HasData(recordInfo, out ignored);
 }
예제 #14
0
 internal abstract RecordInfo GetInfo(Record x);
예제 #15
0
 public abstract void WriteRecordStream(Record rec, System.IO.Stream outputStream);
예제 #16
0
 public bool DataEquals(Record other)
 {
     return other.Size == Size && other.Fingerprint == Fingerprint;
 }
예제 #17
0
        public override bool RecordData(ObjectStoreTransaction transaction, Record newRecord, Record priorRecord, Entry fileEntry)
        {
            StandardObjectStoreTransaction trans = (StandardObjectStoreTransaction)transaction;
            lock (trans)
            {
                if (trans.Inputs.Contains(GetLookup(newRecord)))
                    return true;
                string filename;
                lock (this)
                {
                    if (HasData(newRecord))
                        return true;
                    do
                    {
                        filename = Path.GetRandomFileName();
                    } while (TempFiles.Contains(filename));
                    TempFiles.Add(filename);
                }
                trans.Inputs.Add(GetLookup(newRecord));
                trans.m_PendingCount++;
                trans.m_PendingBytes += newRecord.Size;
                Printer.PrintDiagnostics("Processing {0}", fileEntry.CanonicalName);
                trans.Cleanup.Add(filename);
                long resultSize;
                string fn = Path.Combine(TempFolder.FullName, filename);
                if (priorRecord != null)
                {
                    // try to delta encode it
                    string priorLookup = GetLookup(priorRecord);
                    var priorData = ObjectDatabase.Find<FileObjectStoreData>(x => x.Lookup == priorLookup);
                    if (priorData != null)
                    {
                        if (priorData.Mode == StorageMode.Delta)
                            priorData = ObjectDatabase.Find<FileObjectStoreData>(x => x.Lookup == priorData.DeltaBase);
                        if (priorData.HasSignatureData)
                        {
                            try
                            {
                                var signature = LoadSignature(priorData);
                                long deltaSize;
                                List<ChunkedChecksum.FileBlock> blocks;
                                Printer.PrintDiagnostics(" - Trying delta encoding");
                                Printer.InteractivePrinter printer = null;
                                if (newRecord.Size > 16 * 1024 * 1024)
                                    printer = Printer.CreateSimplePrinter(" Computing Delta", (obj) => { return string.Format("{0:N1}%", (float)((long)obj / (double)newRecord.Size) * 100.0f); });
                                using (var fileInput = fileEntry.Info.OpenRead())
                                {
                                    blocks = ChunkedChecksum.ComputeDelta(fileInput, fileEntry.Length, signature, out deltaSize, (fs, ps) => { if (ps % (512 * 1024) == 0 && printer != null) printer.Update(ps); });
                                }
                                if (printer != null)
                                {
                                    printer.End(newRecord.Size);
                                    printer = null;
                                }
                                // dont encode as delta unless we get a 50% saving
                                if (deltaSize < fileEntry.Length / 2)
                                {
                                    FileObjectStoreData data = new FileObjectStoreData()
                                    {
                                        FileSize = newRecord.Size,
                                        HasSignatureData = false,
                                        Lookup = GetLookup(newRecord),
                                        Mode = StorageMode.Delta,
                                        DeltaBase = priorData.Lookup,
                                        Offset = 0
                                    };
                                    trans.Cleanup.Add(filename + ".delta");
                                    Printer.PrintDiagnostics(" - Delta encoding");
                                    using (var fileInput = fileEntry.Info.OpenRead())
                                    using (var fileOutput = new FileInfo(fn + ".delta").OpenWrite())
                                    {
                                        ChunkedChecksum.WriteDelta(fileInput, fileOutput, blocks);
                                    }
                                    Printer.PrintDiagnostics(" - Compressing data");
                                    deltaSize = new FileInfo(fn + ".delta").Length;
                                    using (var fileInput = new FileInfo(fn + ".delta").OpenRead())
                                    using (var fileOutput = new FileInfo(fn).OpenWrite())
                                    {
                                        fileOutput.Write(new byte[] { (byte)'d', (byte)'b', (byte)'l', (byte)'x' }, 0, 4);
                                        CompressionMode cmode = DefaultCompression;
                                        if (cmode != CompressionMode.None && deltaSize < 16 * 1024)
                                            cmode = CompressionMode.LZ4;
                                        if (cmode != CompressionMode.None && deltaSize < 1024)
                                            cmode = CompressionMode.None;
                                        int sig = (int)cmode;
                                        fileOutput.Write(BitConverter.GetBytes(sig), 0, 4);
                                        fileOutput.Write(BitConverter.GetBytes(newRecord.Size), 0, 8);
                                        fileOutput.Write(BitConverter.GetBytes(deltaSize), 0, 8);
                                        fileOutput.Write(BitConverter.GetBytes(priorData.Lookup.Length), 0, 4);
                                        byte[] lookupBytes = ASCIIEncoding.ASCII.GetBytes(priorData.Lookup);
                                        fileOutput.Write(lookupBytes, 0, lookupBytes.Length);
                                        if (deltaSize > 16 * 1024 * 1024)
                                        {
                                            printer = Printer.CreateProgressBarPrinter(string.Empty, string.Format(" Writing {0} ", cmode), (obj) =>
                                            {
                                                return string.Format("{0}/{1}", Misc.FormatSizeFriendly((long)obj), Misc.FormatSizeFriendly(deltaSize));
                                            },
                                            (obj) =>
                                            {
                                                return (float)((long)obj / (double)deltaSize) * 100.0f;
                                            },
                                            (obj, lol) => { return string.Empty; }, 40);
                                        }
                                        if (cmode == CompressionMode.LZHAM)
                                            LZHAMWriter.CompressToStream(deltaSize, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); });
                                        else if (cmode == CompressionMode.LZ4)
                                            LZ4Writer.CompressToStream(deltaSize, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); });
                                        else if (cmode == CompressionMode.LZ4HC)
                                            LZ4HCWriter.CompressToStream(deltaSize, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); });
                                        else
                                        {
                                            resultSize = deltaSize;
                                            fileInput.CopyTo(fileOutput);
                                        }
                                        if (printer != null)
                                            printer.End(newRecord.Size);
                                    }
                                    Printer.PrintMessage(" - Compressed: {0} ({1} delta) => {2}", Misc.FormatSizeFriendly(newRecord.Size), Misc.FormatSizeFriendly(deltaSize), Misc.FormatSizeFriendly(resultSize));
                                    trans.PendingTransactions.Add(
                                        new StandardObjectStoreTransaction.PendingTransaction()
                                        {
                                            Record = newRecord,
                                            Data = data,
                                            Filename = filename
                                        }
                                    );
                                    return true;
                                }
                            }
                            catch
                            {

                            }
                        }
                    }
                }
                bool computeSignature = newRecord.Size > 1024 * 64;
                FileObjectStoreData storeData = new FileObjectStoreData()
                {
                    FileSize = newRecord.Size,
                    HasSignatureData = computeSignature,
                    Lookup = GetLookup(newRecord),
                    Mode = StorageMode.Flat,
                    Offset = 0
                };
                using (var fileInput = fileEntry.Info.OpenRead())
                using (var fileOutput = new FileInfo(fn).OpenWrite())
                {
                    fileOutput.Write(new byte[] { (byte)'d', (byte)'b', (byte)'l', (byte)'k' }, 0, 4);
                    CompressionMode cmode = DefaultCompression;
                    if (cmode != CompressionMode.None && newRecord.Size < 16 * 1024)
                        cmode = CompressionMode.LZ4;
                    if (cmode != CompressionMode.None && newRecord.Size < 1024)
                        cmode = CompressionMode.None;
                    int sig = (int)cmode;
                    if (computeSignature)
                        sig |= 0x8000;
                    fileOutput.Write(BitConverter.GetBytes(sig), 0, 4);
                    fileOutput.Write(BitConverter.GetBytes(newRecord.Size), 0, 8);
                    Printer.InteractivePrinter printer = null;
                    if (newRecord.Size > 16 * 1024 * 1024)
                        printer = Printer.CreateSimplePrinter(" Computing Signature", (obj) => { return string.Format("{0:N1}%", (float)((long)obj / (double)newRecord.Size) * 100.0f); });
                    if (computeSignature)
                    {
                        Printer.PrintDiagnostics(" - Computing signature");
                        var checksum = ChunkedChecksum.Compute(1024, fileInput, (fs, ps) => { if (ps % (512 * 1024) == 0 && printer != null) printer.Update(ps); });
                        fileInput.Position = 0;
                        ChunkedChecksum.Write(fileOutput, checksum);
                    }
                    Printer.PrintDiagnostics(" - Compressing data");
                    if (printer != null)
                    {
                        printer.End(newRecord.Size);
                        printer = Printer.CreateProgressBarPrinter(string.Empty, string.Format(" Writing {0} ", cmode), (obj) =>
                        {
                            return string.Format("{0}/{1}", Misc.FormatSizeFriendly((long)obj), Misc.FormatSizeFriendly(newRecord.Size));
                        },
                        (obj) =>
                        {
                            return (float)((long)obj / (double)newRecord.Size) * 100.0f;
                        },
                        (obj, lol) => { return string.Empty; }, 40);
                    }
                    if (cmode == CompressionMode.LZHAM)
                        LZHAMWriter.CompressToStream(newRecord.Size, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); });
                    else if (cmode == CompressionMode.LZ4)
                        LZ4Writer.CompressToStream(newRecord.Size, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); });
                    else if (cmode == CompressionMode.LZ4HC)
                        LZ4HCWriter.CompressToStream(newRecord.Size, 16 * 1024 * 1024, out resultSize, fileInput, fileOutput, (fs, ps, cs) => { if (printer != null) printer.Update(ps); });
                    else
                    {
                        resultSize = newRecord.Size;
                        fileInput.CopyTo(fileOutput);
                    }
                    if (printer != null)
                        printer.End(newRecord.Size);
                }
                Printer.PrintMessage(" - Compressed: {1} => {2}{3}", newRecord.CanonicalName, Misc.FormatSizeFriendly(newRecord.Size), Misc.FormatSizeFriendly(resultSize), computeSignature ? " (computed signatures)" : "");
                trans.PendingTransactions.Add(
                    new StandardObjectStoreTransaction.PendingTransaction()
                    {
                        Record = newRecord,
                        Data = storeData,
                        Filename = filename
                    }
                );
                return true;
            }
        }
예제 #18
0
 public override bool HasData(Record recordInfo, out List<string> requestedData)
 {
     requestedData = null;
     if (!recordInfo.HasData)
         return true;
     return HasDataDirect(GetLookup(recordInfo), out requestedData);
 }
예제 #19
0
 private void SerializeRecord(Record x, BinaryWriter bw)
 {
     bw.Write(x.Id);
     bw.Write(x.Parent.HasValue ? x.Parent.Value : -1L);
     bw.Write(x.Size);
     bw.Write((uint)x.Attributes);
     bw.Write(x.Fingerprint);
     bw.Write(x.CanonicalNameId);
     bw.Write(x.ModificationTime.Ticks);
     bw.Write(x.CanonicalName);
 }
예제 #20
0
 public abstract bool TransmitRecordData(Record record, Func<byte[], int, bool, bool> sender, byte[] scratchBuffer, Action beginTransmission = null);