public RavenConfiguration() { _configBuilder = new ConfigurationBuilder(); AddEnvironmentVariables(_configBuilder); AddJsonConfigurationVariables(); Settings = _configBuilder.Build(); Core = new CoreConfiguration(); Replication = new ReplicationConfiguration(); SqlReplication = new SqlReplicationConfiguration(); Storage = new StorageConfiguration(); Encryption = new EncryptionConfiguration(); Indexing = new IndexingConfiguration(() => DatabaseName, () => Core.RunInMemory, () => Core.DataDirectory); WebSockets = new WebSocketsConfiguration(); Monitoring = new MonitoringConfiguration(); Queries = new QueryConfiguration(); Patching = new PatchingConfiguration(); DebugLog = new DebugLoggingConfiguration(); BulkInsert = new BulkInsertConfiguration(); Server = new ServerConfiguration(); Memory = new MemoryConfiguration(this); Expiration = new ExpirationBundleConfiguration(); Studio = new StudioConfiguration(); Databases = new DatabaseConfiguration(); Licensing = new LicenseConfiguration(); Quotas = new QuotasBundleConfiguration(); Tombstones = new TombstoneConfiguration(); }
public void ModifyRequest() { EncryptionConfiguration config = new EncryptionConfiguration { KmsKeyName = "dataset_default_key" }; var options = new CreateDatasetOptions { DefaultTableExpiration = TimeSpan.FromSeconds(10), Description = "A description", FriendlyName = "A friendly name", Location = "EU", DefaultEncryptionConfiguration = config, DefaultPartitionExpiration = TimeSpan.FromSeconds(20) }; Dataset dataset = new Dataset { Location = "US" }; InsertRequest request = new InsertRequest(new BigqueryService(), dataset, "project"); options.ModifyRequest(dataset, request); Assert.Equal(10 * 1000, dataset.DefaultTableExpirationMs); Assert.Equal("A description", dataset.Description); Assert.Equal("A friendly name", dataset.FriendlyName); Assert.Equal("EU", dataset.Location); Assert.Equal(20 * 1000, dataset.DefaultPartitionExpirationMs); Assert.Equal(config, dataset.DefaultEncryptionConfiguration); }
public TransactionsController( ITransactionSigner transactionSigner, EncryptionConfiguration encryptionConfiguration) { _transactionSigner = transactionSigner; _encryptionConfiguration = encryptionConfiguration; }
public ImageEncoder(ImageFormat format, EncryptionConfiguration encryptionConfiguration, CompressionConfiguration compressionConfiguration, EmbeddedImage embeddedImage = null) { _encryptionConfiguration = encryptionConfiguration; _compressionConfiguration = compressionConfiguration; _embeddedImage = embeddedImage; _format = format; }
private EncryptionService GetEncryptionService() { var configuration = new EncryptionConfiguration(); configuration.Password = "******"; configuration.Salt = Convert.FromBase64String("Jtz/AK3teu7F/gevTQgiPA=="); return(new EncryptionService(configuration)); }
public override void ExecuteCmdlet() { if (ParameterSetName.Equals(ParameterSetNames.ByFactoryObject, StringComparison.OrdinalIgnoreCase)) { Name = InputObject.DataFactoryName; ResourceGroupName = InputObject.ResourceGroupName; } else if (ParameterSetName.Equals(ParameterSetNames.ByResourceId, StringComparison.OrdinalIgnoreCase)) { var parsedResourceId = new ResourceIdentifier(ResourceId); Name = parsedResourceId.ResourceName; ResourceGroupName = parsedResourceId.ResourceGroupName; } string factoryIdentityType = FactoryIdentityType.SystemAssigned; if (!string.IsNullOrWhiteSpace(this.IdentityType)) { factoryIdentityType = this.IdentityType; } if (this.UserAssignedIdentity != null && this.UserAssignedIdentity.Count > 0) { if (!factoryIdentityType.ToLower().Contains(FactoryIdentityType.UserAssigned.ToLower())) { factoryIdentityType = FactoryIdentityType.SystemAssignedUserAssigned; } } FactoryIdentity factoryIdentity = new FactoryIdentity(factoryIdentityType, userAssignedIdentities: this.UserAssignedIdentity); EncryptionConfiguration encryption = null; if (!string.IsNullOrWhiteSpace(this.EncryptionVaultBaseUrl) && !string.IsNullOrWhiteSpace(this.EncryptionKeyName)) { CMKIdentityDefinition cmkIdentity = null; if (!string.IsNullOrWhiteSpace(this.EncryptionUserAssignedIdentity)) { cmkIdentity = new CMKIdentityDefinition(this.EncryptionUserAssignedIdentity); } encryption = new EncryptionConfiguration(this.EncryptionKeyName, this.EncryptionVaultBaseUrl, this.EncryptionKeyVersion, cmkIdentity); } var parameters = new UpdatePSDataFactoryParameters() { ResourceGroupName = ResourceGroupName, DataFactoryName = Name, EncryptionConfiguration = encryption, FactoryIdentity = factoryIdentity, Tags = Tag }; if (ShouldProcess(Name)) { WriteObject(DataFactoryClient.UpdatePSDataFactory(parameters)); } }
public override void Setup(IDbConnection connection, IDbTransaction transaction) { ExecuteSql(connection, transaction); var encryption = new EncryptionConfiguration(); encryption.IV = Bytes.GenerateIV(); encryption.Key = Bytes.GenerateKey(); connection.Execute("INSERT INTO Encryption(IV,[Key]) Values(@IV,@Key)", new { encryption.IV, encryption.Key }, transaction); connection.Execute("INSERT INTO Version(Version) Values (@Version)", new { Version }, transaction); }
public StronglyTypedRavenSettings(NameValueCollection settings) { Replication = new ReplicationConfiguration(); Voron = new VoronConfiguration(); Prefetcher = new PrefetcherConfiguration(); FileSystem = new FileSystemConfiguration(); Encryption = new EncryptionConfiguration(); this.settings = settings; }
public StronglyTypedRavenSettings(NameValueCollection settings) { Replication = new ReplicationConfiguration(); Voron = new VoronConfiguration(); Prefetcher = new PrefetcherConfiguration(); FileSystem = new FileSystemConfiguration(); Encryption = new EncryptionConfiguration(); this.settings = settings; }
public ReplicationDestination(AccessControlTranslation accessControlTranslation, string account, string bucketARN, EncryptionConfiguration encryptionConfiguration, Metrics metrics, ReplicationTime replicationTime, string storageClass) { AccessControlTranslation = accessControlTranslation; Account = account; BucketARN = bucketARN; EncryptionConfiguration = encryptionConfiguration; Metrics = metrics; ReplicationTime = replicationTime; StorageClass = storageClass; }
public StronglyTypedRavenSettings(NameValueCollection settings) { Replication = new ReplicationConfiguration(); Voron = new VoronConfiguration(); Esent = new EsentConfiguration(); Prefetcher = new PrefetcherConfiguration(); FileSystem = new FileSystemConfiguration(); Encryption = new EncryptionConfiguration(); Indexing = new IndexingConfiguration(); WebSockets = new WebSocketsConfiguration(); this.settings = settings; }
public StronglyTypedRavenSettings(NameValueCollection settings) { Replication = new ReplicationConfiguration(); Voron = new VoronConfiguration(); Esent = new EsentConfiguration(); Prefetcher = new PrefetcherConfiguration(); FileSystem = new FileSystemConfiguration(); Encryption = new EncryptionConfiguration(); Indexing = new IndexingConfiguration(); WebSockets = new WebSocketsConfiguration(); this.settings = settings; }
private async Task CreateVersionCompatData( ImageFormat format, FileInfo inputZipFIle, EncryptionConfiguration encryption, CompressionConfiguration compression, EmbeddedImage embeddedImage) { var inputFileName = Path.GetFileNameWithoutExtension(inputZipFIle.Name); string outputName = inputFileName.Substring(0, Math.Min(inputFileName.Length, 10)) + "-" + Enum.GetName(typeof(ImageFormat), format); if (embeddedImage != null) { outputName += "-E"; } if (encryption != null) { outputName += "-" + Enum.GetName(typeof(EncryptionType), encryption.Type).Substring(0, 3); } if (compression != null) { outputName += "-" + Enum.GetName(typeof(CompressionType), compression.Type).Substring(0, 3); } var tempInputDataDirectory = _ouputTempVersionDataDirectory.CreateSubdirectory(outputName + "_original_data"); using (var zipFile = ZipFile.Read(inputZipFIle.FullName)) { zipFile.ExtractAll(tempInputDataDirectory.FullName); } DirectoryInfo outputDirectory = _ouputTempVersionDataDirectory.CreateSubdirectory(outputName); var encoder = new ImageEncoder(format, encryption, compression, embeddedImage); using (Stream encodedImageFile = File.Create(Path.Combine(outputDirectory.FullName, inputFileName + "." + encoder.Extension))) { encoder.AddDirectory(tempInputDataDirectory); await encoder.SaveAsync(encodedImageFile, new EncodingConfiguration( password : "******", tempStorageProvider : new MemoryStorageProvider(), bufferSize : 4096, fileGroupSize : 1024 * 500)); inputZipFIle.CopyTo(Path.Combine(outputDirectory.FullName, "Original_Data.zip"), true); } tempInputDataDirectory.Delete(true); }
public void Encryption_With_Text_Success() { EncryptionConfiguration encryptionConfiguration = new EncryptionConfiguration(DefaultKey); IAesEncryptionHelper aesEncryptionHelper = new AesEncryptionHelper(encryptionConfiguration); var text = "aesEncrypt"; var encrypted = aesEncryptionHelper.Encrypt(text); Assert.NotNull(encrypted); Assert.NotEmpty(encrypted); }
public async Task EncodingThenDecoding_ProducesEquivalentResults( ImageFormat format, FileInfo inputZipFile, EncryptionConfiguration encryption, CompressionConfiguration compression, EmbeddedImage embeddedImage, IDictionary <string, string> metadata) { DirectoryInfo inputDirectory = null; DirectoryInfo outputDirectory = null; try { using (var storageStream = new MemoryStream()) { inputDirectory = _inputRootDirectory.CreateSubdirectory(Path.GetFileNameWithoutExtension(inputZipFile.Name)); using (var zipFile = ZipFile.Read(inputZipFile.FullName)) { zipFile.ExtractAll(inputDirectory.FullName); } var encoder = new ImageEncoder(format, encryption, compression, embeddedImage); encoder.Metadata = metadata; encoder.AddDirectory(inputDirectory); await encoder.SaveAsync(storageStream, ImageConfigurations.EncodingConfiguration); storageStream.Position = 0; var decoder = await ImageDecoder.LoadAsync(storageStream, ImageConfigurations.DecodingConfiguration); outputDirectory = _ouputRootDirectory.CreateSubdirectory(inputDirectory.Name); await decoder.DecodeAsync(outputDirectory); AssertEx.AreEqualByJson(metadata, decoder.Metadata); DirectoryAssert.AreEquivalent(inputDirectory, outputDirectory); } } finally { foreach (var directory in new[] { inputDirectory, outputDirectory }) { if (directory != null) { directory.Delete(true); } } } }
public static bool TryToCreateTransactionalStorage(InMemoryRavenConfiguration ravenConfiguration, bool hasCompression, EncryptionConfiguration encryption, out ITransactionalStorage storage) { storage = null; if (File.Exists(Path.Combine(ravenConfiguration.DataDirectory, Voron.Impl.Constants.DatabaseFilename))) { storage = ravenConfiguration.CreateTransactionalStorage(InMemoryRavenConfiguration.VoronTypeName, () => { }, () => { }); } else if (File.Exists(Path.Combine(ravenConfiguration.DataDirectory, "Data"))) { storage = ravenConfiguration.CreateTransactionalStorage(InMemoryRavenConfiguration.EsentTypeName, () => { }, () => { }); } if (storage == null) { return(false); } var orderedPartCollection = new OrderedPartCollection <AbstractDocumentCodec>(); if (encryption != null) { var documentEncryption = new DocumentEncryption(); documentEncryption.SetSettings(new EncryptionSettings(encryption.EncryptionKey, encryption.SymmetricAlgorithmType, encryption.EncryptIndexes, encryption.PreferedEncryptionKeyBitsSize)); orderedPartCollection.Add(documentEncryption); } if (hasCompression) { orderedPartCollection.Add(new DocumentCompression()); } uuidGenerator = new SequentialUuidGenerator(); storage.Initialize(uuidGenerator, orderedPartCollection); /* * Added configuration steps */ storage.Batch(actions => { var nextIdentityValue = actions.General.GetNextIdentityValue("Raven/Etag"); uuidGenerator.EtagBase = nextIdentityValue; }); return(true); }
public StronglyTypedRavenSettings(NameValueCollection settings) { Replication = new ReplicationConfiguration(); Voron = new VoronConfiguration(); Esent = new EsentConfiguration(); Prefetcher = new PrefetcherConfiguration(); FileSystem = new FileSystemConfiguration(); Counter = new CounterConfiguration(); TimeSeries = new TimeSeriesConfiguration(); Encryption = new EncryptionConfiguration(); Indexing = new IndexingConfiguration(); WebSockets = new WebSocketsConfiguration(); Cluster = new ClusterConfiguration(); Monitoring = new MonitoringConfiguration(); Studio = new StudioConfiguration(); this.settings = settings; }
public SymmetricEncryptionService(EncryptionConfiguration encryptionConfiguration = null, ILogger logger = null) { if (logger == null) { _logger = new LoggerConfiguration() .WriteTo.Console() .CreateLogger(); } else { _logger = logger; } if (encryptionConfiguration == null) { _encryptionConfiguration = new EncryptionConfiguration(); } else { _encryptionConfiguration = encryptionConfiguration; } }
public RavenConfiguration() { Settings = new NameValueCollection(StringComparer.OrdinalIgnoreCase); Core = new CoreConfiguration(this); FileSystem = new FileSystemConfiguration(Core); Counter = new CounterConfiguration(Core); TimeSeries = new TimeSeriesConfiguration(Core); Replication = new ReplicationConfiguration(); Prefetcher = new PrefetcherConfiguration(); Storage = new StorageConfiguration(); Encryption = new EncryptionConfiguration(); Indexing = new IndexingConfiguration(); WebSockets = new WebSocketsConfiguration(); Cluster = new ClusterConfiguration(); Monitoring = new MonitoringConfiguration(); Queries = new QueryConfiguration(); Patching = new PatchingConfiguration(); BulkInsert = new BulkInsertConfiguration(); Server = new ServerConfiguration(); Memory = new MemoryConfiguration(); Expiration = new ExpirationBundleConfiguration(); Versioning = new VersioningBundleConfiguration(); Studio = new StudioConfiguration(); Tenants = new TenantConfiguration(); Licensing = new LicenseConfiguration(); Quotas = new QuotasBundleConfiguration(); IndexingClassifier = new DefaultIndexingClassifier(); Catalog = new AggregateCatalog(CurrentAssemblyCatalog); Catalog.Changed += (sender, args) => ResetContainer(); }
public async Task <HttpResponseMessage> Encode() { EncodingJob encodingJob = null; string badRequestMessage = null; try { encodingJob = await BuildEncodingJob(Request); } catch (BadRequestException exception) { badRequestMessage = exception.Message; } catch {} if (encodingJob == null) { return(new HttpResponseMessage(HttpStatusCode.BadRequest) { Content = new StringContent(badRequestMessage) }); } EncryptionConfiguration encryptionConfiguration = null; CompressionConfiguration compressionConfiguration = null; if (encodingJob.HasPassword) { encryptionConfiguration = new EncryptionConfiguration( encodingJob.EncryptionAlgorithm.Value, TranscodingConfiguration.IterationCount); } if (encodingJob.CompressionAlgorithm.HasValue) { compressionConfiguration = new CompressionConfiguration(encodingJob.CompressionAlgorithm.Value, encodingJob.CompressionLevel); } var imageEncoder = new ImageEncoder(encodingJob.Format, encryptionConfiguration, compressionConfiguration, encodingJob.EmbeddedImage); imageEncoder.AddDirectories(encodingJob.Directories); imageEncoder.Metadata.Add("creation-date", DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture)); var guid = SecureGuid(); try { DirectoryInfo jobDirectory = CreateJobDirectory(guid); var outputPath = Path.Combine(jobDirectory.FullName, encodingJob.FileName + "." + imageEncoder.Extension); using (Stream file = new FileStream(outputPath, FileMode.CreateNew)) { await imageEncoder.SaveAsync(file, BuildConfiguration(encodingJob.Password)); } } catch { return(new HttpResponseMessage(HttpStatusCode.InternalServerError)); } return(Request.CreateResponse(HttpStatusCode.OK, new { id = guid.ToString() })); }
public override void ExecuteCmdlet() { this.ValidateParameters(); FactoryRepoConfiguration repoConfiguration = null; if (!string.IsNullOrWhiteSpace(this.ProjectName) || !string.IsNullOrWhiteSpace(this.TenantId)) { var factoryVSTSConfiguration = new FactoryVSTSConfiguration(); factoryVSTSConfiguration.ProjectName = this.ProjectName; factoryVSTSConfiguration.TenantId = this.TenantId; repoConfiguration = factoryVSTSConfiguration; } else if (!string.IsNullOrWhiteSpace(this.HostName)) { var factoryGitHubConfiguration = new FactoryGitHubConfiguration(); factoryGitHubConfiguration.HostName = this.HostName; repoConfiguration = factoryGitHubConfiguration; } if (repoConfiguration != null) { repoConfiguration.CollaborationBranch = this.CollaborationBranch; repoConfiguration.AccountName = this.AccountName; repoConfiguration.LastCommitId = this.LastCommitId; repoConfiguration.RootFolder = this.RootFolder; repoConfiguration.RepositoryName = this.RepositoryName; } string factoryIdentityType = FactoryIdentityType.SystemAssigned; if (!string.IsNullOrWhiteSpace(this.IdentityType)) { factoryIdentityType = this.IdentityType; } if (this.UserAssignedIdentity != null && this.UserAssignedIdentity.Count > 0) { if (!factoryIdentityType.ToLower().Contains(FactoryIdentityType.UserAssigned.ToLower())) { factoryIdentityType = FactoryIdentityType.SystemAssignedUserAssigned; } } FactoryIdentity factoryIdentity = new FactoryIdentity(factoryIdentityType, userAssignedIdentities: this.UserAssignedIdentity); EncryptionConfiguration encryption = null; if (!string.IsNullOrWhiteSpace(this.EncryptionVaultBaseUrl) && !string.IsNullOrWhiteSpace(this.EncryptionKeyName)) { CMKIdentityDefinition cmkIdentity = null; if (!string.IsNullOrWhiteSpace(this.EncryptionUserAssignedIdentity)) { cmkIdentity = new CMKIdentityDefinition(this.EncryptionUserAssignedIdentity); } encryption = new EncryptionConfiguration(this.EncryptionKeyName, this.EncryptionVaultBaseUrl, this.EncryptionKeyVersion, cmkIdentity); } string publicNetworkAccess = Management.DataFactory.Models.PublicNetworkAccess.Enabled; if (!string.IsNullOrWhiteSpace(this.PublicNetworkAccess)) { publicNetworkAccess = this.PublicNetworkAccess; } var parameters = new CreatePSDataFactoryParameters() { ResourceGroupName = ResourceGroupName, DataFactoryName = Name, Location = Location, PublicNetworkAccess = publicNetworkAccess, EncryptionConfiguration = encryption, FactoryIdentity = factoryIdentity, Tags = Tag, Force = Force.IsPresent, RepoConfiguration = repoConfiguration, ConfirmAction = ConfirmAction, GlobalParameters = GlobalParameterDefinition }; WriteObject(DataFactoryClient.CreatePSDataFactory(parameters)); }
public ImageEncoder(ImageFormat format, EncryptionConfiguration encryptionConfiguration, EmbeddedImage embeddedImage = null) : this(format, encryptionConfiguration, null, embeddedImage) { }
static void Main(string[] args) { var ravenConfiguration = new RavenConfiguration { DataDirectory = ConfigurationManager.AppSettings["DataDirectory"], CacheDocumentsInMemory = false, RunInMemory = false, Storage = { PreventSchemaUpdate = true, SkipConsistencyCheck = true } }; Console.WriteLine("Configuration initialized."); var encryptionKey = ConfigurationManager.AppSettings["Raven/Encryption/Key"]; if (!string.IsNullOrWhiteSpace(encryptionKey)) { Encryption = new EncryptionConfiguration() { EncryptIndexes = ConfigurationManager.AppSettings["Raven/Encryption/EncryptIndexes"].Equals("True") }; if (Encryption.TrySavingEncryptionKey(encryptionKey) == false) { ConsoleUtils.ConsoleWriteLineWithColor(ConsoleColor.Red, "Encryption key should be in base64 string format, we got {0}.\n", encryptionKey); } var algorithmType = ConfigurationManager.AppSettings["Raven/Encryption/Algorithm"]; if (Encryption.TrySavingAlgorithmType(algorithmType) == false) { ConsoleUtils.ConsoleWriteLineWithColor(ConsoleColor.Red, "Unknown encryption algorithm type, we got {0}.\n", algorithmType); } var preferedEncryptionKeyBitsSize = ConfigurationManager.AppSettings["Raven/Encryption/KeyBitsPreference"]; if (Encryption.SavePreferedEncryptionKeyBitsSize(preferedEncryptionKeyBitsSize) == false) { ConsoleUtils.ConsoleWriteLineWithColor(ConsoleColor.Red, "Encryption key bit size should be in an int, we got {0}.\n", preferedEncryptionKeyBitsSize); } Console.WriteLine("Encryption configured"); } CreateTransactionalStorage(ravenConfiguration); Console.WriteLine("Transactional storage created."); try { Etag seekAfterEtag = ParseEtag(ConfigurationManager.AppSettings["DocumentTouchCorruptSeekAfterEtag"]), preTouchEtag, afterTouchEtag; // Action info string[] actionAndTimes = ConfigurationManager.AppSettings["ActionAndTimes"].Split(':'); string action = actionAndTimes[0]; int actionTimes = int.TryParse(actionAndTimes.Length > 1 ? actionAndTimes[1] : "1", out actionTimes) ? actionTimes : 1; // Document key and tag info Dictionary <string, Dictionary <string, string> > documentKeyAndEtagJson = JsonConvert.DeserializeObject <Dictionary <string, Dictionary <string, string> > >(File.ReadAllText(ConfigurationManager.AppSettings["DocumentKeyAndEtagJsonPath"])); Dictionary <string, List <KeyValuePair <string, Etag> > > documentKeyAndEtagCasted = new Dictionary <string, List <KeyValuePair <string, Etag> > >(); foreach (var keyAndEtagDic in documentKeyAndEtagJson) { documentKeyAndEtagCasted.Add(keyAndEtagDic.Key, keyAndEtagDic.Value.Select(k => { string documentKey = k.Value; Etag etag = ParseEtag(k.Key); return(new KeyValuePair <string, Etag>(documentKey, etag)); }).ToList()); } var documentKeyAndEtag = documentKeyAndEtagCasted[action]; switch (action) { case "doc_json": /** * Try json document */ var jsonDocumentDic = new Dictionary <string, JsonDocument>(); var metadataOnly = ConfigurationManager.AppSettings["DocumentsJsonMetadataOnly"] == "true"; var outputFields = ConfigurationManager.AppSettings["DocumentsJsonOutputFields"].Split(';'); foreach (var keyValuePair in documentKeyAndEtag) { storage.Batch(a => { try { JsonDocument jsonDocument; if (metadataOnly) { var metadata = a.Documents.DocumentMetadataByKey(keyValuePair.Key); jsonDocument = new JsonDocument() { DataAsJson = RavenJObject.FromObject(new object()), Metadata = metadata.Metadata, Etag = metadata.Etag, LastModified = metadata.LastModified, Key = metadata.Key }; } else { jsonDocument = a.Documents.DocumentByKey(keyValuePair.Key); } Console.WriteLine($"Key: {jsonDocument}, Etag: {jsonDocument.Etag}"); Console.WriteLine($"Metadata: {jsonDocument.Metadata}"); Console.WriteLine(""); jsonDocumentDic.Add(keyValuePair.Key, jsonDocument); } catch (Exception ex) { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine($"Key: {keyValuePair.Key}, Exception: {ex}"); Console.ResetColor(); jsonDocumentDic.Add(keyValuePair.Key, new JsonDocument() { DataAsJson = RavenJObject.FromObject(ex), Metadata = new RavenJObject() }); } }); } var jsonDocumentDicSerialized = jsonDocumentDic.Select(d => { var key = d.Key; //var dataAsJson = d.Value.DataAsJson.ToString(); //var metadata = d.Value.Metadata.ToString(); //var lastModified = d.Value.LastModified.ToString(); var objDictionary = new Dictionary <string, string>() { { "DataAsJson", d.Value.DataAsJson.ToString() }, { "Metadata", d.Value.Metadata.ToString() }, { "LastModified", d.Value.LastModified.ToString() } } .Where(f => outputFields.Contains(f.Key)) .ToDictionary(f => f.Key, f => f.Value); return(new KeyValuePair <string, Dictionary <string, string> >(key, objDictionary)); }).ToDictionary(d => d.Key, d => d.Value); var newJsonName = string.Format(ConfigurationManager.AppSettings["DocumentsJsonPath"], DateTime.Now.ToString("s").Replace(':', '_')); File.WriteAllText(newJsonName, JsonConvert.SerializeObject(jsonDocumentDicSerialized)); break; case "doc_touch_corrupt": /** * Try deleting the document */ foreach (var keyValuePair in documentKeyAndEtag) { for (int i = 0; i < actionTimes; i++) { storage.Batch(a => { a.Documents.TouchCorruptDocumentPub(keyValuePair.Key, out preTouchEtag, out afterTouchEtag, seekAfterEtag); Console.WriteLine($"preTouchEtag: {preTouchEtag}, afterTouchEtag: {afterTouchEtag}"); }); } } break; case "doc_touch": /** * Try deleting the document */ foreach (var keyValuePair in documentKeyAndEtag) { for (int i = 0; i < actionTimes; i++) { storage.Batch(a => { a.Documents.TouchDocument(keyValuePair.Key, out preTouchEtag, out afterTouchEtag); Console.WriteLine($"preTouchEtag: {preTouchEtag}, afterTouchEtag: {afterTouchEtag}"); }); } } break; case "doc_delete": /** * Try deleting the document */ RavenJObject outMetadata; Etag outEtag; foreach (var keyValuePair in documentKeyAndEtag) { for (int i = 0; i < actionTimes; i++) { storage.Batch(a => { Console.WriteLine($"Deleting document by key: '{keyValuePair.Key}'"); a.Documents.DeleteDocument(keyValuePair.Key, null, out outMetadata, out outEtag); Console.WriteLine($"outEtag: {outEtag}, outMetadata: {outMetadata}"); }); } } break; case "doc_update_metadata": /** * Trying to overwrite the metadata with empty metadata */ foreach (var keyValuePair in documentKeyAndEtag) { JsonDocumentMetadata metadata = new JsonDocumentMetadata() { Etag = keyValuePair.Value, Key = keyValuePair.Key, Metadata = new RavenJObject() }; storage.Batch(a => a.Documents.WriteDocumentMetadataPub(metadata, metadata.Key, true)); } break; case "documents_key_etag": /** * Get the table of keys and tags for each document */ storage.Batch(a => { var writeCount = 0; var newFileName = string.Format(ConfigurationManager.AppSettings["DocumentsKeyEtagPath"], DateTime.Now.ToString("s").Replace(':', '_')); var outputCorruptOnly = ConfigurationManager.AppSettings["DocumentsKeyEtagCorruptOnly"] == "true"; var outputCorruptIndicators = ConfigurationManager.AppSettings["DocumentsKeyEtagOutputCorruptIndicators"] == "true"; Etag startAtEtag = ParseEtag(ConfigurationManager.AppSettings["DocumentsKeyEtagStartAtEtag"]); int take = ParseInt(ConfigurationManager.AppSettings["DocumentsKeyEtagTake"], int.MaxValue, "Max"); var timer = Stopwatch.StartNew(); using (var writer = new StreamWriter(newFileName)) { var templateHeader = outputCorruptIndicators ? "Etag, Metadata, Document, DocumentKey, Exception" : "Etag, Document key"; // Write column headers writer.WriteLine(templateHeader); var keyEtags = a.Documents.GetKeysAfterWithIdStartingWithPub(startAtEtag, take, includeMetadataCanBeReadFlag: outputCorruptIndicators, includeDocumentCanBeReadFlag: outputCorruptIndicators); foreach (var etagTuple in keyEtags) { if (writeCount >= 10000) { Console.Write("."); writeCount = 0; } writeCount++; //Console.WriteLine($"{keyValuePair.Key} - {keyValuePair.Value}"); if (outputCorruptOnly && etagTuple.Item5 == null) { continue; } var templateLine = outputCorruptIndicators ? $"{etagTuple.Item1}, {etagTuple.Item3}, {etagTuple.Item4}, {etagTuple.Item2}, {JsonConvert.SerializeObject(etagTuple.Item5)}" : $"{etagTuple.Item1}, {etagTuple.Item2}"; // Write lines writer.WriteLine(templateLine); } Console.WriteLine(""); Console.WriteLine($"Completed in {timer.Elapsed.ToString()}"); Console.WriteLine($"Generated file '{newFileName}'"); } }); break; case "compare_document_key_csvs": var timer1 = Stopwatch.StartNew(); // Load the sets List <HashSet <string> > hashSets = new List <HashSet <string> >(); for (int i = 0; i < 2; i++) { var fileConfig = ConfigurationManager.AppSettings[$"CompareDocumentsKeyCsvsPath{i+1}"].Split(','); int columnNumber = int.TryParse(fileConfig.Length > 1 ? fileConfig[1] : "0", out columnNumber) ? columnNumber : 0; var set = GetDocumentKeysFromCsv(fileConfig[0], columnNumber); hashSets.Add(set); } // Compare the sets List <IEnumerable <string> > diffs = new List <IEnumerable <string> > { hashSets[0].Except(hashSets[1]), hashSets[1].Except(hashSets[0]) }; // Save the diffs for (int i = 0; i < 2; i++) { var newFileName = string.Format(ConfigurationManager.AppSettings["CompareDocumentsKeyPath"], DateTime.Now.ToString("s").Replace(':', '_'), i == 0 ? "1to2" : "2to1"); var writeCount = 0; using (var writer = new StreamWriter(newFileName)) { // Write column headers writer.WriteLine("Document key"); foreach (var diff in diffs[i]) { if (writeCount >= 10000) { Console.Write("."); writeCount = 0; } writeCount++; // Write lines writer.WriteLine(diff); } } Console.WriteLine($"Generated file '{newFileName}'"); } Console.WriteLine($"Completed in {timer1.Elapsed}"); break; default: Console.WriteLine("Allowed actions are: doc_delete, doc_update_metadata, documents_key_etag"); break; } Console.WriteLine("Operation succeeded."); } catch (Exception ex) { Console.WriteLine(ex.ToString()); } finally { storage.Dispose(); } Console.Read(); }