public RavenConfiguration()
{
_configBuilder = new ConfigurationBuilder();
AddEnvironmentVariables(_configBuilder);
AddJsonConfigurationVariables();
Settings = _configBuilder.Build();
Core = new CoreConfiguration();
Replication = new ReplicationConfiguration();
SqlReplication = new SqlReplicationConfiguration();
Storage = new StorageConfiguration();
Encryption = new EncryptionConfiguration();
Indexing = new IndexingConfiguration(() => DatabaseName, () => Core.RunInMemory, () => Core.DataDirectory);
WebSockets = new WebSocketsConfiguration();
Monitoring = new MonitoringConfiguration();
Queries = new QueryConfiguration();
Patching = new PatchingConfiguration();
DebugLog = new DebugLoggingConfiguration();
BulkInsert = new BulkInsertConfiguration();
Server = new ServerConfiguration();
Memory = new MemoryConfiguration(this);
Expiration = new ExpirationBundleConfiguration();
Studio = new StudioConfiguration();
Databases = new DatabaseConfiguration();
Licensing = new LicenseConfiguration();
Quotas = new QuotasBundleConfiguration();
Tombstones = new TombstoneConfiguration();
}
public void ModifyRequest()
{
EncryptionConfiguration config = new EncryptionConfiguration {
KmsKeyName = "dataset_default_key"
};
var options = new CreateDatasetOptions
{
DefaultTableExpiration = TimeSpan.FromSeconds(10),
Description = "A description",
FriendlyName = "A friendly name",
Location = "EU",
DefaultEncryptionConfiguration = config,
DefaultPartitionExpiration = TimeSpan.FromSeconds(20)
};
Dataset dataset = new Dataset {
Location = "US"
};
InsertRequest request = new InsertRequest(new BigqueryService(), dataset, "project");
options.ModifyRequest(dataset, request);
Assert.Equal(10 * 1000, dataset.DefaultTableExpirationMs);
Assert.Equal("A description", dataset.Description);
Assert.Equal("A friendly name", dataset.FriendlyName);
Assert.Equal("EU", dataset.Location);
Assert.Equal(20 * 1000, dataset.DefaultPartitionExpirationMs);
Assert.Equal(config, dataset.DefaultEncryptionConfiguration);
}
public TransactionsController(
ITransactionSigner transactionSigner,
EncryptionConfiguration encryptionConfiguration)
{
_transactionSigner = transactionSigner;
_encryptionConfiguration = encryptionConfiguration;
}
public ImageEncoder(ImageFormat format, EncryptionConfiguration encryptionConfiguration,
CompressionConfiguration compressionConfiguration, EmbeddedImage embeddedImage = null)
{
_encryptionConfiguration = encryptionConfiguration;
_compressionConfiguration = compressionConfiguration;
_embeddedImage = embeddedImage;
_format = format;
}
private EncryptionService GetEncryptionService()
{
var configuration = new EncryptionConfiguration();
configuration.Password = "******";
configuration.Salt = Convert.FromBase64String("Jtz/AK3teu7F/gevTQgiPA==");
return(new EncryptionService(configuration));
}
public override void ExecuteCmdlet()
{
if (ParameterSetName.Equals(ParameterSetNames.ByFactoryObject, StringComparison.OrdinalIgnoreCase))
{
Name = InputObject.DataFactoryName;
ResourceGroupName = InputObject.ResourceGroupName;
}
else if (ParameterSetName.Equals(ParameterSetNames.ByResourceId, StringComparison.OrdinalIgnoreCase))
{
var parsedResourceId = new ResourceIdentifier(ResourceId);
Name = parsedResourceId.ResourceName;
ResourceGroupName = parsedResourceId.ResourceGroupName;
}
string factoryIdentityType = FactoryIdentityType.SystemAssigned;
if (!string.IsNullOrWhiteSpace(this.IdentityType))
{
factoryIdentityType = this.IdentityType;
}
if (this.UserAssignedIdentity != null && this.UserAssignedIdentity.Count > 0)
{
if (!factoryIdentityType.ToLower().Contains(FactoryIdentityType.UserAssigned.ToLower()))
{
factoryIdentityType = FactoryIdentityType.SystemAssignedUserAssigned;
}
}
FactoryIdentity factoryIdentity = new FactoryIdentity(factoryIdentityType, userAssignedIdentities: this.UserAssignedIdentity);
EncryptionConfiguration encryption = null;
if (!string.IsNullOrWhiteSpace(this.EncryptionVaultBaseUrl) && !string.IsNullOrWhiteSpace(this.EncryptionKeyName))
{
CMKIdentityDefinition cmkIdentity = null;
if (!string.IsNullOrWhiteSpace(this.EncryptionUserAssignedIdentity))
{
cmkIdentity = new CMKIdentityDefinition(this.EncryptionUserAssignedIdentity);
}
encryption = new EncryptionConfiguration(this.EncryptionKeyName, this.EncryptionVaultBaseUrl, this.EncryptionKeyVersion, cmkIdentity);
}
var parameters = new UpdatePSDataFactoryParameters()
{
ResourceGroupName = ResourceGroupName,
DataFactoryName = Name,
EncryptionConfiguration = encryption,
FactoryIdentity = factoryIdentity,
Tags = Tag
};
if (ShouldProcess(Name))
{
WriteObject(DataFactoryClient.UpdatePSDataFactory(parameters));
}
}
public override void Setup(IDbConnection connection, IDbTransaction transaction)
{
ExecuteSql(connection, transaction);
var encryption = new EncryptionConfiguration();
encryption.IV = Bytes.GenerateIV();
encryption.Key = Bytes.GenerateKey();
connection.Execute("INSERT INTO Encryption(IV,[Key]) Values(@IV,@Key)", new { encryption.IV, encryption.Key }, transaction);
connection.Execute("INSERT INTO Version(Version) Values (@Version)", new { Version }, transaction);
}
public StronglyTypedRavenSettings(NameValueCollection settings)
{
Replication = new ReplicationConfiguration();
Voron = new VoronConfiguration();
Prefetcher = new PrefetcherConfiguration();
FileSystem = new FileSystemConfiguration();
Encryption = new EncryptionConfiguration();
this.settings = settings;
}
public StronglyTypedRavenSettings(NameValueCollection settings)
{
Replication = new ReplicationConfiguration();
Voron = new VoronConfiguration();
Prefetcher = new PrefetcherConfiguration();
FileSystem = new FileSystemConfiguration();
Encryption = new EncryptionConfiguration();
this.settings = settings;
}
public ReplicationDestination(AccessControlTranslation accessControlTranslation, string account,
string bucketARN, EncryptionConfiguration encryptionConfiguration,
Metrics metrics, ReplicationTime replicationTime, string storageClass)
{
AccessControlTranslation = accessControlTranslation;
Account = account;
BucketARN = bucketARN;
EncryptionConfiguration = encryptionConfiguration;
Metrics = metrics;
ReplicationTime = replicationTime;
StorageClass = storageClass;
}
public StronglyTypedRavenSettings(NameValueCollection settings)
{
Replication = new ReplicationConfiguration();
Voron = new VoronConfiguration();
Esent = new EsentConfiguration();
Prefetcher = new PrefetcherConfiguration();
FileSystem = new FileSystemConfiguration();
Encryption = new EncryptionConfiguration();
Indexing = new IndexingConfiguration();
WebSockets = new WebSocketsConfiguration();
this.settings = settings;
}
public StronglyTypedRavenSettings(NameValueCollection settings)
{
Replication = new ReplicationConfiguration();
Voron = new VoronConfiguration();
Esent = new EsentConfiguration();
Prefetcher = new PrefetcherConfiguration();
FileSystem = new FileSystemConfiguration();
Encryption = new EncryptionConfiguration();
Indexing = new IndexingConfiguration();
WebSockets = new WebSocketsConfiguration();
this.settings = settings;
}
private async Task CreateVersionCompatData(
ImageFormat format,
FileInfo inputZipFIle,
EncryptionConfiguration encryption,
CompressionConfiguration compression,
EmbeddedImage embeddedImage)
{
var inputFileName = Path.GetFileNameWithoutExtension(inputZipFIle.Name);
string outputName = inputFileName.Substring(0, Math.Min(inputFileName.Length, 10)) + "-" + Enum.GetName(typeof(ImageFormat), format);
if (embeddedImage != null)
{
outputName += "-E";
}
if (encryption != null)
{
outputName += "-" + Enum.GetName(typeof(EncryptionType), encryption.Type).Substring(0, 3);
}
if (compression != null)
{
outputName += "-" + Enum.GetName(typeof(CompressionType), compression.Type).Substring(0, 3);
}
var tempInputDataDirectory = _ouputTempVersionDataDirectory.CreateSubdirectory(outputName + "_original_data");
using (var zipFile = ZipFile.Read(inputZipFIle.FullName))
{
zipFile.ExtractAll(tempInputDataDirectory.FullName);
}
DirectoryInfo outputDirectory = _ouputTempVersionDataDirectory.CreateSubdirectory(outputName);
var encoder = new ImageEncoder(format, encryption, compression, embeddedImage);
using (Stream encodedImageFile = File.Create(Path.Combine(outputDirectory.FullName, inputFileName + "." + encoder.Extension)))
{
encoder.AddDirectory(tempInputDataDirectory);
await encoder.SaveAsync(encodedImageFile, new EncodingConfiguration(
password : "******",
tempStorageProvider : new MemoryStorageProvider(),
bufferSize : 4096,
fileGroupSize : 1024 * 500));
inputZipFIle.CopyTo(Path.Combine(outputDirectory.FullName, "Original_Data.zip"), true);
}
tempInputDataDirectory.Delete(true);
}
public void Encryption_With_Text_Success()
{
EncryptionConfiguration encryptionConfiguration = new EncryptionConfiguration(DefaultKey);
IAesEncryptionHelper aesEncryptionHelper = new AesEncryptionHelper(encryptionConfiguration);
var text = "aesEncrypt";
var encrypted = aesEncryptionHelper.Encrypt(text);
Assert.NotNull(encrypted);
Assert.NotEmpty(encrypted);
}
public async Task EncodingThenDecoding_ProducesEquivalentResults(
ImageFormat format,
FileInfo inputZipFile,
EncryptionConfiguration encryption,
CompressionConfiguration compression,
EmbeddedImage embeddedImage,
IDictionary <string, string> metadata)
{
DirectoryInfo inputDirectory = null;
DirectoryInfo outputDirectory = null;
try
{
using (var storageStream = new MemoryStream())
{
inputDirectory = _inputRootDirectory.CreateSubdirectory(Path.GetFileNameWithoutExtension(inputZipFile.Name));
using (var zipFile = ZipFile.Read(inputZipFile.FullName))
{
zipFile.ExtractAll(inputDirectory.FullName);
}
var encoder = new ImageEncoder(format, encryption, compression, embeddedImage);
encoder.Metadata = metadata;
encoder.AddDirectory(inputDirectory);
await encoder.SaveAsync(storageStream, ImageConfigurations.EncodingConfiguration);
storageStream.Position = 0;
var decoder = await ImageDecoder.LoadAsync(storageStream, ImageConfigurations.DecodingConfiguration);
outputDirectory = _ouputRootDirectory.CreateSubdirectory(inputDirectory.Name);
await decoder.DecodeAsync(outputDirectory);
AssertEx.AreEqualByJson(metadata, decoder.Metadata);
DirectoryAssert.AreEquivalent(inputDirectory, outputDirectory);
}
}
finally
{
foreach (var directory in new[] { inputDirectory, outputDirectory })
{
if (directory != null)
{
directory.Delete(true);
}
}
}
}
public static bool TryToCreateTransactionalStorage(InMemoryRavenConfiguration ravenConfiguration,
bool hasCompression, EncryptionConfiguration encryption, out ITransactionalStorage storage)
{
storage = null;
if (File.Exists(Path.Combine(ravenConfiguration.DataDirectory, Voron.Impl.Constants.DatabaseFilename)))
{
storage = ravenConfiguration.CreateTransactionalStorage(InMemoryRavenConfiguration.VoronTypeName, () => { }, () => { });
}
else if (File.Exists(Path.Combine(ravenConfiguration.DataDirectory, "Data")))
{
storage = ravenConfiguration.CreateTransactionalStorage(InMemoryRavenConfiguration.EsentTypeName, () => { }, () => { });
}
if (storage == null)
{
return(false);
}
var orderedPartCollection = new OrderedPartCollection <AbstractDocumentCodec>();
if (encryption != null)
{
var documentEncryption = new DocumentEncryption();
documentEncryption.SetSettings(new EncryptionSettings(encryption.EncryptionKey, encryption.SymmetricAlgorithmType,
encryption.EncryptIndexes, encryption.PreferedEncryptionKeyBitsSize));
orderedPartCollection.Add(documentEncryption);
}
if (hasCompression)
{
orderedPartCollection.Add(new DocumentCompression());
}
uuidGenerator = new SequentialUuidGenerator();
storage.Initialize(uuidGenerator, orderedPartCollection);
/*
* Added configuration steps
*/
storage.Batch(actions =>
{
var nextIdentityValue = actions.General.GetNextIdentityValue("Raven/Etag");
uuidGenerator.EtagBase = nextIdentityValue;
});
return(true);
}
public StronglyTypedRavenSettings(NameValueCollection settings)
{
Replication = new ReplicationConfiguration();
Voron = new VoronConfiguration();
Esent = new EsentConfiguration();
Prefetcher = new PrefetcherConfiguration();
FileSystem = new FileSystemConfiguration();
Counter = new CounterConfiguration();
TimeSeries = new TimeSeriesConfiguration();
Encryption = new EncryptionConfiguration();
Indexing = new IndexingConfiguration();
WebSockets = new WebSocketsConfiguration();
Cluster = new ClusterConfiguration();
Monitoring = new MonitoringConfiguration();
Studio = new StudioConfiguration();
this.settings = settings;
}
public SymmetricEncryptionService(EncryptionConfiguration encryptionConfiguration = null, ILogger logger = null)
{
if (logger == null)
{
_logger = new LoggerConfiguration()
.WriteTo.Console()
.CreateLogger();
}
else
{
_logger = logger;
}
if (encryptionConfiguration == null)
{
_encryptionConfiguration = new EncryptionConfiguration();
}
else
{
_encryptionConfiguration = encryptionConfiguration;
}
}
public RavenConfiguration()
{
Settings = new NameValueCollection(StringComparer.OrdinalIgnoreCase);
Core = new CoreConfiguration(this);
FileSystem = new FileSystemConfiguration(Core);
Counter = new CounterConfiguration(Core);
TimeSeries = new TimeSeriesConfiguration(Core);
Replication = new ReplicationConfiguration();
Prefetcher = new PrefetcherConfiguration();
Storage = new StorageConfiguration();
Encryption = new EncryptionConfiguration();
Indexing = new IndexingConfiguration();
WebSockets = new WebSocketsConfiguration();
Cluster = new ClusterConfiguration();
Monitoring = new MonitoringConfiguration();
Queries = new QueryConfiguration();
Patching = new PatchingConfiguration();
BulkInsert = new BulkInsertConfiguration();
Server = new ServerConfiguration();
Memory = new MemoryConfiguration();
Expiration = new ExpirationBundleConfiguration();
Versioning = new VersioningBundleConfiguration();
Studio = new StudioConfiguration();
Tenants = new TenantConfiguration();
Licensing = new LicenseConfiguration();
Quotas = new QuotasBundleConfiguration();
IndexingClassifier = new DefaultIndexingClassifier();
Catalog = new AggregateCatalog(CurrentAssemblyCatalog);
Catalog.Changed += (sender, args) => ResetContainer();
}
public async Task <HttpResponseMessage> Encode()
{
EncodingJob encodingJob = null;
string badRequestMessage = null;
try
{
encodingJob = await BuildEncodingJob(Request);
}
catch (BadRequestException exception)
{
badRequestMessage = exception.Message;
}
catch {}
if (encodingJob == null)
{
return(new HttpResponseMessage(HttpStatusCode.BadRequest)
{
Content = new StringContent(badRequestMessage)
});
}
EncryptionConfiguration encryptionConfiguration = null;
CompressionConfiguration compressionConfiguration = null;
if (encodingJob.HasPassword)
{
encryptionConfiguration = new EncryptionConfiguration(
encodingJob.EncryptionAlgorithm.Value,
TranscodingConfiguration.IterationCount);
}
if (encodingJob.CompressionAlgorithm.HasValue)
{
compressionConfiguration = new CompressionConfiguration(encodingJob.CompressionAlgorithm.Value,
encodingJob.CompressionLevel);
}
var imageEncoder = new ImageEncoder(encodingJob.Format, encryptionConfiguration, compressionConfiguration, encodingJob.EmbeddedImage);
imageEncoder.AddDirectories(encodingJob.Directories);
imageEncoder.Metadata.Add("creation-date", DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture));
var guid = SecureGuid();
try
{
DirectoryInfo jobDirectory = CreateJobDirectory(guid);
var outputPath = Path.Combine(jobDirectory.FullName, encodingJob.FileName + "." + imageEncoder.Extension);
using (Stream file = new FileStream(outputPath, FileMode.CreateNew))
{
await imageEncoder.SaveAsync(file, BuildConfiguration(encodingJob.Password));
}
}
catch
{
return(new HttpResponseMessage(HttpStatusCode.InternalServerError));
}
return(Request.CreateResponse(HttpStatusCode.OK, new { id = guid.ToString() }));
}
public override void ExecuteCmdlet()
{
this.ValidateParameters();
FactoryRepoConfiguration repoConfiguration = null;
if (!string.IsNullOrWhiteSpace(this.ProjectName) || !string.IsNullOrWhiteSpace(this.TenantId))
{
var factoryVSTSConfiguration = new FactoryVSTSConfiguration();
factoryVSTSConfiguration.ProjectName = this.ProjectName;
factoryVSTSConfiguration.TenantId = this.TenantId;
repoConfiguration = factoryVSTSConfiguration;
}
else if (!string.IsNullOrWhiteSpace(this.HostName))
{
var factoryGitHubConfiguration = new FactoryGitHubConfiguration();
factoryGitHubConfiguration.HostName = this.HostName;
repoConfiguration = factoryGitHubConfiguration;
}
if (repoConfiguration != null)
{
repoConfiguration.CollaborationBranch = this.CollaborationBranch;
repoConfiguration.AccountName = this.AccountName;
repoConfiguration.LastCommitId = this.LastCommitId;
repoConfiguration.RootFolder = this.RootFolder;
repoConfiguration.RepositoryName = this.RepositoryName;
}
string factoryIdentityType = FactoryIdentityType.SystemAssigned;
if (!string.IsNullOrWhiteSpace(this.IdentityType))
{
factoryIdentityType = this.IdentityType;
}
if (this.UserAssignedIdentity != null && this.UserAssignedIdentity.Count > 0)
{
if (!factoryIdentityType.ToLower().Contains(FactoryIdentityType.UserAssigned.ToLower()))
{
factoryIdentityType = FactoryIdentityType.SystemAssignedUserAssigned;
}
}
FactoryIdentity factoryIdentity = new FactoryIdentity(factoryIdentityType, userAssignedIdentities: this.UserAssignedIdentity);
EncryptionConfiguration encryption = null;
if (!string.IsNullOrWhiteSpace(this.EncryptionVaultBaseUrl) && !string.IsNullOrWhiteSpace(this.EncryptionKeyName))
{
CMKIdentityDefinition cmkIdentity = null;
if (!string.IsNullOrWhiteSpace(this.EncryptionUserAssignedIdentity))
{
cmkIdentity = new CMKIdentityDefinition(this.EncryptionUserAssignedIdentity);
}
encryption = new EncryptionConfiguration(this.EncryptionKeyName, this.EncryptionVaultBaseUrl, this.EncryptionKeyVersion, cmkIdentity);
}
string publicNetworkAccess = Management.DataFactory.Models.PublicNetworkAccess.Enabled;
if (!string.IsNullOrWhiteSpace(this.PublicNetworkAccess))
{
publicNetworkAccess = this.PublicNetworkAccess;
}
var parameters = new CreatePSDataFactoryParameters()
{
ResourceGroupName = ResourceGroupName,
DataFactoryName = Name,
Location = Location,
PublicNetworkAccess = publicNetworkAccess,
EncryptionConfiguration = encryption,
FactoryIdentity = factoryIdentity,
Tags = Tag,
Force = Force.IsPresent,
RepoConfiguration = repoConfiguration,
ConfirmAction = ConfirmAction,
GlobalParameters = GlobalParameterDefinition
};
WriteObject(DataFactoryClient.CreatePSDataFactory(parameters));
}
public ImageEncoder(ImageFormat format, EncryptionConfiguration encryptionConfiguration, EmbeddedImage embeddedImage = null)
: this(format, encryptionConfiguration, null, embeddedImage)
{
}
static void Main(string[] args)
{
var ravenConfiguration = new RavenConfiguration
{
DataDirectory = ConfigurationManager.AppSettings["DataDirectory"],
CacheDocumentsInMemory = false,
RunInMemory = false,
Storage =
{
PreventSchemaUpdate = true,
SkipConsistencyCheck = true
}
};
Console.WriteLine("Configuration initialized.");
var encryptionKey = ConfigurationManager.AppSettings["Raven/Encryption/Key"];
if (!string.IsNullOrWhiteSpace(encryptionKey))
{
Encryption = new EncryptionConfiguration()
{
EncryptIndexes = ConfigurationManager.AppSettings["Raven/Encryption/EncryptIndexes"].Equals("True")
};
if (Encryption.TrySavingEncryptionKey(encryptionKey) == false)
{
ConsoleUtils.ConsoleWriteLineWithColor(ConsoleColor.Red, "Encryption key should be in base64 string format, we got {0}.\n", encryptionKey);
}
var algorithmType = ConfigurationManager.AppSettings["Raven/Encryption/Algorithm"];
if (Encryption.TrySavingAlgorithmType(algorithmType) == false)
{
ConsoleUtils.ConsoleWriteLineWithColor(ConsoleColor.Red, "Unknown encryption algorithm type, we got {0}.\n", algorithmType);
}
var preferedEncryptionKeyBitsSize = ConfigurationManager.AppSettings["Raven/Encryption/KeyBitsPreference"];
if (Encryption.SavePreferedEncryptionKeyBitsSize(preferedEncryptionKeyBitsSize) == false)
{
ConsoleUtils.ConsoleWriteLineWithColor(ConsoleColor.Red, "Encryption key bit size should be in an int, we got {0}.\n", preferedEncryptionKeyBitsSize);
}
Console.WriteLine("Encryption configured");
}
CreateTransactionalStorage(ravenConfiguration);
Console.WriteLine("Transactional storage created.");
try
{
Etag seekAfterEtag = ParseEtag(ConfigurationManager.AppSettings["DocumentTouchCorruptSeekAfterEtag"]),
preTouchEtag,
afterTouchEtag;
// Action info
string[] actionAndTimes = ConfigurationManager.AppSettings["ActionAndTimes"].Split(':');
string action = actionAndTimes[0];
int actionTimes = int.TryParse(actionAndTimes.Length > 1 ? actionAndTimes[1] : "1", out actionTimes) ? actionTimes : 1;
// Document key and tag info
Dictionary <string, Dictionary <string, string> > documentKeyAndEtagJson = JsonConvert.DeserializeObject <Dictionary <string, Dictionary <string, string> > >(File.ReadAllText(ConfigurationManager.AppSettings["DocumentKeyAndEtagJsonPath"]));
Dictionary <string, List <KeyValuePair <string, Etag> > > documentKeyAndEtagCasted = new Dictionary <string, List <KeyValuePair <string, Etag> > >();
foreach (var keyAndEtagDic in documentKeyAndEtagJson)
{
documentKeyAndEtagCasted.Add(keyAndEtagDic.Key, keyAndEtagDic.Value.Select(k =>
{
string documentKey = k.Value;
Etag etag = ParseEtag(k.Key);
return(new KeyValuePair <string, Etag>(documentKey, etag));
}).ToList());
}
var documentKeyAndEtag = documentKeyAndEtagCasted[action];
switch (action)
{
case "doc_json":
/**
* Try json document
*/
var jsonDocumentDic = new Dictionary <string, JsonDocument>();
var metadataOnly = ConfigurationManager.AppSettings["DocumentsJsonMetadataOnly"] == "true";
var outputFields = ConfigurationManager.AppSettings["DocumentsJsonOutputFields"].Split(';');
foreach (var keyValuePair in documentKeyAndEtag)
{
storage.Batch(a =>
{
try
{
JsonDocument jsonDocument;
if (metadataOnly)
{
var metadata = a.Documents.DocumentMetadataByKey(keyValuePair.Key);
jsonDocument = new JsonDocument()
{
DataAsJson = RavenJObject.FromObject(new object()),
Metadata = metadata.Metadata,
Etag = metadata.Etag,
LastModified = metadata.LastModified,
Key = metadata.Key
};
}
else
{
jsonDocument = a.Documents.DocumentByKey(keyValuePair.Key);
}
Console.WriteLine($"Key: {jsonDocument}, Etag: {jsonDocument.Etag}");
Console.WriteLine($"Metadata: {jsonDocument.Metadata}");
Console.WriteLine("");
jsonDocumentDic.Add(keyValuePair.Key, jsonDocument);
}
catch (Exception ex)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine($"Key: {keyValuePair.Key}, Exception: {ex}");
Console.ResetColor();
jsonDocumentDic.Add(keyValuePair.Key, new JsonDocument()
{
DataAsJson = RavenJObject.FromObject(ex),
Metadata = new RavenJObject()
});
}
});
}
var jsonDocumentDicSerialized = jsonDocumentDic.Select(d =>
{
var key = d.Key;
//var dataAsJson = d.Value.DataAsJson.ToString();
//var metadata = d.Value.Metadata.ToString();
//var lastModified = d.Value.LastModified.ToString();
var objDictionary = new Dictionary <string, string>()
{
{ "DataAsJson", d.Value.DataAsJson.ToString() },
{ "Metadata", d.Value.Metadata.ToString() },
{ "LastModified", d.Value.LastModified.ToString() }
}
.Where(f => outputFields.Contains(f.Key))
.ToDictionary(f => f.Key, f => f.Value);
return(new KeyValuePair <string, Dictionary <string, string> >(key, objDictionary));
}).ToDictionary(d => d.Key, d => d.Value);
var newJsonName = string.Format(ConfigurationManager.AppSettings["DocumentsJsonPath"], DateTime.Now.ToString("s").Replace(':', '_'));
File.WriteAllText(newJsonName, JsonConvert.SerializeObject(jsonDocumentDicSerialized));
break;
case "doc_touch_corrupt":
/**
* Try deleting the document
*/
foreach (var keyValuePair in documentKeyAndEtag)
{
for (int i = 0; i < actionTimes; i++)
{
storage.Batch(a =>
{
a.Documents.TouchCorruptDocumentPub(keyValuePair.Key, out preTouchEtag, out afterTouchEtag, seekAfterEtag);
Console.WriteLine($"preTouchEtag: {preTouchEtag}, afterTouchEtag: {afterTouchEtag}");
});
}
}
break;
case "doc_touch":
/**
* Try deleting the document
*/
foreach (var keyValuePair in documentKeyAndEtag)
{
for (int i = 0; i < actionTimes; i++)
{
storage.Batch(a =>
{
a.Documents.TouchDocument(keyValuePair.Key, out preTouchEtag, out afterTouchEtag);
Console.WriteLine($"preTouchEtag: {preTouchEtag}, afterTouchEtag: {afterTouchEtag}");
});
}
}
break;
case "doc_delete":
/**
* Try deleting the document
*/
RavenJObject outMetadata;
Etag outEtag;
foreach (var keyValuePair in documentKeyAndEtag)
{
for (int i = 0; i < actionTimes; i++)
{
storage.Batch(a =>
{
Console.WriteLine($"Deleting document by key: '{keyValuePair.Key}'");
a.Documents.DeleteDocument(keyValuePair.Key, null, out outMetadata, out outEtag);
Console.WriteLine($"outEtag: {outEtag}, outMetadata: {outMetadata}");
});
}
}
break;
case "doc_update_metadata":
/**
* Trying to overwrite the metadata with empty metadata
*/
foreach (var keyValuePair in documentKeyAndEtag)
{
JsonDocumentMetadata metadata = new JsonDocumentMetadata()
{
Etag = keyValuePair.Value,
Key = keyValuePair.Key,
Metadata = new RavenJObject()
};
storage.Batch(a => a.Documents.WriteDocumentMetadataPub(metadata, metadata.Key, true));
}
break;
case "documents_key_etag":
/**
* Get the table of keys and tags for each document
*/
storage.Batch(a =>
{
var writeCount = 0;
var newFileName = string.Format(ConfigurationManager.AppSettings["DocumentsKeyEtagPath"], DateTime.Now.ToString("s").Replace(':', '_'));
var outputCorruptOnly = ConfigurationManager.AppSettings["DocumentsKeyEtagCorruptOnly"] == "true";
var outputCorruptIndicators = ConfigurationManager.AppSettings["DocumentsKeyEtagOutputCorruptIndicators"] == "true";
Etag startAtEtag = ParseEtag(ConfigurationManager.AppSettings["DocumentsKeyEtagStartAtEtag"]);
int take = ParseInt(ConfigurationManager.AppSettings["DocumentsKeyEtagTake"], int.MaxValue, "Max");
var timer = Stopwatch.StartNew();
using (var writer = new StreamWriter(newFileName))
{
var templateHeader = outputCorruptIndicators
? "Etag, Metadata, Document, DocumentKey, Exception"
: "Etag, Document key";
// Write column headers
writer.WriteLine(templateHeader);
var keyEtags = a.Documents.GetKeysAfterWithIdStartingWithPub(startAtEtag, take, includeMetadataCanBeReadFlag: outputCorruptIndicators, includeDocumentCanBeReadFlag: outputCorruptIndicators);
foreach (var etagTuple in keyEtags)
{
if (writeCount >= 10000)
{
Console.Write(".");
writeCount = 0;
}
writeCount++;
//Console.WriteLine($"{keyValuePair.Key} - {keyValuePair.Value}");
if (outputCorruptOnly && etagTuple.Item5 == null)
{
continue;
}
var templateLine = outputCorruptIndicators
? $"{etagTuple.Item1}, {etagTuple.Item3}, {etagTuple.Item4}, {etagTuple.Item2}, {JsonConvert.SerializeObject(etagTuple.Item5)}"
: $"{etagTuple.Item1}, {etagTuple.Item2}";
// Write lines
writer.WriteLine(templateLine);
}
Console.WriteLine("");
Console.WriteLine($"Completed in {timer.Elapsed.ToString()}");
Console.WriteLine($"Generated file '{newFileName}'");
}
});
break;
case "compare_document_key_csvs":
var timer1 = Stopwatch.StartNew();
// Load the sets
List <HashSet <string> > hashSets = new List <HashSet <string> >();
for (int i = 0; i < 2; i++)
{
var fileConfig = ConfigurationManager.AppSettings[$"CompareDocumentsKeyCsvsPath{i+1}"].Split(',');
int columnNumber = int.TryParse(fileConfig.Length > 1 ? fileConfig[1] : "0", out columnNumber) ? columnNumber : 0;
var set = GetDocumentKeysFromCsv(fileConfig[0], columnNumber);
hashSets.Add(set);
}
// Compare the sets
List <IEnumerable <string> > diffs = new List <IEnumerable <string> >
{
hashSets[0].Except(hashSets[1]),
hashSets[1].Except(hashSets[0])
};
// Save the diffs
for (int i = 0; i < 2; i++)
{
var newFileName = string.Format(ConfigurationManager.AppSettings["CompareDocumentsKeyPath"], DateTime.Now.ToString("s").Replace(':', '_'), i == 0 ? "1to2" : "2to1");
var writeCount = 0;
using (var writer = new StreamWriter(newFileName))
{
// Write column headers
writer.WriteLine("Document key");
foreach (var diff in diffs[i])
{
if (writeCount >= 10000)
{
Console.Write(".");
writeCount = 0;
}
writeCount++;
// Write lines
writer.WriteLine(diff);
}
}
Console.WriteLine($"Generated file '{newFileName}'");
}
Console.WriteLine($"Completed in {timer1.Elapsed}");
break;
default:
Console.WriteLine("Allowed actions are: doc_delete, doc_update_metadata, documents_key_etag");
break;
}
Console.WriteLine("Operation succeeded.");
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
finally
{
storage.Dispose();
}
Console.Read();
}
