public HttpResponseMessage Get([FromUri] string[] fileNames)
{
var list = new List <FileHeader>();
var startsWith = GetQueryStringValue("startsWith");
if (string.IsNullOrEmpty(startsWith) == false)
{
var matches = GetQueryStringValue("matches");
var endsWithSlash = startsWith.EndsWith("/") || startsWith.EndsWith("\\");
startsWith = FileHeader.Canonize(startsWith);
if (endsWithSlash)
{
startsWith += "/";
}
Storage.Batch(accessor =>
{
var actualStart = 0;
var filesToSkip = Paging.Start;
int fileCount, matchedFiles = 0, addedFiles = 0;
do
{
fileCount = 0;
foreach (var file in accessor.GetFilesStartingWith(startsWith, actualStart, Paging.PageSize))
{
fileCount++;
var keyTest = file.FullPath.Substring(startsWith.Length);
if (WildcardMatcher.Matches(matches, keyTest) == false)
{
continue;
}
if (FileSystem.ReadTriggers.CanReadFile(FileHeader.Canonize(file.FullPath), file.Metadata, ReadOperation.Load) == false)
{
continue;
}
if (file.Metadata.Keys.Contains(SynchronizationConstants.RavenDeleteMarker))
{
continue;
}
matchedFiles++;
if (matchedFiles <= filesToSkip)
{
continue;
}
list.Add(file);
addedFiles++;
}
actualStart += Paging.PageSize;
}while (fileCount > 0 && addedFiles < Paging.PageSize && actualStart > 0 && actualStart < int.MaxValue);
});
}
else
{
if (fileNames != null && fileNames.Length > 0)
{
Storage.Batch(accessor =>
{
foreach (var path in fileNames.Where(x => x != null).Select(FileHeader.Canonize))
{
var file = accessor.ReadFile(path);
if (file == null || file.Metadata.Keys.Contains(SynchronizationConstants.RavenDeleteMarker))
{
list.Add(null);
continue;
}
list.Add(file);
}
});
}
else
{
int results;
var keys = Search.Query(null, null, Paging.Start, Paging.PageSize, out results);
Storage.Batch(accessor => list.AddRange(keys.Select(accessor.ReadFile).Where(x => x != null)));
}
}
return(GetMessageWithObject(list)
.WithNoCache());
}
public FieldTermStack(IndexReader reader, int docId, String fieldName, FieldQuery fieldQuery)
{
this.fieldName = fieldName;
var tfv = reader.GetTermFreqVector(docId, fieldName);
if (tfv == null)
{
return; // just return to make null snippets
}
TermPositionVector tpv = null;
try
{
tpv = (TermPositionVector)tfv;
}
catch (InvalidCastException)
{
return; // just return to make null snippets
}
List <String> termSet = fieldQuery.getTermSet(fieldName);
// just return to make null snippet if un-matched fieldName specified when fieldMatch == true
if (termSet == null)
{
return;
}
var needwildcard = termSet.Any(x => x.IndexOfAny(new char[] { '*', '?' }) != -1);
foreach (String term in tpv.GetTerms())
{
if (needwildcard)
{
if (termSet.Any(ts => WildcardMatcher.Matches(ts, term)) == false)
{
continue;
}
}
else if (!termSet.Contains(term))
{
continue;
}
int index = tpv.IndexOf(term);
TermVectorOffsetInfo[] tvois = tpv.GetOffsets(index);
if (tvois == null)
{
return; // just return to make null snippets
}
int[] poss = tpv.GetTermPositions(index);
if (poss == null)
{
return; // just return to make null snippets
}
for (int i = 0; i < tvois.Length; i++)
{
termList.AddLast(new TermInfo(term, tvois[i].StartOffset, tvois[i].EndOffset, poss[i]));
}
}
// sort by position
//Collections.sort(termList);
Sort(termList);
}
public void CanMatch(string pattern, string input, bool expected)
{
Assert.Equal(expected, WildcardMatcher.Matches(pattern, input));
}
public void GetDocumentsWithIdStartingWith(string idPrefix, string matches, string exclude, int start, int pageSize,
CancellationToken token, ref int nextStart, Action <RavenJObject> addDoc,
string transformer = null, Dictionary <string, RavenJToken> queryInputs = null)
{
if (idPrefix == null)
{
throw new ArgumentNullException("idPrefix");
}
idPrefix = idPrefix.Trim();
var canPerformRapidPagination = nextStart > 0 && start == nextStart;
var actualStart = canPerformRapidPagination ? start : 0;
var addedDocs = 0;
var matchedDocs = 0;
TransactionalStorage.Batch(
actions =>
{
var docsToSkip = canPerformRapidPagination ? 0 : start;
int docCount;
AbstractTransformer storedTransformer = null;
if (transformer != null)
{
storedTransformer = IndexDefinitionStorage.GetTransformer(transformer);
if (storedTransformer == null)
{
throw new InvalidOperationException("No transformer with the name: " + transformer);
}
}
do
{
docCount = 0;
var docs = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, actualStart, pageSize);
var documentRetriever = new DocumentRetriever(actions, Database.ReadTriggers, Database.InFlightTransactionalState, queryInputs);
foreach (var doc in docs)
{
token.ThrowIfCancellationRequested();
docCount++;
var keyTest = doc.Key.Substring(idPrefix.Length);
if (!WildcardMatcher.Matches(matches, keyTest) || WildcardMatcher.MatchesExclusion(exclude, keyTest))
{
continue;
}
DocumentRetriever.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key);
var document = nonAuthoritativeInformationBehavior != null ? nonAuthoritativeInformationBehavior(doc) : doc;
document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load);
if (document == null)
{
continue;
}
matchedDocs++;
if (matchedDocs <= docsToSkip)
{
continue;
}
token.ThrowIfCancellationRequested();
if (storedTransformer != null)
{
using (new CurrentTransformationScope(documentRetriever))
{
var transformed =
storedTransformer.TransformResultsDefinition(new[] { new DynamicJsonObject(document.ToJson()) })
.Select(x => JsonExtensions.ToJObject(x))
.ToArray();
if (transformed.Length == 0)
{
throw new InvalidOperationException("The transform results function failed on a document: " + document.Key);
}
var transformedJsonDocument = new JsonDocument
{
Etag = document.Etag.HashWith(storedTransformer.GetHashCodeBytes()).HashWith(documentRetriever.Etag),
NonAuthoritativeInformation = document.NonAuthoritativeInformation,
LastModified = document.LastModified,
DataAsJson = new RavenJObject {
{ "$values", new RavenJArray(transformed) }
},
};
addDoc(transformedJsonDocument.ToJson());
}
}
else
{
addDoc(document.ToJson());
}
addedDocs++;
if (addedDocs >= pageSize)
{
break;
}
}
actualStart += pageSize;
}while (docCount > 0 && addedDocs < pageSize && actualStart > 0 && actualStart < int.MaxValue);
});
if (addedDocs != pageSize)
{
nextStart = start; // will mark as last page
}
else if (canPerformRapidPagination)
{
nextStart = start + matchedDocs;
}
else
{
nextStart = actualStart;
}
}
public void GetDocumentsWithIdStartingWith(string idPrefix, string matches, string exclude, int start, int pageSize,
CancellationToken token, ref int nextStart, Action <JsonDocument> addDoc,
string transformer = null, Dictionary <string, RavenJToken> transformerParameters = null,
string skipAfter = null)
{
if (idPrefix == null)
{
throw new ArgumentNullException("idPrefix");
}
idPrefix = idPrefix.Trim();
var canPerformRapidPagination = nextStart > 0 && start == nextStart;
var actualStart = canPerformRapidPagination ? start : 0;
var addedDocs = 0;
var docCountOnLastAdd = 0;
var matchedDocs = 0;
TransactionalStorage.Batch(
actions =>
{
var docsToSkip = canPerformRapidPagination ? 0 : start;
int docCount;
AbstractTransformer storedTransformer = null;
if (transformer != null)
{
storedTransformer = IndexDefinitionStorage.GetTransformer(transformer);
if (storedTransformer == null)
{
throw new InvalidOperationException("No transformer with the name: " + transformer);
}
}
do
{
Database.WorkContext.UpdateFoundWork();
docCount = 0;
var docs = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, actualStart, pageSize, string.IsNullOrEmpty(skipAfter) ? null : skipAfter);
var documentRetriever = new DocumentRetriever(Database.Configuration, actions, Database.ReadTriggers, transformerParameters);
foreach (var doc in docs)
{
token.ThrowIfCancellationRequested();
docCount++;
if (docCount - docCountOnLastAdd > 1000)
{
addDoc(null); // heartbeat
}
var keyTest = doc.Key.Substring(idPrefix.Length);
if (!WildcardMatcher.Matches(matches, keyTest) || WildcardMatcher.MatchesExclusion(exclude, keyTest))
{
continue;
}
JsonDocument.EnsureIdInMetadata(doc);
var document = documentRetriever.ExecuteReadTriggers(doc, ReadOperation.Load);
if (document == null)
{
continue;
}
matchedDocs++;
if (matchedDocs <= docsToSkip)
{
continue;
}
token.ThrowIfCancellationRequested();
document = TransformDocumentIfNeeded(document, storedTransformer, documentRetriever);
addDoc(document);
addedDocs++;
docCountOnLastAdd = docCount;
if (addedDocs >= pageSize)
{
break;
}
}
actualStart += pageSize;
}while (docCount > 0 && addedDocs < pageSize && actualStart > 0 && actualStart < int.MaxValue);
});
if (addedDocs != pageSize)
{
nextStart = start; // will mark as last page
}
else if (canPerformRapidPagination)
{
nextStart = start + matchedDocs;
}
else
{
nextStart = actualStart;
}
}