lucene - 如何在 Lucene.Net 4.8 中使用 HIGH_COMPRESSION
问题描述
我正在尝试尽可能地压缩索引大小，请问有什么帮助吗？ https://lucenenet.apache.org/docs/4.8.0-beta00013/api/core/Lucene.Net.Codecs.Compressing.CompressionMode.html#Lucene_Net_Codecs_Compressing_CompressionMode_HIGH_COMPRESSION
public class LuceneIndexer
{
private Analyzer _analyzer = new ArabicAnalyzer(Lucene.Net.Util.LuceneVersion.LUCENE_48);
private string _indexPath;
private Directory _indexDirectory;
public IndexWriter _indexWriter;
public LuceneIndexer(string indexPath)
{
this._indexPath = indexPath;
_indexDirectory = new SimpleFSDirectory(new System.IO.DirectoryInfo(_indexPath));
}
public void BuildCompleteIndex(IEnumerable<Document> documents)
{
IndexWriterConfig indexWriterConfig = new IndexWriterConfig(Lucene.Net.Util.LuceneVersion.LUCENE_48, _analyzer) { OpenMode = OpenMode.CREATE_OR_APPEND };
indexWriterConfig.MaxBufferedDocs = 2;
indexWriterConfig.RAMBufferSizeMB = 128;
indexWriterConfig.MaxThreadStates = 2;
_indexWriter = new IndexWriter(_indexDirectory, indexWriterConfig);
_indexWriter.AddDocuments(documents);
_indexWriter.Flush(true, true);
_indexWriter.Commit();
_indexWriter.Dispose();
}
public IEnumerable<Document> Search(string searchTerm, string searchField, int limit)
{
IndexReader indexReader = DirectoryReader.Open(_indexDirectory);
var searcher = new IndexSearcher(indexReader);
var termQuery = new TermQuery(new Term(searchField, searchTerm)); // Lucene.Net.Util.LuceneVersion.LUCENE_48, searchField, _analyzer
var hits = searcher.Search(termQuery, limit).ScoreDocs;
var documents = new List<Document>();
foreach (var hit in hits)
{
documents.Add(searcher.Doc(hit.Doc));
}
_analyzer.Dispose();
return documents;
}
}
解决方案
首先要知道的是，“Lucene Index”有很多方面。当不使用复合文件时，这会在创建的各种文件中体现出来。只看其中两个，我们可以谈论倒排索引，也就是所谓的帖子，我们可以谈论存储的文档。在这两者中，据我所知，没有任何现成的关于倒排索引压缩的可调设置。
HIGH_COMPRESSION 模式与存储的字段有关。如果您不存储字段并且仅使用 Lucene.Net 创建倒排索引，那么为存储字段打开高压缩不会减少“Lucene 索引”的大小。
也就是说，如果您正在存储字段并希望对该存储的字段数据使用高压缩，那么您将需要创建自己的编解码器，为存储的字段启用高压缩。为此，您首先需要一个启用了高压缩的存储字段类。下面是这两个类，后面是使用我为您编写的新编解码器的单元测试。我还没有在大量数据上尝试过这段代码来查看效果，我把它留给你作为练习，但这应该指出使用高压缩压缩存储字段的方法。
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
public sealed class Lucene41StoredFieldsHighCompressionFormat : CompressingStoredFieldsFormat {
/// <summary>
/// Sole constructor. </summary>
public Lucene41StoredFieldsHighCompressionFormat()
: base("Lucene41StoredFieldsHighCompression", CompressionMode.HIGH_COMPRESSION, 1 << 14) {
}
}
这是使用这种高压缩格式的自定义编解码器：
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using Lucene40LiveDocsFormat = Lucene.Net.Codecs.Lucene40.Lucene40LiveDocsFormat;
using Lucene41StoredFieldsFormat = Lucene.Net.Codecs.Lucene41.Lucene41StoredFieldsFormat;
using Lucene42NormsFormat = Lucene.Net.Codecs.Lucene42.Lucene42NormsFormat;
using Lucene42TermVectorsFormat = Lucene.Net.Codecs.Lucene42.Lucene42TermVectorsFormat;
using PerFieldDocValuesFormat = Lucene.Net.Codecs.PerField.PerFieldDocValuesFormat;
using PerFieldPostingsFormat = Lucene.Net.Codecs.PerField.PerFieldPostingsFormat;
/// <summary>
/// Implements the Lucene 4.6 index format, with configurable per-field postings
/// and docvalues formats.
/// <para/>
/// If you want to reuse functionality of this codec in another codec, extend
/// <see cref="FilterCodec"/>.
/// <para/>
/// See <see cref="Lucene.Net.Codecs.Lucene46"/> package documentation for file format details.
/// <para/>
/// @lucene.experimental
/// </summary>
// NOTE: if we make largish changes in a minor release, easier to just make Lucene46Codec or whatever
// if they are backwards compatible or smallish we can probably do the backwards in the postingsreader
// (it writes a minor version, etc).
[CodecName("Lucene46HighCompression")]
public class Lucene46HighCompressionCodec : Codec {
private readonly StoredFieldsFormat fieldsFormat = new Lucene41StoredFieldsHighCompressionFormat(); //<--This is the only line different then the stock Lucene46Codec
private readonly TermVectorsFormat vectorsFormat = new Lucene42TermVectorsFormat();
private readonly FieldInfosFormat fieldInfosFormat = new Lucene46FieldInfosFormat();
private readonly SegmentInfoFormat segmentInfosFormat = new Lucene46SegmentInfoFormat();
private readonly LiveDocsFormat liveDocsFormat = new Lucene40LiveDocsFormat();
private readonly PostingsFormat postingsFormat;
private class PerFieldPostingsFormatAnonymousInnerClassHelper : PerFieldPostingsFormat {
private readonly Lucene46HighCompressionCodec outerInstance;
public PerFieldPostingsFormatAnonymousInnerClassHelper(Lucene46HighCompressionCodec outerInstance) {
this.outerInstance = outerInstance;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override PostingsFormat GetPostingsFormatForField(string field) {
return outerInstance.GetPostingsFormatForField(field);
}
}
private readonly DocValuesFormat docValuesFormat;
private class PerFieldDocValuesFormatAnonymousInnerClassHelper : PerFieldDocValuesFormat {
private readonly Lucene46HighCompressionCodec outerInstance;
public PerFieldDocValuesFormatAnonymousInnerClassHelper(Lucene46HighCompressionCodec outerInstance) {
this.outerInstance = outerInstance;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override DocValuesFormat GetDocValuesFormatForField(string field) {
return outerInstance.GetDocValuesFormatForField(field);
}
}
/// <summary>
/// Sole constructor. </summary>
public Lucene46HighCompressionCodec()
: base() {
postingsFormat = new PerFieldPostingsFormatAnonymousInnerClassHelper(this);
docValuesFormat = new PerFieldDocValuesFormatAnonymousInnerClassHelper(this);
}
public override sealed StoredFieldsFormat StoredFieldsFormat => fieldsFormat;
public override sealed TermVectorsFormat TermVectorsFormat => vectorsFormat;
public override sealed PostingsFormat PostingsFormat => postingsFormat;
public override sealed FieldInfosFormat FieldInfosFormat => fieldInfosFormat;
public override sealed SegmentInfoFormat SegmentInfoFormat => segmentInfosFormat;
public override sealed LiveDocsFormat LiveDocsFormat => liveDocsFormat;
/// <summary>
/// Returns the postings format that should be used for writing
/// new segments of <paramref name="field"/>.
/// <para/>
/// The default implementation always returns "Lucene41"
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public virtual PostingsFormat GetPostingsFormatForField(string field) {
// LUCENENET specific - lazy initialize the codec to ensure we get the correct type if overridden.
if (defaultFormat == null) {
defaultFormat = Lucene.Net.Codecs.PostingsFormat.ForName("Lucene41");
}
return defaultFormat;
}
/// <summary>
/// Returns the docvalues format that should be used for writing
/// new segments of <paramref name="field"/>.
/// <para/>
/// The default implementation always returns "Lucene45"
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public virtual DocValuesFormat GetDocValuesFormatForField(string field) {
// LUCENENET specific - lazy initialize the codec to ensure we get the correct type if overridden.
if (defaultDVFormat == null) {
defaultDVFormat = Lucene.Net.Codecs.DocValuesFormat.ForName("Lucene45");
}
return defaultDVFormat;
}
public override sealed DocValuesFormat DocValuesFormat => docValuesFormat;
// LUCENENET specific - lazy initialize the codecs to ensure we get the correct type if overridden.
private PostingsFormat defaultFormat;
private DocValuesFormat defaultDVFormat;
private readonly NormsFormat normsFormat = new Lucene42NormsFormat();
public override sealed NormsFormat NormsFormat => normsFormat;
}
感谢@NightOwl888，我现在了解到您还需要在启动时注册新的编解码器，如下所示：
Codec.SetCodecFactory(new DefaultCodecFactory {
CustomCodecTypes = new Type[] { typeof(Lucene46HighCompressionCodec) }
});
这是一个演示使用高压缩编解码器的单元测试：
public class TestCompression {
[Fact]
public void HighCompression() {
FxTest.Setup();
Directory indexDir = new RAMDirectory();
Analyzer standardAnalyzer = new StandardAnalyzer(LuceneVersion.LUCENE_48);
IndexWriterConfig indexConfig = new IndexWriterConfig(LuceneVersion.LUCENE_48, standardAnalyzer);
indexConfig.Codec = new Lucene46HighCompressionCodec(); //<--------Install the High Compression codec.
indexConfig.UseCompoundFile = true;
IndexWriter writer = new IndexWriter(indexDir, indexConfig);
//souce: https://github.com/apache/lucenenet/blob/Lucene.Net_4_8_0_beta00006/src/Lucene.Net/Search/SearcherFactory.cs
SearcherManager searcherManager = new SearcherManager(writer, applyAllDeletes: true, new SearchWarmer());
Document doc = new Document();
doc.Add(new StringField("examplePrimaryKey", "001", Field.Store.YES));
doc.Add(new TextField("exampleField", "Unique gifts are great gifts.", Field.Store.YES));
writer.AddDocument(doc);
doc = new Document();
doc.Add(new StringField("examplePrimaryKey", "002", Field.Store.YES));
doc.Add(new TextField("exampleField", "Everyone is gifted.", Field.Store.YES));
writer.AddDocument(doc);
doc = new Document();
doc.Add(new StringField("examplePrimaryKey", "003", Field.Store.YES));
doc.Add(new TextField("exampleField", "Gifts are meant to be shared.", Field.Store.YES));
writer.AddDocument(doc);
writer.Commit();
searcherManager.MaybeRefreshBlocking();
IndexSearcher indexSearcher = searcherManager.Acquire();
try {
QueryParser parser = new QueryParser(LuceneVersion.LUCENE_48, "exampleField", standardAnalyzer);
Query query = parser.Parse("everyone");
TopDocs topDocs = indexSearcher.Search(query, int.MaxValue);
int numMatchingDocs = topDocs.ScoreDocs.Length;
Assert.Equal(1, numMatchingDocs);
Document docRead = indexSearcher.Doc(topDocs.ScoreDocs[0].Doc);
string primaryKey = docRead.Get("examplePrimaryKey");
Assert.Equal("002", primaryKey);
} finally {
searcherManager.Release(indexSearcher);
}
}
}
虽然我最初的回应是通过Lucene.Net github issue，但我在这里提供了答案，它将对 Lucene.Net 社区有更好的可见性，希望它也能帮助其他人。对于那些感兴趣的人，有更多关于在该问题线程末尾使用自定义编解码器的背景信息。