java - Solr 8.6 中的矢量评分插件
问题描述
我正在尝试将VectorScoringPlugin升级到 Solr 8.6。在检查 Lucene 8.x 的迁移日志后，我了解到插件中使用的类 viz.CustomScoreQuery 和 CustomScoreProvider 已被弃用，相反，我们必须将 FunctionScoreQuery 与 DoubleValuesSource 一起使用。我做了很多搜索，但找不到使用上述类实现自定义记分器的任何示例。我在 java-lucene 论坛上偶然发现了这 2 个线程[thread1 , thread2]，它们基本上讨论了相同的问题，并且提到的解决方案是实现一个自定义的 DoubleValuesSource 类，该类将具有自定义逻辑。下面是实现；
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.analysis.payloads.PayloadHelper;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.DoubleValues;
import org.apache.lucene.search.DoubleValuesSource;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.SolrException;
public class CustomDoubleValueSource extends DoubleValuesSource {
List<Double> vector;
private String field;
private boolean cosine;
double queryVectorNorm = 0;
public CustomDoubleValueSource(String field, String Vector, boolean cosine) {
// TODO Auto-generated constructor stub
super();
this.field = field;
this.cosine = cosine;
this.vector = new ArrayList<Double>();
String[] vectorArray = Vector.split(",");
for (int i = 0; i < vectorArray.length; i++) {
double v = Double.parseDouble(vectorArray[i]);
vector.add(v);
if (cosine) {
queryVectorNorm += Math.pow(v, 2.0);
}
}
System.out.println("Vector size:"+this.vector.size());
}
@Override
public boolean isCacheable(LeafReaderContext ctx) {
// TODO Auto-generated method stub
return false;
}
@Override
public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
Terms terms = ctx.reader().terms(field);
TermsEnum te = terms == null ? null : terms.iterator();
System.out.println("Term size:"+terms.size());
if (vector.size() != terms.size()) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"indexed and input vector array must have same length");
}
final PostingsEnum pe = te.postings(null);
// TODO Auto-generated method stub
return new DoubleValues() {
@Override
public double doubleValue() throws IOException {
// TODO Auto-generated method stub
float score = 0;
double docVectorNorm = 0;
BytesRef text;
while ((text = te.next()) != null) {
String term = text.utf8ToString();
float payloadValue = 0f;
PostingsEnum postings = te.postings(null, PostingsEnum.ALL);
while (postings.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
int freq = postings.freq();
while (freq-- > 0)
postings.nextPosition();
BytesRef payload = postings.getPayload();
payloadValue = PayloadHelper.decodeFloat(payload.bytes, payload.offset);
if (cosine)
docVectorNorm += Math.pow(payloadValue, 2.0);
}
score = (float) (score + payloadValue * (vector.get(Integer.parseInt(term))));
}
if (cosine) {
if ((docVectorNorm == 0) || (queryVectorNorm == 0))
return 0f;
return (float) (score / (Math.sqrt(docVectorNorm) * Math.sqrt(queryVectorNorm)));
}
return score;
}
@Override
public boolean advanceExact(int doc) throws IOException {
// TODO Auto-generated method stub
if (pe.docID() > doc)
return false;
return pe.docID() == doc || pe.advance(doc) == doc;
}
};
}
@Override
public boolean needsScores() {
// TODO Auto-generated method stub
return true;
}
@Override
public DoubleValuesSource rewrite(IndexSearcher reader) throws IOException {
// TODO Auto-generated method stub
return null;
}
@Override
public int hashCode() {
// TODO Auto-generated method stub
return 0;
}
@Override
public boolean equals(Object obj) {
// TODO Auto-generated method stub
return false;
}
@Override
public String toString() {
// TODO Auto-generated method stub
return null;
}
}
我正在使用上述自定义类，如下所示；
import org.apache.lucene.queries.function.FunctionScoreQuery;
import org.apache.lucene.search.Query;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.FieldType;
import org.apache.solr.search.QParser;
import org.apache.solr.search.QParserPlugin;
import org.apache.solr.search.QueryParsing;
import org.apache.solr.search.SyntaxError;
public class VectorQParserPlugin extends QParserPlugin {
@Override
public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
return new QParser(qstr, localParams, params, req) {
@Override
public Query parse() throws SyntaxError {
String field = localParams.get(QueryParsing.F);
String vector = localParams.get("vector");
boolean cosine = localParams.getBool("cosine", true);
if (field == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "'f' not specified");
}
if (vector == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "vector missing");
}
System.out.println("FIELD:"+field);
Query subQuery = subQuery(localParams.get(QueryParsing.V), null).getQuery();
FieldType ft = req.getCore().getLatestSchema().getFieldType(field);
if(ft != null) {
System.out.println("in here");
VectorQuery q = new VectorQuery(subQuery);
q.setQueryString(localParams.toLocalParamsString());
query = q;
}
System.out.println("QUERY:"+query);
if (query == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Query is null");
}
return FunctionScoreQuery.boostByValue(query, new CustomDoubleValueSource(field, vector, cosine));
}
};
}
}
此外，我将自定义 Query 实现升级到特定于 8.6 的版本，以避免Query 没有实现 createWieght错误。
下面是 VectorQuery 的实现；
import java.io.IOException;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.ConstantScoreScorer;
import org.apache.lucene.search.ConstantScoreWeight;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Weight;
public class VectorQuery extends Query {
String queryStr = "";
Query q;
public VectorQuery(Query subQuery) {
this.q = subQuery;
}
public void setQueryString(String queryString){
this.queryStr = queryString;
}
public Weight createWeight(IndexSearcher searcher, ScoreMode needsScores, float boost) throws IOException {
Weight w;
if(q == null){
w = new ConstantScoreWeight(this, boost) {
@Override
public Scorer scorer(LeafReaderContext context) throws IOException {
return new ConstantScoreScorer(this, score(), needsScores, DocIdSetIterator.all(context.reader().maxDoc()));
}
@Override
public boolean isCacheable(LeafReaderContext ctx) {
// TODO Auto-generated method stub
return false;
}
};
}else{
w = searcher.createWeight(q, needsScores, boost);
}
return w;
}
@Override
public String toString(String field) {
return queryStr;
}
@Override
public boolean equals(Object other) {
return sameClassAs(other) &&
queryStr.equals(other.toString());
}
@Override
public int hashCode() {
return classHash() ^ queryStr.hashCode();
}
}
我添加了打印语句来检查执行流程，并调用了 CustomDoubleValueSource 类。下面是日志的截图。
但是执行流程没有到达 getValues 方法。我收到以下错误；
2020-10-21 16:55:09.578 ERROR (qtp1962826816-19) [ x:example_vector] o.a.s.s.HttpSolrCall null:java.lang.NullPointerException
at org.apache.lucene.queries.function.FunctionScoreQuery$MultiplicativeBoostValuesSource.getValues(FunctionScoreQuery.java:261)
at org.apache.lucene.queries.function.FunctionScoreQuery$FunctionScoreWeight.scorer(FunctionScoreQuery.java:224)
at org.apache.lucene.search.Weight.bulkScorer(Weight.java:181)
at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:658)
at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
at org.apache.solr.search.SolrIndexSearcher.buildAndRunCollectorChain(SolrIndexSearcher.java:208)
at org.apache.solr.search.SolrIndexSearcher.getDocListNC(SolrIndexSearcher.java:1593)
at org.apache.solr.search.SolrIndexSearcher.getDocListC(SolrIndexSearcher.java:1410)
at org.apache.solr.search.SolrIndexSearcher.search(SolrIndexSearcher.java:593)
at org.apache.solr.handler.component.QueryComponent.doProcessUngroupedSearch(QueryComponent.java:1513)
at org.apache.solr.handler.component.QueryComponent.process(QueryComponent.java:403)
at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:331)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:2606)
at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:812)
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:588)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1596)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:590)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:235)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:221)
at org.eclipse.jetty.server.handler.InetAccessHandler.handle(InetAccessHandler.java:177)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:146)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
at org.eclipse.jetty.server.Server.handle(Server.java:500)
at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:273)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
at java.lang.Thread.run(Thread.java:748)
我强烈怀疑这与 CustomDoubleValuesSource 中未实现的方法有关。我阅读了DoubleValuesSource 的java 文档，但它们不是描述性的或不包含任何示例。
感谢任何能让我前进的帮助:)
解决方案
推荐阅读
- json - 可在 JSON 中解码字典 - Swift
- graphics - 如果您需要深度缓冲区或使用 WebGL 中的 DEPTH_BUFFER_BIT 标志进行 2D 和/或视差
- c# - 我收到此代码的 StackOverflowException 错误。我一直无法找出错误
- audiokit - 使用 AKDevice 设置 AKInput
- ios - @HostListener 在 iPad 或 IOS 上不起作用。我正在使用蓝牙键盘尝试控制角度应用程序
- python-3.x - re.sub 正在用空格替换我文件中的文本。不知道为什么
- python - python方法作为变量
- python - 将 32 位二进制数转换为 4 个单独的 8 位段的函数
- java - 从 WAR 文件中打包的类路径加载 PNG 文件
- scala - 处理许多特定类的通用方法