Passing judgments into SearchMetrics for calculating metrics.

o19s · Dec 3, 2024 · 0e240b8 · 0e240b8
1 parent 1d5f773
commit 0e240b8
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 7 deletions.
diff --git a/...evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java b/...evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java
@@ -17,6 +17,7 @@
 import org.opensearch.client.Client;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.eval.SearchQualityEvaluationPlugin;
+import org.opensearch.eval.judgments.model.Judgment;
 import org.opensearch.index.query.QueryBuilders;
 import org.opensearch.search.SearchHit;
 import org.opensearch.search.builder.SearchSourceBuilder;
@@ -49,6 +50,9 @@ public OpenSearchQuerySetRunner(final Client client) {
     @Override
     public QuerySetRunResult run(final String querySetId, final String judgmentsId, final String index, final String idField, final String query, final int k) {
 
+        // TODO: Get the judgments we will use for metric calculation.
+        final List<Judgment> judgments = new ArrayList<>();
+
         // Get the query set.
         final SearchSourceBuilder getQuerySetSearchSourceBuilder = new SearchSourceBuilder();
         getQuerySetSearchSourceBuilder.query(QueryBuilders.matchQuery("_id", querySetId));
@@ -105,7 +109,7 @@ public void onResponse(final SearchResponse searchResponse) {
 
                             }
 
-                            queryResults.add(new QueryResult(query, orderedDocumentIds, k));
+                            queryResults.add(new QueryResult(query, orderedDocumentIds, judgments, k));
 
                         }
 
@@ -120,7 +124,7 @@ public void onFailure(Exception ex) {
             }
 
             // TODO: Calculate the search metrics given the results and the judgments.
-            final SearchMetrics searchMetrics = new SearchMetrics(queryResults, k);
+            final SearchMetrics searchMetrics = new SearchMetrics(queryResults, judgments, k);
 
             return new QuerySetRunResult(queryResults, searchMetrics);
 

diff --git a/...arch-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QueryResult.java b/...arch-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QueryResult.java
@@ -8,6 +8,8 @@
  */
 package org.opensearch.eval.runners;
 
+import org.opensearch.eval.judgments.model.Judgment;
+
 import java.util.List;
 
 /**
@@ -24,12 +26,13 @@ public class QueryResult {
      * @param query The query used to generate this result.
      * @param orderedDocumentIds A list of ordered document IDs in the same order as they appeared
      *                           in the query.
+     * @param judgments A list of {@link Judgment judgments} used for metric calculation.
      * @param k The k used for metrics calculation, i.e. DCG@k.
      */
-    public QueryResult(final String query, final List<String> orderedDocumentIds, final int k) {
+    public QueryResult(final String query, final List<String> orderedDocumentIds, final List<Judgment> judgments, final int k) {
         this.query = query;
         this.orderedDocumentIds = orderedDocumentIds;
-        this.searchMetrics = new SearchMetrics(query, orderedDocumentIds, k);
+        this.searchMetrics = new SearchMetrics(query, orderedDocumentIds, judgments, k);
     }
 
     /**

diff --git a/...ch-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/SearchMetrics.java b/...ch-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/SearchMetrics.java
@@ -8,6 +8,8 @@
  */
 package org.opensearch.eval.runners;
 
+import org.opensearch.eval.judgments.model.Judgment;
+
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -26,20 +28,23 @@ public class SearchMetrics {
     /**
      * Create the search metrics for an entire query set.
      * @param queryResults A list of {@link QueryResult}.
+     * @param judgments A list of {@link Judgment judgments} used for metric calculation.
      * @param k The k used for metrics calculation, i.e. DCG@k.
      */
-    public SearchMetrics(final List<QueryResult> queryResults, final int k) {
+    public SearchMetrics(final List<QueryResult> queryResults, final List<Judgment> judgments, final int k) {
         this.k = k;
 
         // TODO: Calculate the metrics for the whole query set.
     }
 
     /**
      * Create the search metrics for a single query.
-
+     * @param query The user query.
+     * @param orderedDocumentIds The documents returned for the user query in order.
+     * @param judgments A list of {@link Judgment judgments} used for metric calculation.
      * @param k The k used for metrics calculation, i.e. DCG@k.
      */
-    public SearchMetrics(final String query, final List<String> orderedDocumentIds, final int k) {
+    public SearchMetrics(final String query, final List<String> orderedDocumentIds, final List<Judgment> judgments, final int k) {
         this.k = k;
 
         // TODO: Calculate the metrics for the single query.