diff --git a/opensearch-search-quality-evaluation-plugin/scripts/create-query-set-using-pptss-sampling.sh b/opensearch-search-quality-evaluation-plugin/scripts/create-query-set-using-pptss-sampling.sh index 283afef..ef241bf 100755 --- a/opensearch-search-quality-evaluation-plugin/scripts/create-query-set-using-pptss-sampling.sh +++ b/opensearch-search-quality-evaluation-plugin/scripts/create-query-set-using-pptss-sampling.sh @@ -1,7 +1,7 @@ #!/bin/bash -e #QUERY_SET=`curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/queryset?name=test&description=fake&sampling=pptss" | jq .query_set | tr -d '"'` -curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/queryset?name=test&description=fake&sampling=pptss&query_set_size=5000" +curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/queryset?name=test&description=fake&sampling=pptss&query_set_size=100" #echo ${QUERY_SET} diff --git a/opensearch-search-quality-evaluation-plugin/scripts/get-query-set-run-results.sh b/opensearch-search-quality-evaluation-plugin/scripts/get-query-set-run-results.sh new file mode 100755 index 0000000..a739828 --- /dev/null +++ b/opensearch-search-quality-evaluation-plugin/scripts/get-query-set-run-results.sh @@ -0,0 +1,3 @@ +#!/bin/bash -e + +curl -s "http://localhost:9200/search_quality_eval_query_sets_run_results/_search" | jq diff --git a/opensearch-search-quality-evaluation-plugin/scripts/get-query-sets.sh b/opensearch-search-quality-evaluation-plugin/scripts/get-query-sets.sh new file mode 100755 index 0000000..0bcb3ff --- /dev/null +++ b/opensearch-search-quality-evaluation-plugin/scripts/get-query-sets.sh @@ -0,0 +1,3 @@ +#!/bin/bash -e + +curl -s "http://localhost:9200/search_quality_eval_query_sets/_search" | jq diff --git a/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh b/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh new file mode 100755 index 0000000..f6c4f99 --- /dev/null +++ b/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh @@ -0,0 +1,5 @@ +#!/bin/bash -e + +QUERY_SET_ID="${1}" + +curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/run?id=${QUERY_SET_ID}" | jq diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationPlugin.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationPlugin.java index 199eb26..599147e 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationPlugin.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationPlugin.java @@ -76,6 +76,11 @@ public class SearchQualityEvaluationPlugin extends Plugin implements ActionPlugi */ public static final String QUERY_SETS_INDEX_NAME = "search_quality_eval_query_sets"; + /** + * The name of the index that stores the query set run results. + */ + public static final String QUERY_SETS_RUN_RESULTS = "search_quality_eval_query_sets_run_results"; + @Override public Collection createComponents( final Client client, diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java index cbb94c9..3c6fb94 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java @@ -98,7 +98,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli if (AllQueriesQuerySampler.NAME.equalsIgnoreCase(sampling)) { // If we are not sampling queries, the query sets should just be directly - // indexed into OpenSearch using the `ubu_queries` index directly. + // indexed into OpenSearch using the `ubi_queries` index directly. try { @@ -153,15 +153,14 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli final OpenSearchQuerySetRunner openSearchQuerySetRunner = new OpenSearchQuerySetRunner(client); final QuerySetRunResult querySetRunResult = openSearchQuerySetRunner.run(querySetId); - - // TODO: Index the querySetRunResult. + openSearchQuerySetRunner.save(querySetRunResult); } catch (Exception ex) { - LOGGER.error("Unable to retrieve query set with ID {}", querySetId); + LOGGER.error("Unable to run query set with ID {}: ", querySetId, ex); return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, ex.getMessage())); } - return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "{\"message\": \"Query set " + querySetId + " run initiated.\"}")); + return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "{\"message\": \"Run initiated for query set " + querySetId + "\"}")); // Handle the on-demand creation of implicit judgments. } else if(IMPLICIT_JUDGMENTS_URL.equalsIgnoreCase(request.path())) { diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java index 6729ea5..b2a7861 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java @@ -8,9 +8,14 @@ */ package org.opensearch.eval.runners; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.index.IndexResponse; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; +import org.opensearch.core.action.ActionListener; import org.opensearch.eval.SearchQualityEvaluationPlugin; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.SearchHit; @@ -18,18 +23,29 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.List; +import java.util.Map; -public class OpenSearchQuerySetRunner extends QuerySetRunner { +/** + * A {@link QuerySetRunner} for Amazon OpenSearch. + */ +public class OpenSearchQuerySetRunner implements QuerySetRunner { + + private static final Logger LOGGER = LogManager.getLogger(OpenSearchQuerySetRunner.class); final Client client; + /** + * Creates a new query set runner + * @param client An OpenSearch {@link Client}. + */ public OpenSearchQuerySetRunner(final Client client) { this.client = client; } @Override - public QuerySetRunResult run(String querySetId) { + public QuerySetRunResult run(final String querySetId) { // Get the query set. final SearchSourceBuilder getQuerySetSearchSourceBuilder = new SearchSourceBuilder(); @@ -43,35 +59,72 @@ public QuerySetRunResult run(String querySetId) { final SearchResponse searchResponse = client.search(getQuerySetSearchRequest).get(); // The queries from the query set that will be run. - final Collection queries = (Collection) searchResponse.getHits().getAt(0).getSourceAsMap().get("queries"); + final Collection> queries = (Collection>) searchResponse.getHits().getAt(0).getSourceAsMap().get("queries"); // The results of each query. - final Collection queryResults = new ArrayList<>(); + final List queryResults = new ArrayList<>(); - // TODO: Initiate the running of the query set. - for(final String query : queries) { + for(Map queryMap : queries) { - // TODO: What should this query be? - final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(QueryBuilders.matchQuery("title", query)); - // TODO: Just fetch the id ("asin") field and not all the unnecessary fields. + // Loop over each query in the map and run each one. + for (final String query : queryMap.keySet()) { - // TODO: Allow for setting this index name. - final SearchRequest searchRequest = new SearchRequest("ecommerce"); - getQuerySetSearchRequest.source(getQuerySetSearchSourceBuilder); + final String index = "ecommerce"; + final String idField = "asin"; - final SearchResponse sr = client.search(searchRequest).get(); + final String q = "{\n" + + " \"query\": {\n" + + " \"match\": {\n" + + " \"description\": {\n" + + " \"query\": \" + query + \"\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; - final List orderedDocumentIds = new ArrayList<>(); + // TODO: What should this query be? + final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + //searchSourceBuilder.query(QueryBuilders.wrapperQuery(q)); + searchSourceBuilder.query(QueryBuilders.matchQuery("description", query)); + searchSourceBuilder.from(0); + searchSourceBuilder.size(10); - for(final SearchHit hit : sr.getHits().getHits()) { + String[] includeFields = new String[] {idField}; + String[] excludeFields = new String[] {}; + searchSourceBuilder.fetchSource(includeFields, excludeFields); - // TODO: This field needs to be customizable. - orderedDocumentIds.add(hit.getFields().get("asin").toString()); + // TODO: Allow for setting this index name. + final SearchRequest searchRequest = new SearchRequest(index); + getQuerySetSearchRequest.source(searchSourceBuilder); - } + client.search(searchRequest, new ActionListener<>() { + + @Override + public void onResponse(final SearchResponse searchResponse) { + + final List orderedDocumentIds = new ArrayList<>(); + + for (final SearchHit hit : searchResponse.getHits().getHits()) { + + final Map sourceAsMap = hit.getSourceAsMap(); + final String documentId = sourceAsMap.get(idField).toString(); - queryResults.add(new QueryResult(orderedDocumentIds)); + orderedDocumentIds.add(documentId); + + } + + queryResults.add(new QueryResult(query, orderedDocumentIds)); + + } + + @Override + public void onFailure(Exception ex) { + LOGGER.error("Unable to search for query: {}", query, ex); + } + }); + + + } } @@ -86,4 +139,31 @@ public QuerySetRunResult run(String querySetId) { } + @Override + public void save(final QuerySetRunResult result) throws Exception { + + // Index the results into OpenSearch. + + final Map results = new HashMap<>(); + + results.put("run_id", result.getRunId()); + results.put("search_metrics", result.getSearchMetrics().getSearchMetricsAsMap()); + results.put("query_results", result.getQueryResultsAsMap()); + + final IndexRequest indexRequest = new IndexRequest(SearchQualityEvaluationPlugin.QUERY_SETS_RUN_RESULTS); + indexRequest.source(results); + + client.index(indexRequest, new ActionListener<>() { + @Override + public void onResponse(IndexResponse indexResponse) { + LOGGER.debug("Query set results indexed."); + } + + @Override + public void onFailure(Exception ex) { + throw new RuntimeException(ex); + } + }); + } + } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QueryResult.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QueryResult.java index e26679f..477e439 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QueryResult.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QueryResult.java @@ -10,16 +10,50 @@ import java.util.List; +/** + * Contains the search results for a query. + */ public class QueryResult { + private final String query; private final List orderedDocumentIds; - public QueryResult(final List orderedDocumentIds) { + // TODO: Calculate these metrics. + private final SearchMetrics searchMetrics = new SearchMetrics(); + + /** + * Creates the search results. + * @param query The query used to generate this result. + * @param orderedDocumentIds A list of ordered document IDs in the same order as they appeared + * in the query. + */ + public QueryResult(final String query, final List orderedDocumentIds) { + this.query = query; this.orderedDocumentIds = orderedDocumentIds; } + /** + * Gets the query used to generate this result. + * @return The query used to generate this result. + */ + public String getQuery() { + return query; + } + + /** + * Gets the list of ordered document IDs. + * @return A list of ordered documented IDs. + */ public List getOrderedDocumentIds() { return orderedDocumentIds; } + /** + * Gets the search metrics for this query. + * @return The {@link SearchMetrics} for this query. + */ + public SearchMetrics getSearchMetrics() { + return searchMetrics; + } + } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunResult.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunResult.java index 2491e3d..a27730f 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunResult.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunResult.java @@ -8,24 +8,77 @@ */ package org.opensearch.eval.runners; +import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +/** + * The results of a query set run. + */ public class QuerySetRunResult { - private final Collection queryResults; + private final String runId; + private final List queryResults; private final SearchMetrics searchMetrics; - public QuerySetRunResult(final Collection queryResults, final SearchMetrics searchMetrics) { + /** + * Creates a new query set run result. A random UUID is generated as the run ID. + * @param queryResults A collection of {@link QueryResult} that contains the queries and search results. + * @param searchMetrics The {@link SearchMetrics metrics} calculated from the search results. + */ + public QuerySetRunResult(final List queryResults, final SearchMetrics searchMetrics) { + this.runId = UUID.randomUUID().toString(); this.queryResults = queryResults; this.searchMetrics = searchMetrics; } + /** + * Get the run's ID. + * @return The run's ID. + */ + public String getRunId() { + return runId; + } + + /** + * Gets the {@link SearchMetrics metrics} calculated from the run. + * @return The {@link SearchMetrics metrics} calculated from the run. + */ public SearchMetrics getSearchMetrics() { return searchMetrics; } + /** + * Gets the results of the query set run. + * @return A collection of {@link QueryResult results}. + */ public Collection getQueryResults() { return queryResults; } + public Collection> getQueryResultsAsMap() { + + final Collection> qs = new ArrayList<>(); + + for(final QueryResult queryResult : queryResults) { + + final Map q = new HashMap<>(); + + q.put("query", queryResult.getQuery()); + q.put("document_ids", queryResult.getOrderedDocumentIds()); + q.put("search_metrics", queryResult.getSearchMetrics().getSearchMetricsAsMap()); + + qs.add(q); + + + } + + return qs; + + } + + } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunner.java index 412be51..ef9ec09 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunner.java @@ -8,8 +8,23 @@ */ package org.opensearch.eval.runners; -public abstract class QuerySetRunner { +/** + * Interface for query set runners. Classes that implement this interface + * should be specific to a search engine. See the {@link OpenSearchQuerySetRunner} for an example. + */ +public interface QuerySetRunner { + + /** + * Runs the query set. + * @param querySetId The ID of the query set to run. + * @return The query set {@link QuerySetRunResult results} and calculated metrics. + */ + QuerySetRunResult run(String querySetId); - abstract QuerySetRunResult run(String querySetId); + /** + * Saves the query set results to a persistent store, which may be the search engine itself. + * @param result The {@link QuerySetRunResult results}. + */ + void save(QuerySetRunResult result) throws Exception; } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/SearchMetrics.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/SearchMetrics.java index 759b44b..61e0fe8 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/SearchMetrics.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/SearchMetrics.java @@ -8,6 +8,55 @@ */ package org.opensearch.eval.runners; +import java.util.HashMap; +import java.util.Map; + +/** + * Provides the ability to calculate search metrics and stores them. + */ public class SearchMetrics { + private double dcg_at_10 = 0.0; + private double ndcg_at_10 = 0.0; + private double prec_at_10 = 0.0; + + /** + * Gets the metrics as a map for ease of indexing. + * @return A map of the search metrics. + */ + public Map getSearchMetricsAsMap() { + + final Map metrics = new HashMap<>(); + metrics.put("dcg_at_10", dcg_at_10); + metrics.put("ndcg_at_10", ndcg_at_10); + metrics.put("prec_at_10", prec_at_10); + + return metrics; + + } + + public double getDcg_at_10() { + return dcg_at_10; + } + + public void setDcg_at_10(double dcg_at_10) { + this.dcg_at_10 = dcg_at_10; + } + + public double getNdcg_at_10() { + return ndcg_at_10; + } + + public void setNdcg_at_10(double ndcg_at_10) { + this.ndcg_at_10 = ndcg_at_10; + } + + public double getPrec_at_10() { + return prec_at_10; + } + + public void setPrec_at_10(double prec_at_10) { + this.prec_at_10 = prec_at_10; + } + } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/samplers/AbstractQuerySampler.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/samplers/AbstractQuerySampler.java index 4fcfa27..6fa051c 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/samplers/AbstractQuerySampler.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/samplers/AbstractQuerySampler.java @@ -51,13 +51,17 @@ protected String indexQuerySet(final NodeClient client, final String name, final LOGGER.info("Indexing {} queries for query set {}", queries.size(), name); - final Collection querySetQueries = new ArrayList<>(); + final Collection> querySetQueries = new ArrayList<>(); // Convert the queries map to an object. for(final String query : queries.keySet()) { + // Map of the query itself to the frequency of the query. + final Map querySetQuery = new HashMap<>(); + querySetQuery.put(query, queries.get(query)); + final long frequency = queries.get(query); - querySetQueries.add(new QuerySetQuery(query, frequency)); + querySetQueries.add(querySetQuery); }