initial

README.md

@@ -5,7 +5,7 @@ datasets:
 tags:
 - evaluate
 - metric
-description: "TODO: add a description here"
+description: "a statistic measure for evaluating any process that produces a list of possible responses to a sample of queries, ordered by probability of correctness."
 sdk: gradio
 sdk_version: 3.0.2
 app_file: app.py
@@ -14,19 +14,21 @@ pinned: false
 
 # Metric Card for Mean Reciprocal Rank
 
-***Module Card Instructions:*** *Fill out the following subsections. Feel free to take a look at existing metric cards if you'd like examples.*
+a statistic measure for evaluating any process that produces a list of possible responses to a sample of queries, ordered by probability of correctness.
 
 ## Metric Description
-*Give a brief overview of this metric, including what task(s) it is usually used for, if any.*
+The reciprocal rank of a query response is the multiplicative inverse of the rank of the first correct answer: 1 for first place, 1⁄2 for second place, 1⁄3 for third place and so on. The mean reciprocal rank is the average of the reciprocal ranks of results for a sample of queries Q 
+
+{\text{MRR}}={\frac  {1}{|Q|}}\sum _{{i=1}}^{{|Q|}}{\frac  {1}{{\text{rank}}_{i}}}.\!
 
 ## How to Use
-*Give general statement of how to use the metric*
+Provide a list of gold ranks, where each item is rank of gold item of which the first rank starts with zero.
+
 
-*Provide simplest possible example for using the metric*
 
 ### Inputs
 *List all input arguments in the format below*
-- **input_field** *(type): Definition of input, with explanation if necessary. State any default value(s).*
+- **input_field** *(List[int]): a list of integer where each integer is the rank of gold item 
 
 ### Output Values
 

mean_reciprocal_rank.py

@@ -19,47 +19,39 @@ import datasets
 
 # TODO: Add BibTeX citation
 _CITATION = """\
-@InProceedings{huggingface:module,
-title = {A great new module},
-authors={huggingface, Inc.},
-year={2020}
+@InProceedings{posicube:module,
+title = {Mean reciprocal mean},
+authors={Pocicube, Inc.},
+year={2022}
 }
 """
 
 # TODO: Add description of the module here
 _DESCRIPTION = """\
-This new module is designed to solve this great ML task and is crafted with a lot of care.
+This module is designed to evaluate a system ranks the list of item. 
+mean reciprocal rank is a statistic measure for evaluating any process that produces a list of possible responses to a sample of queries, ordered by probability of correctness
 """
 
 
 # TODO: Add description of the arguments of the module here
 _KWARGS_DESCRIPTION = """
-Calculates how good are predictions given some references, using certain scores
+Calculates how good are ranks, using certain scores
 Args:
-    predictions: list of predictions to score. Each predictions
-        should be a string with tokens separated by spaces.
-    references: list of reference for each prediction. Each
-        reference should be a string with tokens separated by spaces.
+    predictions: list of predicted ranks of gold item, the first rank starts with 0
 Returns:
-    accuracy: description of the first score,
-    another_score: description of the second score,
+    mean reciprocal rank: mean of inverse of rank of gold item
 Examples:
-    Examples should be written in doctest format, and should illustrate how
-    to use the function.
-
-    >>> my_new_module = evaluate.load("my_new_module")
-    >>> results = my_new_module.compute(references=[0, 1], predictions=[0, 1])
+    
+    >>> mrr = evaluate.load("poscicube/mean_reciprocal_rank")
+    >>> results = mrr.compute(predictions=[0, 4])
     >>> print(results)
-    {'accuracy': 1.0}
+    {'mrr': 0.6}
 """
 
-# TODO: Define external resources urls if needed
-BAD_WORDS_URL = "http://url/to/external/resource/bad_words.txt"
-
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
 class MeanReciprocalRank(evaluate.Metric):
-    """TODO: Short description of my evaluation module."""
+    """a statistic measure for evaluating any process that produces a list of possible responses to a sample of queries, ordered by probability of correctness."""
 
     def _info(self):
         # TODO: Specifies the evaluate.EvaluationModuleInfo object
@@ -72,24 +64,26 @@ class MeanReciprocalRank(evaluate.Metric):
             # This defines the format of each prediction and reference
             features=datasets.Features({
                 'predictions': datasets.Value('int64'),
-                'references': datasets.Value('int64'),
             }),
             # Homepage of the module for documentation
-            homepage="http://module.homepage",
+            homepage="https://huggingface.co/spaces/posicube/mean_reciprocal_rank",
             # Additional links to the codebase or references
-            codebase_urls=["http://github.com/path/to/codebase/of/new_module"],
-            reference_urls=["http://path.to.reference.url/new_module"]
+            codebase_urls=["https://huggingface.co/spaces/posicube/mean_reciprocal_rank"],
+            reference_urls=["https://en.wikipedia.org/wiki/Mean_reciprocal_rank"]
         )
 
     def _download_and_prepare(self, dl_manager):
         """Optional: download external resources useful to compute the scores"""
-        # TODO: Download external resources if needed
         pass
 
-    def _compute(self, predictions, references):
+    def _compute(self, predictions):
         """Returns the scores"""
         # TODO: Compute the different scores of the module
-        accuracy = sum(i == j for i, j in zip(predictions, references)) / len(predictions)
+        q = len(predictions)
+        sum_rr = 0.0
+        for p in predictions:
+            sum_rr += 1/(p+1)
+        mrr = sum_rr / q
         return {
-            "accuracy": accuracy,
+            "mrr": mrr
         }

requirements.txt

@@ -1 +1,2 @@
-git+https://github.com/huggingface/evaluate@main
+git+https://github.com/huggingface/evaluate@main
+datasets

tests.py

@@ -1,17 +1,14 @@
 test_cases = [
     {
-        "predictions": [0, 0],
-        "references": [1, 1],
-        "result": {"metric_score": 0}
+        "predictions": [0, 0, 2],
+        "result": {"mrr": 0.7777777777777778}
     },
     {
-        "predictions": [1, 1],
-        "references": [1, 1],
-        "result": {"metric_score": 1}
+        "predictions": [0, 0],
+        "result": {"mrr": 1}
     },
     {
-        "predictions": [1, 0],
-        "references": [1, 1],
-        "result": {"metric_score": 0.5}
+        "predictions": [0, 4],
+        "result": {"mrr": 0.6}
     }
 ]