Skip to content
Merged
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions src/lighteval/metrics/metrics_corpus.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
"""
import math

from lighteval.logging.hierarchical_logger import hlog_warn
import numpy as np
import sacrebleu
import sklearn.metrics
Expand All @@ -35,7 +36,6 @@
LogprobCorpusMetricInput,
PerplexityCorpusMetricInput,
)
from lighteval.utils.utils import as_list


# General aggregations
Expand Down Expand Up @@ -103,7 +103,11 @@ def __init__(self, metric_type: str):
def compute(self, items: list[GenerativeCorpusMetricInput]) -> float:
"""Computes the metric score over all the corpus generated items, by using the sacrebleu implementation."""
golds = [i.golds for i in items]
preds = [as_list(i.preds) for i in items]
preds = []
for i in items:
if len(i.preds) > 1:
hlog_warn("Multiple predictions present, keeping only the first prediction.")
preds.append(i.preds[0])
return float(self.metric(hypotheses=preds, references=golds).score)


Expand Down