From 53ed345cd02d1e7364ef9277abd21279657c38e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Mayoral=20Vilches?= Date: Thu, 8 May 2025 18:26:52 +0200 Subject: [PATCH] Re-try after fixes in servers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Víctor Mayoral Vilches --- .../cybermetric/CyberMetric_evaluator.py | 2 +- ci/benchmarks/.benchmarks.yml | 29 ++++++++++++------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/benchmarks/cybermetric/CyberMetric_evaluator.py b/benchmarks/cybermetric/CyberMetric_evaluator.py index 6f674e65..7135e3cc 100644 --- a/benchmarks/cybermetric/CyberMetric_evaluator.py +++ b/benchmarks/cybermetric/CyberMetric_evaluator.py @@ -293,7 +293,7 @@ class CyberMetricEvaluator: print(f"Expected Answer: {item['correct_answer']}, LLM Answer: {item['llm_answer']}\n") if __name__ == "__main__": - litellm._turn_on_debug() + #litellm._turn_on_debug() # Create argument parser parser = argparse.ArgumentParser(description='CyberMetric Evaluator for LLMs') diff --git a/ci/benchmarks/.benchmarks.yml b/ci/benchmarks/.benchmarks.yml index 227d970c..8d42a698 100644 --- a/ci/benchmarks/.benchmarks.yml +++ b/ci/benchmarks/.benchmarks.yml @@ -28,16 +28,16 @@ - if: $CI_COMMIT_BRANCH when: on_success -benchmarks-test-seceval: - <<: *run_benchmarks - variables: - OLLAMA_API_BASE: "http://localhost:8000" - OPENROUTER_API_BASE: "https://openrouter.ai/api/v1" - OPENAI_API_KEY: "fake-api-key" - script: - - pip3 install -e . - - pip install -r benchmarks/seceval/eval/requirements.txt - - python3 benchmarks/seceval/eval/eval.py --dataset_file benchmarks/seceval/eval/datasets/questions-2.json --output_dir benchmarks/seceval/eval/outputs --backend ollama --models ollama/qwen2.5:14b +# benchmarks-test-seceval: +# <<: *run_benchmarks +# variables: +# OLLAMA_API_BASE: "http://localhost:8000" +# OPENROUTER_API_BASE: "https://openrouter.ai/api/v1" +# OPENAI_API_KEY: "fake-api-key" +# script: +# - pip3 install -e . +# - pip install -r benchmarks/seceval/eval/requirements.txt +# - python3 benchmarks/seceval/eval/eval.py --dataset_file benchmarks/seceval/eval/datasets/questions-2.json --output_dir benchmarks/seceval/eval/outputs --backend ollama --models ollama/qwen2.5:14b benchmarks-test-cybermetric: <<: *run_benchmarks @@ -47,3 +47,12 @@ benchmarks-test-cybermetric: script: - pip3 install -e . - python3 benchmarks/cybermetric/CyberMetric_evaluator.py --model_name ollama/qwen2.5:14b --file_path benchmarks/cybermetric/CyberMetric-2-v1.json + +benchmarks-test-cybermetric-openai: + <<: *run_benchmarks + variables: + OPENAI_API_KEY: $OPENAI_API_KEY + OPENAI_API_BASE: "https://api.openai.com/v1" + script: + - pip3 install -e . + - python3 benchmarks/cybermetric/CyberMetric_evaluator.py --model_name openai/gpt-4o --file_path benchmarks/cybermetric/CyberMetric-2-v1.json