From 984f4e5b35ed12023a5398812588a75d9bd7a38a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Mayoral=20Vilches?= Date: Thu, 8 May 2025 17:57:48 +0200 Subject: [PATCH] Let's see how it reacts with debug MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Víctor Mayoral Vilches --- .gitignore | 5 ++++- benchmarks/cybermetric/CyberMetric_evaluator.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index f0416924..4d94f1f4 100644 --- a/.gitignore +++ b/.gitignore @@ -145,4 +145,7 @@ cython_debug/ # CAI files .cai/ -.vscode/ \ No newline at end of file +.vscode/ + +# benchmarks +benchmarks/cybermetric/outputs \ No newline at end of file diff --git a/benchmarks/cybermetric/CyberMetric_evaluator.py b/benchmarks/cybermetric/CyberMetric_evaluator.py index 7b6d0b8a..6f674e65 100644 --- a/benchmarks/cybermetric/CyberMetric_evaluator.py +++ b/benchmarks/cybermetric/CyberMetric_evaluator.py @@ -48,7 +48,7 @@ import requests # Default API bases OPENROUTER_API_BASE = "https://openrouter.ai/api/v1" -OLLAMA_LITELLM_API_BASE = os.environ["OLLAMA_API_BASE"] #"http://localhost:8000/v1" +OLLAMA_LITELLM_API_BASE = os.environ["OLLAMA_API_BASE"] # "http://localhost:8000/v1" os.environ["OPENAI_API_KEY"] = "test_key_for_ci_environment" class CyberMetricEvaluator: @@ -293,7 +293,7 @@ class CyberMetricEvaluator: print(f"Expected Answer: {item['correct_answer']}, LLM Answer: {item['llm_answer']}\n") if __name__ == "__main__": - #litellm._turn_on_debug() + litellm._turn_on_debug() # Create argument parser parser = argparse.ArgumentParser(description='CyberMetric Evaluator for LLMs')