diff --git a/soup_cli/commands/generate.py b/soup_cli/commands/generate.py index 4342e79..8b8130a 100644 --- a/soup_cli/commands/generate.py +++ b/soup_cli/commands/generate.py @@ -853,44 +853,14 @@ def _generate_server( def _parse_json_array(content: str) -> list[dict]: - """Parse a JSON array from LLM output, handling markdown code blocks.""" - content = content.strip() + """Parse a JSON array from LLM output, handling markdown code blocks. - # Strip markdown code fences - if content.startswith("```"): - lines = content.split("\n") - # Remove first line (```json or ```) - lines = lines[1:] - # Remove last line if it's ```) - if lines and lines[-1].strip() == "```": - lines = lines[:-1] - content = "\n".join(lines).strip() + Delegates to soup_cli.data.providers._utils.parse_json_array to avoid + circular imports (providers import this, this imports providers). + """ + from soup_cli.data.providers._utils import parse_json_array - # Try to find JSON array in content - start = content.find("[") - end = content.rfind("]") - if start != -1 and end != -1 and end > start: - content = content[start:end + 1] - - try: - result = json.loads(content) - if isinstance(result, list): - return [item for item in result if isinstance(item, dict)] - except json.JSONDecodeError: - pass - - # Try line-by-line JSON objects - results = [] - for line in content.split("\n"): - line = line.strip() - if line.startswith("{"): - try: - obj = json.loads(line) - if isinstance(obj, dict): - results.append(obj) - except json.JSONDecodeError: - continue - return results + return parse_json_array(content) def _validate_example(example: dict, fmt: str) -> bool: diff --git a/soup_cli/data/providers/_utils.py b/soup_cli/data/providers/_utils.py new file mode 100644 index 0000000..bc467c6 --- /dev/null +++ b/soup_cli/data/providers/_utils.py @@ -0,0 +1,49 @@ +"""Shared utilities for data generation providers.""" + +import json + + +def parse_json_array(content: str) -> list[dict]: + """Parse a JSON array from LLM output, handling markdown code blocks. + + Handles: + - Clean JSON arrays + - Markdown code fences (```json ... ```) + - JSON arrays with surrounding text + - Line-by-line JSON objects (NDJSON fallback) + """ + content = content.strip() + + # Strip markdown code fences + if content.startswith("```"): + lines = content.split("\n") + lines = lines[1:] + if lines and lines[-1].strip() == "```": + lines = lines[:-1] + content = "\n".join(lines).strip() + + # Try to find JSON array in content + start = content.find("[") + end = content.rfind("]") + if start != -1 and end != -1 and end > start: + content = content[start:end + 1] + + try: + result = json.loads(content) + if isinstance(result, list): + return [item for item in result if isinstance(item, dict)] + except json.JSONDecodeError: + pass + + # Try line-by-line JSON objects + results = [] + for line in content.split("\n"): + line = line.strip() + if line.startswith("{"): + try: + obj = json.loads(line) + if isinstance(obj, dict): + results.append(obj) + except json.JSONDecodeError: + continue + return results diff --git a/soup_cli/data/providers/anthropic.py b/soup_cli/data/providers/anthropic.py index 82f4cec..4763aa9 100644 --- a/soup_cli/data/providers/anthropic.py +++ b/soup_cli/data/providers/anthropic.py @@ -87,6 +87,6 @@ def generate_anthropic( except (KeyError, IndexError, TypeError) as exc: raise ValueError(f"Unexpected Anthropic response format: {exc}") from exc - from soup_cli.commands.generate import _parse_json_array + from soup_cli.data.providers._utils import parse_json_array - return _parse_json_array(content) + return parse_json_array(content) diff --git a/soup_cli/data/providers/ollama.py b/soup_cli/data/providers/ollama.py index d28685a..ee21e83 100644 --- a/soup_cli/data/providers/ollama.py +++ b/soup_cli/data/providers/ollama.py @@ -30,11 +30,11 @@ def detect_ollama(base_url: str = DEFAULT_OLLAMA_BASE) -> Optional[str]: ver_response = httpx.get(f"{base_url}/api/version", timeout=5.0) if ver_response.status_code == 200: return ver_response.json().get("version", "unknown") - except Exception: - pass + except (httpx.HTTPError, KeyError, ValueError): + logger.debug("Ollama version check failed", exc_info=True) return "unknown" - except Exception: - pass + except (httpx.HTTPError, OSError): + logger.debug("Ollama not reachable at %s", base_url, exc_info=True) return None @@ -118,6 +118,6 @@ def generate_ollama( except (KeyError, IndexError, TypeError) as exc: raise ValueError(f"Unexpected Ollama response format: {exc}") from exc - from soup_cli.commands.generate import _parse_json_array + from soup_cli.data.providers._utils import parse_json_array - return _parse_json_array(content) + return parse_json_array(content) diff --git a/soup_cli/data/providers/vllm.py b/soup_cli/data/providers/vllm.py index 09e0d8d..5f982e9 100644 --- a/soup_cli/data/providers/vllm.py +++ b/soup_cli/data/providers/vllm.py @@ -102,6 +102,6 @@ def generate_vllm( except (KeyError, IndexError, TypeError) as exc: raise ValueError(f"Unexpected vLLM response format: {exc}") from exc - from soup_cli.commands.generate import _parse_json_array + from soup_cli.data.providers._utils import parse_json_array - return _parse_json_array(content) + return parse_json_array(content) diff --git a/tests/test_synth_data_pro.py b/tests/test_synth_data_pro.py index 989fa7f..46444a7 100644 --- a/tests/test_synth_data_pro.py +++ b/tests/test_synth_data_pro.py @@ -139,7 +139,7 @@ class TestOllamaProvider: """detect_ollama should return None when Ollama is not running.""" from soup_cli.data.providers.ollama import detect_ollama - with mock_patch("httpx.get", side_effect=Exception("connection refused")): + with mock_patch("httpx.get", side_effect=OSError("connection refused")): version = detect_ollama() assert version is None @@ -151,10 +151,7 @@ class TestOllamaProvider: mock_tags.status_code = 200 mock_tags.json.return_value = {"models": []} - mock_ver = MagicMock() - mock_ver.status_code = 500 - - with mock_patch("httpx.get", side_effect=[mock_tags, Exception("fail")]): + with mock_patch("httpx.get", side_effect=[mock_tags, ValueError("fail")]): version = detect_ollama() assert version == "unknown"