mirror of https://github.com/razor-ai/soup.git
fix: address review findings — immutable rows, response guard, GPU cleanup
- Stop mutating dataset rows in-place in _validate_audio_files (use shallow copy) - Guard _generate_server response parsing against unexpected JSON shape - Add empty dataset guard in _prepare_audio_dataset - Free GPU memory after perplexity scoring in compute_perplexity_scores
This commit is contained in:
parent
3d66b41d00
commit
0b7759898c
|
|
@ -454,7 +454,10 @@ def _generate_server(
|
|||
)
|
||||
|
||||
data = response.json()
|
||||
content = data["choices"][0]["message"]["content"]
|
||||
try:
|
||||
content = data["choices"][0]["message"]["content"]
|
||||
except (KeyError, IndexError, TypeError) as exc:
|
||||
raise ValueError(f"Unexpected server response format: {exc}") from exc
|
||||
|
||||
return _parse_json_array(content)
|
||||
|
||||
|
|
|
|||
|
|
@ -198,8 +198,7 @@ def _validate_audio_files(data: list[dict], audio_dir: Path) -> list[dict]:
|
|||
if not str(resolved).startswith(str(resolved_base)):
|
||||
traversal += 1
|
||||
continue
|
||||
row["audio"] = str(resolved)
|
||||
valid.append(row)
|
||||
valid.append({**row, "audio": str(resolved)})
|
||||
|
||||
if missing > 0:
|
||||
console.print(f"[yellow]Warning: {missing} rows skipped (missing audio path)[/]")
|
||||
|
|
|
|||
|
|
@ -516,6 +516,12 @@ class SFTTrainerWrapper:
|
|||
result["sampling_rate"] = sampling_rate
|
||||
return result
|
||||
|
||||
if not dataset["train"]:
|
||||
raise ValueError(
|
||||
"Audio training dataset is empty after validation. "
|
||||
"Check audio file paths and audio_dir."
|
||||
)
|
||||
|
||||
remove_cols = ["messages", "audio"]
|
||||
train_ds = Dataset.from_list(dataset["train"]).map(
|
||||
load_and_format_audio,
|
||||
|
|
|
|||
|
|
@ -78,6 +78,11 @@ def compute_perplexity_scores(
|
|||
ppl = math.exp(min(loss_val.item(), 100)) # cap to avoid overflow
|
||||
scores.append(ppl)
|
||||
|
||||
# Cleanup GPU memory
|
||||
del model
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
return scores
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue