End the hint prompt in a full stop: an unterminated list made Whisper drop punctuation and loop
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
1c5952f212
commit
8236ea3504
4 files changed
+39
-17
No files matched your search
@@ -65,9 +65,15 @@ def normalize_hint(raw):
|
||||
return ", ".join(part for part in parts if part)
|
||||
|
||||
|
||||
def _prompt_sentence(text):
|
||||
return " " + text if re.search(r"[.?!]$", text) else " " + text + "."
|
||||
|
||||
|
||||
def fit_prompt_ids(hint, encode, budget=PROMPT_TOKEN_BUDGET):
|
||||
"""Prompt token ids for `hint`, cut to `budget` tokens at a whole word.
|
||||
|
||||
The prompt is a space-led sentence ending in a full stop (unless it ends
|
||||
in . ? ! already) - an unterminated list drops punctuation and loops.
|
||||
The start of the list is kept and the end dropped; a dangling ","/";" is
|
||||
removed; a first word longer than the budget is cut at the budget.
|
||||
`encode` must add no special tokens.
|
||||
@@ -75,13 +81,13 @@ def fit_prompt_ids(hint, encode, budget=PROMPT_TOKEN_BUDGET):
|
||||
text = normalize_hint(hint)
|
||||
if not text:
|
||||
return []
|
||||
full = encode(" " + text)
|
||||
full = encode(_prompt_sentence(text))
|
||||
if len(full) <= budget:
|
||||
return full
|
||||
words = text.split(" ")
|
||||
|
||||
def encode_prefix(count):
|
||||
return encode(" " + re.sub(r"[,;]+$", "", " ".join(words[:count])))
|
||||
return encode(_prompt_sentence(re.sub(r"[,;]+$", "", " ".join(words[:count]))))
|
||||
|
||||
low, high = 0, len(words) - 1
|
||||
while low < high:
|
||||
|
||||
Reference in new issue
Block a user