Fix AttributeError for GloVe vocab in encode function

Co-authored-by: leestott <2511341+leestott@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot] 2025-10-03 13:10:36 +00:00
parent cfb48e13ff
commit badc35cac5
4 changed files with 20 additions and 4 deletions

View File

@ -30,7 +30,11 @@ def encode(x,voc=None,unk=0,tokenizer=tokenizer):
if v in stoi_hash.keys():
stoi = stoi_hash[v]
else:
stoi = v.get_stoi()
# Handle both vocab.vocab objects (with get_stoi() method) and GloVe objects (with stoi attribute)
if hasattr(v, 'get_stoi'):
stoi = v.get_stoi()
else:
stoi = v.stoi
stoi_hash[v]=stoi
return [stoi.get(s,unk) for s in tokenizer(x)]

View File

@ -30,7 +30,11 @@ def encode(x,voc=None,unk=0,tokenizer=tokenizer):
if v in stoi_hash.keys():
stoi = stoi_hash[v]
else:
stoi = v.get_stoi()
# Handle both vocab.vocab objects (with get_stoi() method) and GloVe objects (with stoi attribute)
if hasattr(v, 'get_stoi'):
stoi = v.get_stoi()
else:
stoi = v.stoi
stoi_hash[v]=stoi
return [stoi.get(s,unk) for s in tokenizer(x)]

View File

@ -30,7 +30,11 @@ def encode(x,voc=None,unk=0,tokenizer=tokenizer):
if v in stoi_hash.keys():
stoi = stoi_hash[v]
else:
stoi = v.get_stoi()
# Handle both vocab.vocab objects (with get_stoi() method) and GloVe objects (with stoi attribute)
if hasattr(v, 'get_stoi'):
stoi = v.get_stoi()
else:
stoi = v.stoi
stoi_hash[v]=stoi
return [stoi.get(s,unk) for s in tokenizer(x)]

View File

@ -30,7 +30,11 @@ def encode(x,voc=None,unk=0,tokenizer=tokenizer):
if v in stoi_hash.keys():
stoi = stoi_hash[v]
else:
stoi = v.get_stoi()
# Handle both vocab.vocab objects (with get_stoi() method) and GloVe objects (with stoi attribute)
if hasattr(v, 'get_stoi'):
stoi = v.get_stoi()
else:
stoi = v.stoi
stoi_hash[v]=stoi
return [stoi.get(s,unk) for s in tokenizer(x)]