Fix bug in world tokenizer (#93)

RWKV · Jun 11, 2023 · b88ae59 · b88ae59
1 parent 82c4ac7
commit b88ae59
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/rwkv/rwkv_tokenizer.py b/rwkv/rwkv_tokenizer.py
@@ -106,7 +106,8 @@ def printTokens(self, tokens):
 def get_tokenizer(tokenizer="20B"):
     if tokenizer == "world":
         print('Loading world tokenizer')
-        tokenizer = TRIE_TOKENIZER('rwkv_vocab_v20230424.txt')
+        tokenizer_path = pathlib.Path(os.path.abspath(__file__)).parent / 'rwkv_vocab_v20230424.txt'
+        tokenizer = TRIE_TOKENIZER(tokenizer_path)
         tokenizer_encode = lambda prompt: tokenizer.encode(prompt)
     elif tokenizer == "20B":
         print('Loading 20B tokenizer')