gauthiier/NATURESPEAK-ML-UTTER
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

GPT2Config

Browse Source
This commit is contained in:
gauthiier 2022-02-09 18:50:33 +01:00
parent c13bfa037a
commit 240e9c4535
1 changed files with 1 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tokenise+train.py
View File

@ -66,6 +66,7 @@ def encode(filepath: str, blocksize: int, vocabsize: int, ouputdir: Path, verbos
from aitextgen.TokenDataset import TokenDataset
from aitextgen.tokenizers import train_tokenizer
from transformers import GPT2Config
#NOTE: vocab_size is fixed since this is not yet in train_tokenizer
#see https://github.com/minimaxir/aitextgen/blob/master/aitextgen/tokenizers.py