gauthiier/NATURESPEAK-ML-UTTER
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

GPT2Config

Browse Source
This commit is contained in:
gauthiier 2022-02-09 18:52:17 +01:00
parent 240e9c4535
commit aff1c17d2e
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
tokenise+train.py
View File

@ -15,7 +15,8 @@ def suffix(bs: int, ns: int, vs: int) -> str:
def train(ouputdir: Path, blocksize: int, vocabsize: int, num_steps: int, gpu: bool = False) -> str: def train(ouputdir: Path, blocksize: int, vocabsize: int, num_steps: int, gpu: bool = False) -> str:
from aitextgen.TokenDataset import TokenDataset from aitextgen.TokenDataset import TokenDataset
from aitextgen.utils import build_gpt2_config from transformers import GPT2Config
# from aitextgen.utils import build_gpt2_config
from aitextgen import aitextgen from aitextgen import aitextgen
exts = ['.json', '.gz'] exts = ['.json', '.gz']
@ -65,8 +66,7 @@ def encode(filepath: str, blocksize: int, vocabsize: int, ouputdir: Path, verbos
return "text input is not valid" return "text input is not valid"
from aitextgen.TokenDataset import TokenDataset from aitextgen.TokenDataset import TokenDataset
from aitextgen.tokenizers import train_tokenizer from aitextgen.tokenizers import train_tokenizer
from transformers import GPT2Config
#NOTE: vocab_size is fixed since this is not yet in train_tokenizer #NOTE: vocab_size is fixed since this is not yet in train_tokenizer
#see https://github.com/minimaxir/aitextgen/blob/master/aitextgen/tokenizers.py #see https://github.com/minimaxir/aitextgen/blob/master/aitextgen/tokenizers.py