From 240e9c453591b47daca252a6affb5ae978a5f4a8 Mon Sep 17 00:00:00 2001 From: gauthiier Date: Wed, 9 Feb 2022 18:50:33 +0100 Subject: [PATCH] GPT2Config --- tokenise+train.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tokenise+train.py b/tokenise+train.py index 720bf91..cf59b64 100644 --- a/tokenise+train.py +++ b/tokenise+train.py @@ -66,6 +66,7 @@ def encode(filepath: str, blocksize: int, vocabsize: int, ouputdir: Path, verbos from aitextgen.TokenDataset import TokenDataset from aitextgen.tokenizers import train_tokenizer + from transformers import GPT2Config #NOTE: vocab_size is fixed since this is not yet in train_tokenizer #see https://github.com/minimaxir/aitextgen/blob/master/aitextgen/tokenizers.py