diff --git a/tokenise+train.py b/tokenise+train.py index 3dc3cc5..7d8f950 100644 --- a/tokenise+train.py +++ b/tokenise+train.py @@ -19,7 +19,7 @@ def train(ouputdir: Path, blocksize: int, vocabsize: int, num_steps: int, gpu: b from aitextgen import aitextgen exts = ['.json', '.gz'] - files = [x for x in ouputdir.glob('*') if x.suffix in exts] + files = [x for x in ouputdir.glob('*') if x.suffix in exts and x != "config.json"] if len(files) == 2: if files[0].suffix == '.json': tok = str(files[0])