diff --git a/tokenise+train.py b/tokenise+train.py index d83208f..5c99e73 100644 --- a/tokenise+train.py +++ b/tokenise+train.py @@ -20,7 +20,10 @@ def train(ouputdir: Path, blocksize: int, vocabsize: int, num_steps: int, gpu: b from aitextgen import aitextgen exts = ['.json', '.gz'] - files = [x for x in ouputdir.glob('*') if x.suffix in exts and x != "config.json"] + files = [x for x in ouputdir.glob('*') if x.suffix in exts and x.name != "config.json"] + + print(files) + if len(files) == 2: if files[0].suffix == '.json': tok = str(files[0])