From b8d6a33bf8330bd9a19f7fbaaf92f5f8daa27705 Mon Sep 17 00:00:00 2001 From: gauthiier Date: Thu, 10 Feb 2022 08:56:33 +0100 Subject: [PATCH] fixed rerun --- tokenise+train.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tokenise+train.py b/tokenise+train.py index d83208f..5c99e73 100644 --- a/tokenise+train.py +++ b/tokenise+train.py @@ -20,7 +20,10 @@ def train(ouputdir: Path, blocksize: int, vocabsize: int, num_steps: int, gpu: b from aitextgen import aitextgen exts = ['.json', '.gz'] - files = [x for x in ouputdir.glob('*') if x.suffix in exts and x != "config.json"] + files = [x for x in ouputdir.glob('*') if x.suffix in exts and x.name != "config.json"] + + print(files) + if len(files) == 2: if files[0].suffix == '.json': tok = str(files[0])