Full_digest_rescheduled/collate_indexes.py

45 lines
936 B
Python
Raw Normal View History

2020-01-25 10:57:13 +01:00
import argparse, os, glob, json, logging
import config
def list_all(d, ext):
if not os.path.isdir(d):
logging.error(d + " is not a valid directory.")
return None
return [f for f in glob.glob(os.path.join(d, "*." + ext))]
if __name__ == '__main__':
files = list_all(config.index['path'], 'js')
index = {}
master_fn = os.path.join(config.index['path'], config.index['master'])
if os.path.isfile(master_fn):
with open(master_fn) as master_fp:
try:
index = json.load(master_fp)
except Exception as e:
pass
for f in files:
logging.info("Reading : " + f)
with open(f) as fp:
d = json.load(fp)
selected = d['selected']
for s, vv in selected.items():
if s not in index:
index[s] = {'regex': [s], 'indx': []}
# no duplicates
for v in vv:
if v not in index[s]['indx']:
index[s]['indx'].append(v)
print(json.dumps(index, indent=4, sort_keys=True, ensure_ascii=False))