387 lines
9.6 KiB
Python
387 lines
9.6 KiB
Python
from flask import render_template, request, jsonify
|
|
from www import app
|
|
import json, logging, os, glob
|
|
from lxml import etree as et
|
|
import config
|
|
from collections import OrderedDict
|
|
import db.db as db
|
|
|
|
'''
|
|
UTILS
|
|
'''
|
|
|
|
def list_all(d, ext):
|
|
|
|
if not os.path.isdir(d):
|
|
logging.error(d + " is not a valid directory.")
|
|
return None
|
|
|
|
return [os.path.basename(f) for f in glob.glob(os.path.join(d, "*." + ext))]
|
|
|
|
def read_index(d, fn):
|
|
fp = os.path.join(d, fn)
|
|
if not os.path.isfile(fp):
|
|
return None
|
|
|
|
with open(fp) as f:
|
|
index_data = json.load(f, object_pairs_hook=OrderedDict)
|
|
|
|
return index_data
|
|
|
|
'''
|
|
ROOT
|
|
'''
|
|
|
|
@app.route('/')
|
|
def top():
|
|
index = read_index(config.index['path'], config.index['all'])
|
|
if index is not None:
|
|
return render_template("index.html", index=index);
|
|
else:
|
|
return "Error no index...."
|
|
|
|
'''
|
|
SAVE
|
|
'''
|
|
|
|
@app.route('/save')
|
|
def save():
|
|
index = read_index(config.index['path'], config.index['all'])
|
|
if index is not None:
|
|
rendered = render_template("index.html", index=index);
|
|
with open(os.path.join(config.deploy['path'], config.deploy['name']), 'w') as fp:
|
|
fp.write(rendered)
|
|
return rendered
|
|
else:
|
|
return "Error no index...."
|
|
|
|
'''
|
|
SEARCH
|
|
'''
|
|
|
|
def format_chapter_section(nbr):
|
|
tok = nbr.split('.')
|
|
return "Ch." + tok[0] + " §" + tok[1] + "." + tok[2]
|
|
|
|
def key_from_chapter_section(str):
|
|
tok = str.replace("Ch.", "").replace(" §", ".")
|
|
if '-' in tok:
|
|
tok = tok.split('-')[0]
|
|
return tuple([int(j) for j in tok.split('.')])
|
|
|
|
def add_term_index(term, refs):
|
|
|
|
index = read_index(config.index['path'], config.index['all'])
|
|
if index is None:
|
|
logging.error("No index.all...")
|
|
return False
|
|
|
|
if term in index:
|
|
logging.warning("Term: " + term + " is already indexed... skipping add")
|
|
return True
|
|
|
|
refs = sorted(refs, key=lambda x: key_from_chapter_section(x['nbr']))
|
|
index[term] = refs
|
|
|
|
index = OrderedDict(sorted(index.items(), key=lambda x: x[0].lower()))
|
|
|
|
with open(os.path.join(config.index['path'], config.index['all']), 'w') as fout:
|
|
json.dump(index, fout, indent=4, ensure_ascii=False)
|
|
|
|
return True
|
|
|
|
|
|
@app.route('/search', methods = ['GET', 'POST'])
|
|
def search():
|
|
if request.method == 'GET':
|
|
|
|
data = request.values
|
|
if len(data) == 0:
|
|
index = read_index(config.index['path'], config.index['all'])
|
|
if index is not None:
|
|
return render_template("search.html", index=index);
|
|
else:
|
|
return "Error no index...."
|
|
else:
|
|
a = data.get('action')
|
|
if a == "search":
|
|
k_arg = request.args.get('keyword')
|
|
f_arg = request.args.get('field')
|
|
|
|
print(k_arg)
|
|
print(f_arg)
|
|
|
|
with db.DB(config.list_server_busy_db) as dba:
|
|
|
|
if f_arg == 'content':
|
|
res = dba.content_search(k_arg)
|
|
else:
|
|
res = dba.from_search(k_arg)
|
|
|
|
print(res)
|
|
|
|
# format data to return
|
|
search_results = { "keyword": k_arg, "field": f_arg, "results": [] }
|
|
for r in res:
|
|
# {'nbr': nbr_, 'from': author_name_, 'to': to_, 'subject': subject_, 'date': date_, 'url': url_}
|
|
search_results['results'].append({'nbr': format_chapter_section(r['nbr']), 'url': r['url'] if r['url'] != 'n/a' else ''})
|
|
|
|
# sort?
|
|
search_results['results'] = sorted(search_results['results'], key=lambda x: key_from_chapter_section(x['nbr']))
|
|
return jsonify(search_results)
|
|
else:
|
|
return "-"
|
|
|
|
elif request.method == 'POST':
|
|
|
|
data = request.json
|
|
a = data.get('action')
|
|
|
|
if a == "add":
|
|
term = data['term']
|
|
refs = data['refs']
|
|
if add_term_index(term, refs):
|
|
return 'ok'
|
|
|
|
return "-"
|
|
|
|
@app.route('/save_to_index', methods = ['POST'])
|
|
def save_to_index():
|
|
if request.method == 'POST':
|
|
k_arg = request.args.get('keyword')
|
|
f_arg = request.args.get('field')
|
|
|
|
with db.DB(config.list_server_busy_db) as dba:
|
|
|
|
if f_arg == 'content':
|
|
res = dba.content_search(k_arg)
|
|
else:
|
|
res = dba.from_search(k_arg)
|
|
|
|
|
|
|
|
|
|
|
|
'''
|
|
INDEX
|
|
'''
|
|
|
|
|
|
def modify_selected_kw_index(d, fn, kw, action="add"):
|
|
fp = os.path.join(d, fn)
|
|
if not os.path.isfile(fp):
|
|
return False
|
|
|
|
with open(fp) as f:
|
|
index_data = json.load(f)
|
|
|
|
if action == 'add':
|
|
in_dic = index_data['selected']
|
|
out_dic = index_data['orphan']
|
|
elif action == 'delete':
|
|
out_dic = index_data['selected']
|
|
in_dic = index_data['orphan']
|
|
else:
|
|
return False
|
|
|
|
if kw not in out_dic:
|
|
return False
|
|
|
|
v = out_dic.pop(kw)
|
|
if kw not in in_dic:
|
|
in_dic[kw] = []
|
|
|
|
in_dic[kw] += v
|
|
|
|
with open(fp, 'w') as fout:
|
|
json.dump(index_data, fout, indent=4, sort_keys=True, ensure_ascii=False)
|
|
|
|
return True
|
|
|
|
def read_index_master(d, fn):
|
|
fp = os.path.join(d, fn)
|
|
if not os.path.isfile(fp):
|
|
return False
|
|
|
|
with open(fp) as f:
|
|
index_data = json.load(f, object_pairs_hook=OrderedDict)
|
|
|
|
return index_data
|
|
|
|
@app.route('/index', methods = ['GET'])
|
|
def index():
|
|
if request.method == 'GET':
|
|
li = list_all(config.index['path'], 'js')
|
|
li = sorted(li, key=lambda x: int(x.split('.')[0]))
|
|
return render_template("list_files_all.html", title="INDEX [all]", prefix="/index/", files=li)
|
|
|
|
@app.route('/index/<path:fn>', methods = ['GET', 'POST'])
|
|
def indexfn(fn):
|
|
if request.method == 'GET':
|
|
data = read_index(config.index['path'], fn)
|
|
if data is not None:
|
|
return render_template("indx.html", fn=fn, selected=data['selected'], orphan=data['orphan'])
|
|
else:
|
|
return "File: " + fn + "does not exist."
|
|
elif request.method == 'POST':
|
|
data = request.form
|
|
a = data.get('action')
|
|
if a == "add":
|
|
logging.info("POST ADD " + fn + " -- " + data.get('kw') + " ++ " + data.get('list'))
|
|
if modify_selected_kw_index(config.index['path'], fn, data.get('kw')):
|
|
return "ok"
|
|
elif a == "delete":
|
|
logging.info("POST DELETE " + fn + " -- " + data.get('kw') + " ++ " + data.get('list'))
|
|
if modify_selected_kw_index(config.index['path'], fn, data.get('kw'), action="delete"):
|
|
return "ok"
|
|
return "-"
|
|
|
|
@app.route('/index-master', methods = ['GET', 'POST'])
|
|
def indexmaster():
|
|
if request.method == 'GET':
|
|
data = read_index_master(config.index['path'], config.index['master'])
|
|
if data is not None:
|
|
return render_template("indx.master.html", fn="INDEX [MASTER]", master=data)
|
|
else:
|
|
return "File: " + os.path.join(config.index['path'], config.index['master']) + "does not exist."
|
|
elif request.method == 'POST':
|
|
data = request.form
|
|
a = data.get('action')
|
|
print(a)
|
|
if a == "regex":
|
|
logging.info("POST REGEX " + data.get('kw') + " ++ " + data.get('reg'))
|
|
reg = json.loads(data.get('reg').replace("'", ""))
|
|
print(type(data.get('reg')))
|
|
return "POST REGEX " + data.get('kw') + " ++ " + data.get('reg')
|
|
# if modify_selected_kw_index(config.index['path'], fn, data.get('kw')):
|
|
# return "ok"
|
|
elif a == "collate":
|
|
logging.info("POST COLLATE " + data.get('kw') + " ++ " + data.get('col'))
|
|
return "POST COLLATE " + data.get('kw') + " ++ " + data.get('col')
|
|
# if modify_selected_kw_index(config.index['path'], fn, data.get('kw'), action="delete"):
|
|
# return "ok"
|
|
elif a == "delete":
|
|
logging.info("POST DELETE " + data.get('kw'))
|
|
return "POST DELETE " + data.get('kw')
|
|
# if modify_selected_kw_index(config.index['path'], fn, data.get('kw'), action="delete"):
|
|
# return "ok"
|
|
return "-"
|
|
|
|
|
|
'''
|
|
XML
|
|
'''
|
|
|
|
def read_xml(d, fn):
|
|
fp = os.path.join(d, fn)
|
|
if not os.path.isfile(fp):
|
|
return None
|
|
|
|
root = et.parse(fp).getroot()
|
|
xml_data = []
|
|
for m in root.findall('mails/mail'):
|
|
nbr_str = m.find('nbr').text
|
|
date_str = m.find('date').text
|
|
from_str = m.find('from').text
|
|
subject_str = m.find('subject').text
|
|
xml_data.append({'nbr': nbr_str, 'date': date_str, 'from': from_str, 'subject': subject_str})
|
|
|
|
# sort
|
|
xml_data[:] = sorted(xml_data, key=lambda x: parse_nbr_xml(x['nbr']))
|
|
return xml_data
|
|
|
|
|
|
def parse_nbr_xml(nbr_str):
|
|
if '-' in nbr_str:
|
|
nbr_str = nbr_str.split('-')[0]
|
|
# split number and return a tuple
|
|
# ex. 15.4 -> (15, 4)
|
|
tok = nbr_str.split('.')
|
|
tup = (int(tok[0]), int(tok[1]))
|
|
return tup
|
|
|
|
def update_nbr_xml(d, fn, nbr, new_nbr, date):
|
|
fp = os.path.join(d, fn)
|
|
if not os.path.isfile(fp):
|
|
return False
|
|
|
|
tr = et.parse(fp)
|
|
m = tr.xpath(".//nbr[text()='" + nbr + "']")
|
|
|
|
if m is None:
|
|
return False
|
|
|
|
mm = None
|
|
for k in m:
|
|
if k.getparent().find('date').text == date:
|
|
mm = k
|
|
break
|
|
|
|
if mm is None:
|
|
return False
|
|
|
|
# replace nbr
|
|
mm.text = new_nbr
|
|
|
|
# order all mails according to their nbr
|
|
mails = tr.find('mails')
|
|
mails[:] = sorted(mails, key=lambda x: parse_nbr_xml(x.find('nbr').text))
|
|
|
|
tr.write(fp)
|
|
return True
|
|
|
|
|
|
def delete_nbr_xml(d, fn, nbr, date):
|
|
fp = os.path.join(d, fn)
|
|
if not os.path.isfile(fp):
|
|
return False
|
|
|
|
tr = et.parse(fp)
|
|
m = tr.xpath(".//nbr[text()='" + nbr + "']")
|
|
|
|
if m is None:
|
|
return False
|
|
|
|
mm = None
|
|
for k in m:
|
|
if k.getparent().find('date').text == date:
|
|
mm = k
|
|
break
|
|
|
|
if mm is None:
|
|
return False
|
|
|
|
mail = mm.getparent()
|
|
mail.getparent().remove(mail)
|
|
|
|
tr.write(fp)
|
|
return True
|
|
|
|
@app.route('/xml', methods = ['GET'])
|
|
def xml():
|
|
if request.method == 'GET':
|
|
li = list_all(config.xml['path'], 'xml')
|
|
li = sorted(li, key=lambda x: int(x.split('.')[0]))
|
|
return render_template("list_files_all.html", title="XML [all]", prefix="/xml/", files=li)
|
|
|
|
@app.route('/xml/<path:fn>', methods = ['GET', 'POST'])
|
|
def xmlfn(fn):
|
|
if request.method == 'GET':
|
|
data = read_xml(config.xml['path'], fn)
|
|
if data is not None:
|
|
return render_template("xml.html", fn=fn, data=data)
|
|
else:
|
|
return "File: " + fn + "does not exist."
|
|
elif request.method == 'POST':
|
|
data = request.form
|
|
a = data.get('action')
|
|
if a == "update":
|
|
logging.info("POST UPDATE " + fn + " -- " + data.get('nbr') + " ++ " + data.get('new_nbr'))
|
|
if update_nbr_xml(config.xml['path'], fn, data.get('nbr'), data.get('new_nbr'), data.get('date')):
|
|
return "ok"
|
|
elif a == "delete":
|
|
logging.info("POST DELETE " + fn + " -- " + data.get('nbr'))
|
|
if delete_nbr_xml(config.xml['path'], fn, data.get('nbr'), data.get('date')):
|
|
return "ok"
|
|
return "-"
|