List_server_busy/selection/sel.py

import os, json, glob, logging
from selection import strutil
from threading import Lock
import config
import export.exportxml

sel = os.path.join(config.selection['path'], config.selection['sel'])
sel_dump = os.path.join(config.selection['path'], config.selection['sel_dump'])

LL = Lock()

# TAGS
def update(tag, newtag, newdesc):
	with LL:
		d = load_selection()
		if not tag in list(d.keys()):
			return False
		else:
			if newtag != tag:
				d[newtag] = d.pop(tag)
			else:
				d[tag]['desc'] = newdesc
		write_selection(d)

		sd = load_selection_dump()
		if not tag in list(sd.keys()):
			logging.warning("possible inconsistency between sel and sel_dump...")
		else:
			if newtag != tag:
				sd[newtag] = sd.pop(tag)
			else:
				sd[tag]['desc'] = newdesc
		write_selection_dump(sd)

		return True

def delete(tag):
	with LL:
		d = load_selection()
		if not tag in list(d.keys()):
			return False
		else:
			d.pop(tag)
		write_selection(d)

		sd = load_selection_dump()
		if not tag in list(sd.keys()):
			logging.warning("possible inconsistency between sel and sel_dump...")
		else:
			sd.pop(tag)
		write_selection_dump(sd)

		return True

def new(tag, desc):
	with LL:
		d = load_selection()
		if tag in list(d.keys()):
			return False
		else:
			d[tag] = {"desc": desc, "lists": []}
		write_selection(d)

		sd = load_selection_dump()
		sd[tag] = {"desc": desc, "lists": []}
		write_selection_dump(sd)

		return True

# URL
def delete_url(tag, url):
	with LL:
		d = load_selection()
		if tag not in list(d.keys()):
			return False

		for m in d[tag]['lists']:
			if m['url'] == url:
				d[tag]['lists'].remove(m)
				break
		write_selection(d)

		sd = load_selection_dump()
		if not tag in list(sd.keys()):
			logging.warning("possible inconsistency between sel and sel_dump...")
		else:
			for m in sd[tag]['lists']:
				if recursive_delete(m, sd[tag]['lists'], url):
					break			
		write_selection_dump(sd)

		return True


def load_selection():
	with open(sel, encoding='utf-8') as f:
		d = json.load(f)
	return d

def load_selection_dump():
	with open(sel_dump, encoding='utf-8') as f:
		d = json.load(f)
	return d

def write_selection(d):
	with open(sel, 'w+', encoding='utf-8') as f:
		json.dump(d, f, indent=4)

def write_selection_dump(d):
	with open(sel_dump, 'w+', encoding='utf-8') as f:
		json.dump(d, f, indent=4)

def lists():
	return os.listdir(config.archives)

def tags():
	d = load_selection()
	tags = []
	for k, v in d.items():
		tags.append({'tag': k, 'desc': v['desc']})
	return tags

def tags_list():
	d = load_selection()
	return list(d.keys())


def tags_w_lists():
	d = load_selection_dump()
	tags = []
	for k, v in d.items():
		t = {'tag': k, 'desc': v['desc']}
		l = []
		for m in v['lists']:
			l += recursive_info(m, keep_hierachy=True)
		t['lists'] = l
		t['export'] = export.exportxml.export_generate_path(k)
		tags.append(t)
	return tags

def recursive_find(msg, li, url):
	if msg['url'] == url:
		msg['list'] = li 									# <-- taggin
		return msg
	if 'follow-up' in list(msg.keys()):
		for m in msg['follow-up']:
			f = recursive_find(m, li, url)
			if f is not None:
				return msg 									# <-- parent thread
	return None

def recursive_delete(msg, parent_list, url):
	if msg['url'] == url:
		parent_list.remove(msg)
		return True
	d = False
	if 'follow-up' in list(msg.keys()):
		for m in msg['follow-up']:
			d = d | recursive_delete(m, msg['follow-up'], url)
	return d



def find(li, url):

	d = os.path.join(config.archives, li)

	if not os.path.isdir(d):
		logging.warning("Invalid archive path: " + d)
		print("Invalid archive path: " + d)
		return None

	dir_files = [f for f in glob.glob(os.path.join(d, "*.json"))]

	for f in dir_files:
		with open(f, encoding='utf-8') as fp:
			dj = json.load(fp)

			for msg in dj['threads']:
				f = recursive_find(msg, li, url)
				if f is not None:
					return f

	return None

def recursive_urls(msg):
	r = [msg['url']]
	if 'follow-up' in list(msg.keys()):
		for m in msg['follow-up']:
			r += recursive_urls(m)
	return r

# <li><a href="' + h.url+ '" target="_blank">' + h.subject + '</a> -- <i>' + h.author_name + '</i>
def recursive_info(msg, keep_hierachy=False):

	print(msg['url'])

	r = [{'url': msg['url'], 'subject': msg['subject'], 'author_name': msg['author_name']}]

	if keep_hierachy: 
		rr = []

	if 'follow-up' in list(msg.keys()):
		for m in msg['follow-up']:
			if keep_hierachy:
				rr += recursive_info(m)
			else:
				r += recursive_info(m)

	if keep_hierachy: 
		r[0]['follow'] = rr 						#note: change of field name 

	return r

def commit_selection(li, url, tag):

	d = load_selection()

	for i in d[tag]['lists']:
		if i['url'] == url:
			return False

	d[tag]['lists'].append({'list': li, 'url': url})

	with open(sel, 'w', encoding='utf-8') as f:
		json.dump(d, f, ensure_ascii=False, indent=4)

	return True

def commit_dump(li, url, tag):

	if not commit_selection(li, url, tag):
		return None

	m = find(li, url)												# <--- time

	if m is not None:

		dump = load_selection_dump()

		if tag not in list(dump.keys()):
			dump[tag] = {"desc": desc, "lists": []}

		m['list'] = li
		dump[tag]['lists'].append(m)
 
		with open(sel_dump, 'w+', encoding='utf-8') as fout:
			json.dump(dump, fout, ensure_ascii=False, indent=4)

		commited = recursive_urls(m)
		return commited

	return None

def commit_from_selection():

	dump = {}
	d = load_selection()

	for k, v in d.items():
		dump[k] = {'desc': v['desc'], 'lists': []}
		for i in v['lists']:
			m = find(i['list'], i['url'])							# <--- time
			if m is not None:
				m['list'] = i['list']
				dump[k]['lists'].append(m)

	with open(sel_dump, 'w+', encoding='utf-8') as f:
		json.dump(dump, f, ensure_ascii=False, indent=4)

	return True

# def report():

# 	d = load_selection()

# 	re = "Report: \n"
# 	for k, v in d.items():
# 		lre = {}
# 		for i in v:
# 			if i['list'] not in lre:
# 				lre[i['list']] = 0
# 			lre[i['list']] += 1
# 		re += "<" + k + ">: " + str(len(v)) + " ("
# 		for kk, vv in lre.items():
# 			re += kk + ": " + str(vv) + " / "
# 		re += ")\n"

# 	return re		

def recursive_format(msg):
	msg.pop('id')
	msg['len'] = len(msg['content'])
	msg.pop('content')
	msg.pop('content-type')
	if 'to' in msg:
		msg.pop('to')
	if 'message-id' in msg:
		msg.pop('message-id')
	if 'follow-up' in msg:
		for i in msg['follow-up']:
			recursive_format(i)

def format_selection():

	d = load_selection_dump()

	for k, v in d.items():
		for i in v['lists']:
			recursive_format(i)

	return d

def recursive_hashmap(msg, tag, hm):
	hm[msg['url']] = tag
	if 'follow-up' in msg:
		for i in msg['follow-up']:
			recursive_hashmap(i, tag, hm)

def hashmap():
	d = load_selection_dump()	
	hm = {}
	for k, v in d.items():
		for i in v['lists']:
			recursive_hashmap(i, k, hm)
	return hm

if __name__ == "__main__":

	d = format_selection()
	print(json.dumps(d, indent=4, sort_keys=True))

def reorder_selection_orphans(tag):

	d = load_selection_dump()
	if tag not in list(d.keys()):
		return

	msgs = d[tag]['lists']
	threads = []
	orphans = []
	for m in msgs:
		if 'follow-up' in list(m.keys()):
			threads.append(m)
		else:
			orphans.append(m)

	for o in orphans:
		subject = o['subject']
		threads.sort(key=lambda x: strutil.cosine_dist(x['subject'], subject), reverse=True)

		if strutil.cosine_dist(threads[0]['subject'], subject) > 0.1:
			msgs[msgs.index(threads[0])]['follow-up'].append(o)
			msgs.remove(o)


	d[tag]['lists'] = msgs
	with open(sel_dump, 'w', encoding='utf-8') as f:
		json.dump(d, f, ensure_ascii=False, indent=4)