gauthiier/List_server_busy
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

semi working

Browse Source
This commit is contained in:
gauthiier
2019-12-23 14:54:12 +01:00
parent 74fb0d6f0b
commit 45393ec97d
10 changed files with 768 additions and 4338 deletions
Download Patch File Download Diff File Expand all files Collapse all files
selection/sel.py
+129 -35
View File
@@ -1,10 +1,70 @@
import os, json, glob, logging
from selection import strutil
from threading import Lock
import config
sel = os.path.join(config.selection, "tm-selection.js")
sel_dump = os.path.join(config.selection, "tm-selection-dump.js")
LL = Lock()
def update(tag, newtag, newdesc):
with LL:
d = load_selection()
if not tag in list(d.keys()):
return False
else:
if newtag != tag:
d[newtag] = d.pop(tag)
else:
d[tag]['desc'] = newdesc
write_selection(d)
sd = load_selection_dump()
if not tag in list(sd.keys()):
logging.warning("possible inconsistency between sel and sel_dump...")
else:
if newtag != tag:
sd[newtag] = sd.pop(tag)
else:
sd[tag]['desc'] = newdesc
write_selection_dump(sd)
return True
def delete(tag):
with LL:
d = load_selection()
if not tag in list(d.keys()):
return False
else:
d.pop(tag)
write_selection(d)
sd = load_selection_dump()
if not tag in list(sd.keys()):
logging.warning("possible inconsistency between sel and sel_dump...")
else:
sd.pop(tag)
write_selection_dump(sd)
return True
def new(tag, desc):
with LL:
d = load_selection()
if tag in list(d.keys()):
return False
else:
d[tag] = {"desc": desc, "lists": []}
write_selection(d)
sd = load_selection_dump()
sd[tag] = {"desc": desc, "lists": []}
write_selection_dump(sd)
return True
ARCH = "archives/"
EXP = "selection/"
sel = os.path.join(EXP, "tm-selection.js")
sel_dump = os.path.join(EXP, "tm-selection-dump.js")
def load_selection():
with open(sel, encoding='utf-8') as f:
@@ -16,14 +76,41 @@ def load_selection_dump():
d = json.load(f)
return d
def write_selection(d):
with open(sel, 'w+', encoding='utf-8') as f:
json.dump(d, f, indent=4)
def write_selection_dump(d):
with open(sel_dump, 'w+', encoding='utf-8') as f:
json.dump(d, f, indent=4)
def lists():
return os.listdir(ARCH)
return os.listdir(config.archives)
def tags():
d = load_selection()
tags = []
for k, v in d.items():
tags.append({'tag': k, 'desc': v['desc']})
return tags
def tags_list():
d = load_selection()
return list(d.keys())
def tags_w_lists():
d = load_selection_dump()
tags = []
for k, v in d.items():
t = {'tag': k, 'desc': v['desc']}
l = []
for m in v['lists']:
l += recursive_info(m)
t['lists'] = l
tags.append(t)
return tags
def recursive_find(msg, li, url):
if msg['url'] == url:
msg['list'] = li # <-- taggin
@@ -37,7 +124,7 @@ def recursive_find(msg, li, url):
def find(li, url):
d = os.path.join(ARCH, li)
d = os.path.join(config.archives, li)
if not os.path.isdir(d):
logging.warning("Invalid archive path: " + d)
@@ -64,19 +151,23 @@ def recursive_urls(msg):
r += recursive_urls(m)
return r
# <li><a href="' + h.url+ '" target="_blank">' + h.subject + '</a> -- <i>' + h.author_name + '</i>
def recursive_info(msg):
r = [{'url': msg['url'], 'subject': msg['subject'], 'author_name': msg['author_name']}]
if 'follow-up' in list(msg.keys()):
for m in msg['follow-up']:
r += recursive_info(m)
return r
def commit_selection(li, url, tag):
d = load_selection()
if tag not in list(d.keys()):
print("new tag: " + tag)
d[tag] = []
for i in d[tag]:
for i in d[tag]['lists']:
if i['url'] == url:
return False
d[tag].append({'list': li, 'url': url})
d[tag]['lists'].append({'list': li, 'url': url})
with open(sel, 'w', encoding='utf-8') as f:
json.dump(d, f, ensure_ascii=False, indent=4)
@@ -89,14 +180,15 @@ def commit_dump(li, url, tag):
return None
m = find(li, url) # <--- time
if m is not None:
dump = load_selection_dump()
if tag not in list(dump.keys()):
dump[tag] = []
dump[tag] = {"desc": desc, "lists": []}
dump[tag].append(m)
dump[tag]['lists'].append(m)
with open(sel_dump, 'w+', encoding='utf-8') as fout:
json.dump(dump, fout, ensure_ascii=False, indent=4)
@@ -112,33 +204,35 @@ def commit_from_selection():
d = load_selection()
for k, v in d.items():
dump[k] = []
for i in v:
dump[k] = {'desc': v['desc'], 'lists': []}
for i in v['lists']:
m = find(i['list'], i['url']) # <--- time
if m is not None:
m['list'] = i['list']
dump[k].append(m)
dump[k]['lists'].append(m)
with open(sel_dump, 'w+', encoding='utf-8') as f:
json.dump(dump, f, ensure_ascii=False, indent=4)
def report():
return True
d = load_selection()
# def report():
re = "Report: \n"
for k, v in d.items():
lre = {}
for i in v:
if i['list'] not in lre:
lre[i['list']] = 0
lre[i['list']] += 1
re += "<" + k + ">: " + str(len(v)) + " ("
for kk, vv in lre.items():
re += kk + ": " + str(vv) + " / "
re += ")\n"
# d = load_selection()
return re
# re = "Report: \n"
# for k, v in d.items():
# lre = {}
# for i in v:
# if i['list'] not in lre:
# lre[i['list']] = 0
# lre[i['list']] += 1
# re += "<" + k + ">: " + str(len(v)) + " ("
# for kk, vv in lre.items():
# re += kk + ": " + str(vv) + " / "
# re += ")\n"
# return re
def recursive_format(msg):
msg.pop('id')
@@ -158,7 +252,7 @@ def format_selection():
d = load_selection_dump()
for k, v in d.items():
for i in v:
for i in v['lists']:
recursive_format(i)
return d
@@ -173,7 +267,7 @@ def hashmap():
d = load_selection_dump()
hm = {}
for k, v in d.items():
for i in v:
for i in v['lists']:
recursive_hashmap(i, k, hm)
return hm
@@ -188,7 +282,7 @@ def reorder_selection_orphans(tag):
if tag not in list(d.keys()):
return
msgs = d[tag]
msgs = d[tag]['lists']
threads = []
orphans = []
for m in msgs:
@@ -206,7 +300,7 @@ def reorder_selection_orphans(tag):
msgs.remove(o)
d[tag] = msgs
d[tag]['lists'] = msgs
with open(sel_dump, 'w', encoding='utf-8') as f:
json.dump(d, f, ensure_ascii=False, indent=4)
selection/tm-selection-dump.js
+415 -4170
View File
File diff suppressed because one or more lines are too long
selection/tm-selection.js
+17 -123
View File
@@ -1,125 +1,19 @@
{
"cyber": [
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9706/msg00111.html"
}
],
"end2end": [],
"net.art": [
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=15160"
},
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=40578"
},
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=41637"
},
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=42354"
},
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=43292"
},
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=47543"
},
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=48346"
},
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=54012"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9708/msg00010.html"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9708/msg00009.html"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9703/msg00038.html"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9703/msg00060.html"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9703/msg00096.html"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9610/msg00029.html"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9705/msg00053.html"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9705/msg00003.html"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9707/msg00014.html"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00109.html"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00117.html"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00084.html"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00162.html"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00153.html"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00202.html"
},
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1310&L=new-media-curating&F=&S=&P=2942"
},
{
"list": "empyre",
"url": "http://lists.cofa.unsw.edu.au/pipermail/empyre/2016-September/009250.html"
},
{
"list": "empyre",
"url": "http://lists.cofa.unsw.edu.au/pipermail/empyre/2016-September/009253.html"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-0009/msg00209.html"
}
],
"new media art": [
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-0905/msg00038.html"
}
]
"net.art": {
"desc": "...",
"lists": [
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1310&L=new-media-curating&F=&S=&P=2942"
},
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1207&L=new-media-curating&F=&S=&P=1869"
}
]
},
"end2end": {
"desc": "...",
"lists": []
}
}