gauthiier/List_server_busy
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

semi working

Browse Source
This commit is contained in:
gauthiier 2019-12-23 14:54:12 +01:00
parent 74fb0d6f0b
commit 45393ec97d
10 changed files with 768 additions and 4338 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
config/__init__.py Normal file
View File

@ -0,0 +1,3 @@
listservs = {'url': 'http://0.0.0.0:5555', 'lists_to_serve': []}
archives = "/Users/gauthiier/DDDDDDD/D/__projects/MAILINGLISTSARCHIVES"
selection = "selection/"

164
selection/sel.py
View File

@ -1,10 +1,70 @@
import os, json, glob, logging import os, json, glob, logging
from selection import strutil from selection import strutil
from threading import Lock
import config
sel = os.path.join(config.selection, "tm-selection.js")
sel_dump = os.path.join(config.selection, "tm-selection-dump.js")
LL = Lock()
def update(tag, newtag, newdesc):
with LL:
d = load_selection()
if not tag in list(d.keys()):
return False
else:
if newtag != tag:
d[newtag] = d.pop(tag)
else:
d[tag]['desc'] = newdesc
write_selection(d)
sd = load_selection_dump()
if not tag in list(sd.keys()):
logging.warning("possible inconsistency between sel and sel_dump...")
else:
if newtag != tag:
sd[newtag] = sd.pop(tag)
else:
sd[tag]['desc'] = newdesc
write_selection_dump(sd)
return True
def delete(tag):
with LL:
d = load_selection()
if not tag in list(d.keys()):
return False
else:
d.pop(tag)
write_selection(d)
sd = load_selection_dump()
if not tag in list(sd.keys()):
logging.warning("possible inconsistency between sel and sel_dump...")
else:
sd.pop(tag)
write_selection_dump(sd)
return True
def new(tag, desc):
with LL:
d = load_selection()
if tag in list(d.keys()):
return False
else:
d[tag] = {"desc": desc, "lists": []}
write_selection(d)
sd = load_selection_dump()
sd[tag] = {"desc": desc, "lists": []}
write_selection_dump(sd)
return True
ARCH = "archives/"
EXP = "selection/"
sel = os.path.join(EXP, "tm-selection.js")
sel_dump = os.path.join(EXP, "tm-selection-dump.js")
def load_selection(): def load_selection():
with open(sel, encoding='utf-8') as f: with open(sel, encoding='utf-8') as f:
@ -16,14 +76,41 @@ def load_selection_dump():
d = json.load(f) d = json.load(f)
return d return d
def write_selection(d):
with open(sel, 'w+', encoding='utf-8') as f:
json.dump(d, f, indent=4)
def write_selection_dump(d):
with open(sel_dump, 'w+', encoding='utf-8') as f:
json.dump(d, f, indent=4)
def lists(): def lists():
return os.listdir(ARCH) return os.listdir(config.archives)
def tags(): def tags():
d = load_selection()
tags = []
for k, v in d.items():
tags.append({'tag': k, 'desc': v['desc']})
return tags
def tags_list():
d = load_selection() d = load_selection()
return list(d.keys()) return list(d.keys())
def tags_w_lists():
d = load_selection_dump()
tags = []
for k, v in d.items():
t = {'tag': k, 'desc': v['desc']}
l = []
for m in v['lists']:
l += recursive_info(m)
t['lists'] = l
tags.append(t)
return tags
def recursive_find(msg, li, url): def recursive_find(msg, li, url):
if msg['url'] == url: if msg['url'] == url:
msg['list'] = li # <-- taggin msg['list'] = li # <-- taggin
@ -37,7 +124,7 @@ def recursive_find(msg, li, url):
def find(li, url): def find(li, url):
d = os.path.join(ARCH, li) d = os.path.join(config.archives, li)
if not os.path.isdir(d): if not os.path.isdir(d):
logging.warning("Invalid archive path: " + d) logging.warning("Invalid archive path: " + d)
@ -64,19 +151,23 @@ def recursive_urls(msg):
r += recursive_urls(m) r += recursive_urls(m)
return r return r
# <li><a href="' + h.url+ '" target="_blank">' + h.subject + '</a> -- <i>' + h.author_name + '</i>
def recursive_info(msg):
r = [{'url': msg['url'], 'subject': msg['subject'], 'author_name': msg['author_name']}]
if 'follow-up' in list(msg.keys()):
for m in msg['follow-up']:
r += recursive_info(m)
return r
def commit_selection(li, url, tag): def commit_selection(li, url, tag):
d = load_selection() d = load_selection()
if tag not in list(d.keys()): for i in d[tag]['lists']:
print("new tag: " + tag)
d[tag] = []
for i in d[tag]:
if i['url'] == url: if i['url'] == url:
return False return False
d[tag].append({'list': li, 'url': url}) d[tag]['lists'].append({'list': li, 'url': url})
with open(sel, 'w', encoding='utf-8') as f: with open(sel, 'w', encoding='utf-8') as f:
json.dump(d, f, ensure_ascii=False, indent=4) json.dump(d, f, ensure_ascii=False, indent=4)
@ -89,14 +180,15 @@ def commit_dump(li, url, tag):
return None return None
m = find(li, url) # <--- time m = find(li, url) # <--- time
if m is not None: if m is not None:
dump = load_selection_dump() dump = load_selection_dump()
if tag not in list(dump.keys()): if tag not in list(dump.keys()):
dump[tag] = [] dump[tag] = {"desc": desc, "lists": []}
dump[tag].append(m) dump[tag]['lists'].append(m)
with open(sel_dump, 'w+', encoding='utf-8') as fout: with open(sel_dump, 'w+', encoding='utf-8') as fout:
json.dump(dump, fout, ensure_ascii=False, indent=4) json.dump(dump, fout, ensure_ascii=False, indent=4)
@ -112,33 +204,35 @@ def commit_from_selection():
d = load_selection() d = load_selection()
for k, v in d.items(): for k, v in d.items():
dump[k] = [] dump[k] = {'desc': v['desc'], 'lists': []}
for i in v: for i in v['lists']:
m = find(i['list'], i['url']) # <--- time m = find(i['list'], i['url']) # <--- time
if m is not None: if m is not None:
m['list'] = i['list'] m['list'] = i['list']
dump[k].append(m) dump[k]['lists'].append(m)
with open(sel_dump, 'w+', encoding='utf-8') as f: with open(sel_dump, 'w+', encoding='utf-8') as f:
json.dump(dump, f, ensure_ascii=False, indent=4) json.dump(dump, f, ensure_ascii=False, indent=4)
def report(): return True
d = load_selection() # def report():
re = "Report: \n" # d = load_selection()
for k, v in d.items():
lre = {}
for i in v:
if i['list'] not in lre:
lre[i['list']] = 0
lre[i['list']] += 1
re += "<" + k + ">: " + str(len(v)) + " ("
for kk, vv in lre.items():
re += kk + ": " + str(vv) + " / "
re += ")\n"
return re # re = "Report: \n"
# for k, v in d.items():
# lre = {}
# for i in v:
# if i['list'] not in lre:
# lre[i['list']] = 0
# lre[i['list']] += 1
# re += "<" + k + ">: " + str(len(v)) + " ("
# for kk, vv in lre.items():
# re += kk + ": " + str(vv) + " / "
# re += ")\n"
# return re
def recursive_format(msg): def recursive_format(msg):
msg.pop('id') msg.pop('id')
@ -158,7 +252,7 @@ def format_selection():
d = load_selection_dump() d = load_selection_dump()
for k, v in d.items(): for k, v in d.items():
for i in v: for i in v['lists']:
recursive_format(i) recursive_format(i)
return d return d
@ -173,7 +267,7 @@ def hashmap():
d = load_selection_dump() d = load_selection_dump()
hm = {} hm = {}
for k, v in d.items(): for k, v in d.items():
for i in v: for i in v['lists']:
recursive_hashmap(i, k, hm) recursive_hashmap(i, k, hm)
return hm return hm
@ -188,7 +282,7 @@ def reorder_selection_orphans(tag):
if tag not in list(d.keys()): if tag not in list(d.keys()):
return return
msgs = d[tag] msgs = d[tag]['lists']
threads = [] threads = []
orphans = [] orphans = []
for m in msgs: for m in msgs:
@ -206,7 +300,7 @@ def reorder_selection_orphans(tag):
msgs.remove(o) msgs.remove(o)
d[tag] = msgs d[tag]['lists'] = msgs
with open(sel_dump, 'w', encoding='utf-8') as f: with open(sel_dump, 'w', encoding='utf-8') as f:
json.dump(d, f, ensure_ascii=False, indent=4) json.dump(d, f, ensure_ascii=False, indent=4)

4585
selection/tm-selection-dump.js
View File

File diff suppressed because one or more lines are too long

140
selection/tm-selection.js
View File

@ -1,125 +1,19 @@
{ {
"cyber": [ "net.art": {
{ "desc": "...",
"list": "nettime_l", "lists": [
"url": "https://nettime.org/Lists-Archives/nettime-l-9706/msg00111.html" {
} "list": "crumb",
], "url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1310&L=new-media-curating&F=&S=&P=2942"
"end2end": [], },
"net.art": [ {
{ "list": "crumb",
"list": "crumb", "url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1207&L=new-media-curating&F=&S=&P=1869"
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=15160" }
}, ]
{ },
"list": "crumb", "end2end": {
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=40578" "desc": "...",
}, "lists": []
{ }
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=41637"
},
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=42354"
},
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=43292"
},
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=47543"
},
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=48346"
},
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=54012"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9708/msg00010.html"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9708/msg00009.html"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9703/msg00038.html"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9703/msg00060.html"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9703/msg00096.html"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9610/msg00029.html"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9705/msg00053.html"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9705/msg00003.html"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9707/msg00014.html"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00109.html"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00117.html"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00084.html"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00162.html"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00153.html"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00202.html"
},
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1310&L=new-media-curating&F=&S=&P=2942"
},
{
"list": "empyre",
"url": "http://lists.cofa.unsw.edu.au/pipermail/empyre/2016-September/009250.html"
},
{
"list": "empyre",
"url": "http://lists.cofa.unsw.edu.au/pipermail/empyre/2016-September/009253.html"
},
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-0009/msg00209.html"
}
],
"new media art": [
{
"list": "nettime_l",
"url": "https://nettime.org/Lists-Archives/nettime-l-0905/msg00038.html"
}
]
} }

44
www/routes.py
View File

@ -4,9 +4,9 @@ import json, logging
from selection import sel from selection import sel
import urllib.parse import urllib.parse
import urllib.request import urllib.request
import config
LISTSERVS_URL = "http://127.0.0.1:5555" lists_to_serve = config.listservs['lists_to_serve']
lists_to_serve = []
@app.route('/') @app.route('/')
def index(): def index():
@ -19,9 +19,35 @@ def selection():
return render_template("selection.html", sel=s) return render_template("selection.html", sel=s)
@app.route('/tags') @app.route('/tags', methods = ['GET', 'POST'])
def tags(): def tags():
return jsonify(result=sel.tags()) if request.method == 'GET':
return render_template("selection_tags.html", tags=sel.tags())
if request.method == 'POST':
data = request.form
a = data.get('action')
r = False
if a == "delete":
r = sel.delete(data.get('tagid'))
elif a == "update":
r = sel.update(data.get('tagid'), data.get('tag'), data.get('desc'))
elif a == "new":
r = sel.new(data.get('tag'), data.get('desc'))
if r:
return "ok"
return "-"
@app.route('/tags_w_lists', methods = ['GET', 'POST'])
def tags_w_lists():
if request.method == 'GET':
return render_template("selection_tags_w_lists.html", tags=sel.tags_w_lists())
if request.method == 'POST':
data = request.form
a = data.get('action')
if a == "dump":
if sel.commit_from_selection():
return "done"
@app.route('/report') @app.route('/report')
def report(): def report():
@ -44,7 +70,7 @@ def collect():
msg = url + " already exists..." msg = url + " already exists..."
commited = [] commited = []
return jsonify({ "msg": msg, "report": sel.report(), "commited": commited, "tag": tag}) return jsonify({ "msg": msg, "commited": commited, "tag": tag})
def filter_results_selection(res): def filter_results_selection(res):
@ -64,7 +90,7 @@ def searh():
if len(request.args) < 1: if len(request.args) < 1:
# q: list all table or keep predefined wconfig.lists_to_serve? # q: list all table or keep predefined wconfig.lists_to_serve?
# wconfig.lists_to_serve = archive.list_tables_db(config.db['database'], config.db['host'], config.db['user'], config.db['password']) # wconfig.lists_to_serve = archive.list_tables_db(config.db['database'], config.db['host'], config.db['user'], config.db['password'])
r = urllib.request.urlopen(LISTSERVS_URL + '/lists') r = urllib.request.urlopen(config.listservs['url'] + '/lists')
data = json.loads(r.read().decode('utf8')) data = json.loads(r.read().decode('utf8'))
lists_to_serve = data['lists'] lists_to_serve = data['lists']
@ -86,16 +112,16 @@ def searh():
data = {'keyword': k_arg, 'list': l_arg, 'field': f_arg} data = {'keyword': k_arg, 'list': l_arg, 'field': f_arg}
val = urllib.parse.urlencode(data) val = urllib.parse.urlencode(data)
URL = LISTSERVS_URL + '/search?' + val URL = config.listservs['url'] + '/search?' + val
logging.debug(val) logging.debug(val)
r = urllib.request.urlopen(LISTSERVS_URL + '/search?' + val) r = urllib.request.urlopen(config.listservs['url'] + '/search?' + val)
data = json.loads(r.read().decode('utf8')) data = json.loads(r.read().decode('utf8'))
filter_results_selection(data['result']) filter_results_selection(data['result'])
return jsonify({'result': data['result'], 'tags': sel.tags()}) return jsonify({'result': data['result'], 'tags': sel.tags_list()})

16
www/static/lestyle.css
View File

@ -21,4 +21,18 @@ aaa:hover {
bb { bb {
background-color: green; background-color: green;
} }
/*tags*/
.tag_item {
width: 30em;
display: block;
border: 1px black;
margin: 1em;
}
.tag_item textarea {
width: 100%;
margin-top: 0.5em;
}

47
www/static/tags.js Normal file
View File

@ -0,0 +1,47 @@
$(document).ready(function(){
$('.tagedit').prop("readonly", true);
$('.tagedit').click( function() {
console.log("tagedit");
$('.tagedit').prop("readonly", false);
});
$('.tag_item').submit(function(e) { return false; });
$('.update, .delete').click(function(e) {
var form = $(this).parent("form");
var g = form.serialize() + "&tagid=" + form.data('tagid') + "&action=" + $(this).attr("value");
$.post('/tags', g, function(d) {
if(d === 'ok') {
location.reload();
}
});
});
$('#new').click(function(e) {
var v = $(this).text();
console.log(v)
if(v == "NEW") {
var form = $(this).parent("form");
form.append('<input name="tag" type="text" id="edit-tag" value="...">');
form.append('<textarea name="desc" id="edit-desc">...</textarea>');
$(this).text("ADD");
} else if (v == "ADD") {
if($("#edit-tag").val() === "..." & $("#edit-desc").text() === "...") {
$(this).text("NEW");
$("#edit-tag").remove()
$("#edit-desc").remove()
return;
}
var form = $(this).parent("form");
var g = form.serialize() + "&action=new";
$.post('/tags', g, function(d) {
if(d === 'ok') {
location.reload();
}
});
$(this).text("NEW");
$("#edit-tag").remove()
$("#edit-desc").remove()
}
});
});

47
www/static/tags_w_lists.js Normal file
View File

@ -0,0 +1,47 @@
$(document).ready(function(){
$('.tagedit').prop("readonly", true);
$('.tagedit').click( function() {
console.log("tagedit");
$('.tagedit').prop("readonly", false);
});
$('.tag_item').submit(function(e) { return false; });
$('.update, .delete').click(function(e) {
var form = $(this).parent("form");
var g = form.serialize() + "&tagid=" + form.data('tagid') + "&action=" + $(this).attr("value");
$.post('/tags', g, function(d) {
if(d === 'ok') {
location.reload();
}
});
});
$('#new').click(function(e) {
var v = $(this).text();
console.log(v)
if(v == "NEW") {
var form = $(this).parent("form");
form.append('<input name="tag" type="text" id="edit-tag" value="...">');
form.append('<textarea name="desc" id="edit-desc">...</textarea>');
$(this).text("ADD");
} else if (v == "ADD") {
if($("#edit-tag").val() === "..." & $("#edit-desc").text() === "...") {
$(this).text("NEW");
$("#edit-tag").remove()
$("#edit-desc").remove()
return;
}
var form = $(this).parent("form");
var g = form.serialize() + "&action=new";
$.post('/tags', g, function(d) {
if(d === 'ok') {
location.reload();
}
});
$(this).text("NEW");
$("#edit-tag").remove()
$("#edit-desc").remove()
}
});
});

26
www/templates/selection_tags.html Normal file
View File

@ -0,0 +1,26 @@
<html>
<head>
<meta charset="utf-8">
<title>Selection [tags]</title>
<link rel="stylesheet" href="{{ url_for('static',filename='lestyle.css') }}">
<script type="text/javascript" src="{{ url_for('static',filename='jquery-3.2.1.min.js') }}" charset="utf-8"></script>
<script type="text/javascript" src="{{ url_for('static',filename='tags.js') }}"></script>
</head>
<body>
<h1>Selection [tags]</h1>
<form class="tag_item">
<button id="new">NEW</button>
</form>
<hr>
<div id="all">
{% for v in tags %}
<form class="tag_item" id="{{v.tag}}" method="post" data-tagid="{{v.tag}}">
<input name="tag" class="tagedit" type="text" value="{{v.tag}}">
<textarea name="desc">{{v.desc}}</textarea>
<input type="submit" class="update" value="update">
<input type="submit" class="delete" value="delete">
</form>
{% endfor %}
</div>
</body>
</html>

34
www/templates/selection_tags_w_lists.html Normal file
View File

@ -0,0 +1,34 @@
<html>
<head>
<meta charset="utf-8">
<title>Selection [tags w lists]</title>
<link rel="stylesheet" href="{{ url_for('static',filename='lestyle.css') }}">
<script type="text/javascript" src="{{ url_for('static',filename='jquery-3.2.1.min.js') }}" charset="utf-8"></script>
<script type="text/javascript" src="{{ url_for('static',filename='tags_w_lists.js') }}"></script>
</head>
<body>
<h1>Selection [tags w lists]</h1>
<hr>
<button id="commit">commit</button>
<div id="all">
{% for v in tags %}
<hr>
<div class="sel">
<form class="tag_item" id="{{v.tag}}" method="post" data-tagid="{{v.tag}}">
<input name="tag" class="tagedit" type="text" value="{{v.tag}}">
<textarea name="desc">{{v.desc}}</textarea>
<input type="submit" class="update" value="update">
<input type="submit" class="delete" value="delete">
</form>
<lists>
<ul>
{% for h in v.lists %}
<li><a href="{{h.url}}" target="_blank">{{h.subject}}</a> -- <i>{{h.author_name}}</i></li>
{% endfor %}
</ul>
</lists>
</div>
{% endfor %}
</div>
</body>
</html>