tm20 selection [ready for rene]
This commit is contained in:
gauthiier
parent
46a0cc8c0b
commit
abea0f0597
0
selection/__init__.py
Normal file
0
selection/__init__.py
Normal file
@ -1,51 +0,0 @@
|
|||||||
import os, json, glob, logging
|
|
||||||
|
|
||||||
ARCH = "archives/"
|
|
||||||
EXP = "selection/"
|
|
||||||
sel = os.path.join(EXP, "tm-selection.js")
|
|
||||||
|
|
||||||
def find(li, url):
|
|
||||||
|
|
||||||
d = os.path.join(ARCH, li)
|
|
||||||
|
|
||||||
if not os.path.isdir(d):
|
|
||||||
logging.warning("Invalid archive path: " + d)
|
|
||||||
return None
|
|
||||||
|
|
||||||
dir_files = [f for f in glob.glob(os.path.join(d, "*.json"))]
|
|
||||||
|
|
||||||
for f in dir_files:
|
|
||||||
with open(f) as fp:
|
|
||||||
dj = json.load(fp)
|
|
||||||
for t in dj['threads']:
|
|
||||||
if t['url'] == url:
|
|
||||||
return t
|
|
||||||
|
|
||||||
return None
|
|
||||||
|
|
||||||
def construct():
|
|
||||||
|
|
||||||
dump = {}
|
|
||||||
|
|
||||||
with open(sel) as f:
|
|
||||||
d = json.load(f)
|
|
||||||
|
|
||||||
for k, v in d.items():
|
|
||||||
dump[k] = []
|
|
||||||
for i in v:
|
|
||||||
m = find(i['list'], i['url'])
|
|
||||||
if m is not None:
|
|
||||||
m['list'] = i['list']
|
|
||||||
dump[k].append(m)
|
|
||||||
|
|
||||||
fout = os.path.join(EXP, "tm-selection-dump.js")
|
|
||||||
|
|
||||||
with open(fout, 'w+', encoding='utf-8') as f:
|
|
||||||
json.dump(dump, f, ensure_ascii=False, indent=4)
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
|
|
||||||
construct()
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
163
selection/sel.py
Normal file
163
selection/sel.py
Normal file
@ -0,0 +1,163 @@
|
|||||||
|
import os, json, glob, logging
|
||||||
|
|
||||||
|
ARCH = "archives/"
|
||||||
|
EXP = "selection/"
|
||||||
|
sel = os.path.join(EXP, "tm-selection.js")
|
||||||
|
sel_dump = os.path.join(EXP, "tm-selection-dump.js")
|
||||||
|
|
||||||
|
with open(sel) as f:
|
||||||
|
d = json.load(f)
|
||||||
|
|
||||||
|
def lists():
|
||||||
|
return os.listdir(sel.ARCH)
|
||||||
|
|
||||||
|
def tags():
|
||||||
|
global d
|
||||||
|
return list(d.keys())
|
||||||
|
|
||||||
|
|
||||||
|
def find(li, url):
|
||||||
|
|
||||||
|
d = os.path.join(ARCH, li)
|
||||||
|
|
||||||
|
if not os.path.isdir(d):
|
||||||
|
logging.warning("Invalid archive path: " + d)
|
||||||
|
print("Invalid archive path: " + d)
|
||||||
|
return None
|
||||||
|
|
||||||
|
dir_files = [f for f in glob.glob(os.path.join(d, "*.json"))]
|
||||||
|
|
||||||
|
for f in dir_files:
|
||||||
|
with open(f) as fp:
|
||||||
|
dj = json.load(fp)
|
||||||
|
for t in dj['threads']:
|
||||||
|
if t['url'] == url: # one level..... not recursive
|
||||||
|
return t
|
||||||
|
return None
|
||||||
|
|
||||||
|
def recursive_urls(msg):
|
||||||
|
r = [msg['url']]
|
||||||
|
if 'follow-up' in list(msg.keys()):
|
||||||
|
for m in msg['follow-up']:
|
||||||
|
r += recursive_urls(m)
|
||||||
|
return r
|
||||||
|
|
||||||
|
def commit_selection(li, url, tag):
|
||||||
|
|
||||||
|
if tag not in list(d.keys()):
|
||||||
|
print("new tag: " + tag)
|
||||||
|
d[tag] = []
|
||||||
|
|
||||||
|
for i in d[tag]:
|
||||||
|
if i['url'] == url:
|
||||||
|
return False
|
||||||
|
|
||||||
|
d[tag].append({'list': li, 'url': url})
|
||||||
|
|
||||||
|
with open(sel, 'w', encoding='utf-8') as f:
|
||||||
|
json.dump(d, f, ensure_ascii=False, indent=4)
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
def commit_dump(li, url, tag):
|
||||||
|
|
||||||
|
if not commit_selection(li, url, tag):
|
||||||
|
return None
|
||||||
|
|
||||||
|
m = find(li, url) # <--- time
|
||||||
|
if m is not None:
|
||||||
|
|
||||||
|
with open(sel_dump) as f:
|
||||||
|
dump = json.load(f)
|
||||||
|
|
||||||
|
if tag not in list(dump.keys()):
|
||||||
|
dump[tag] = []
|
||||||
|
|
||||||
|
dump[tag].append(m)
|
||||||
|
|
||||||
|
with open(sel_dump, 'w+', encoding='utf-8') as fout:
|
||||||
|
json.dump(dump, fout, ensure_ascii=False, indent=4)
|
||||||
|
|
||||||
|
commited = recursive_urls(m)
|
||||||
|
return commited
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def commit_from_selection():
|
||||||
|
|
||||||
|
dump = {}
|
||||||
|
with open(sel) as f:
|
||||||
|
d = json.load(f)
|
||||||
|
|
||||||
|
for k, v in d.items():
|
||||||
|
dump[k] = []
|
||||||
|
for i in v:
|
||||||
|
m = find(i['list'], i['url']) # <--- time
|
||||||
|
if m is not None:
|
||||||
|
m['list'] = i['list']
|
||||||
|
dump[k].append(m)
|
||||||
|
|
||||||
|
with open(sel_dump, 'w+', encoding='utf-8') as f:
|
||||||
|
json.dump(dump, f, ensure_ascii=False, indent=4)
|
||||||
|
|
||||||
|
def report():
|
||||||
|
|
||||||
|
re = "Report: \n"
|
||||||
|
for k, v in d.items():
|
||||||
|
lre = {}
|
||||||
|
for i in v:
|
||||||
|
if i['list'] not in lre:
|
||||||
|
lre[i['list']] = 0
|
||||||
|
lre[i['list']] += 1
|
||||||
|
re += "<" + k + ">: " + str(len(v)) + " ("
|
||||||
|
for kk, vv in lre.items():
|
||||||
|
re += kk + ": " + str(vv) + " / "
|
||||||
|
re += ")\n"
|
||||||
|
|
||||||
|
return re
|
||||||
|
|
||||||
|
def recursive_format(msg):
|
||||||
|
msg.pop('id')
|
||||||
|
msg['len'] = len(msg['content'])
|
||||||
|
msg.pop('content')
|
||||||
|
msg.pop('content-type')
|
||||||
|
if 'to' in msg:
|
||||||
|
msg.pop('to')
|
||||||
|
if 'message-id' in msg:
|
||||||
|
msg.pop('message-id')
|
||||||
|
if 'follow-up' in msg:
|
||||||
|
for i in msg['follow-up']:
|
||||||
|
recursive_format(i)
|
||||||
|
|
||||||
|
def format_selection():
|
||||||
|
|
||||||
|
with open(sel_dump) as f:
|
||||||
|
d = json.load(f)
|
||||||
|
|
||||||
|
for k, v in d.items():
|
||||||
|
for i in v:
|
||||||
|
recursive_format(i)
|
||||||
|
|
||||||
|
return d
|
||||||
|
|
||||||
|
def recursive_hashmap(msg, tag, hm):
|
||||||
|
hm[msg['url']] = tag
|
||||||
|
if 'follow-up' in msg:
|
||||||
|
for i in msg['follow-up']:
|
||||||
|
recursive_hashmap(i, tag, hm)
|
||||||
|
|
||||||
|
def hashmap():
|
||||||
|
with open(sel_dump) as f:
|
||||||
|
d = json.load(f)
|
||||||
|
hm = {}
|
||||||
|
for k, v in d.items():
|
||||||
|
for i in v:
|
||||||
|
recursive_hashmap(i, k, hm)
|
||||||
|
return hm
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
|
||||||
|
d = format_selection()
|
||||||
|
print(json.dumps(d, indent=4, sort_keys=True))
|
||||||
|
|
||||||
125
selection/tm-selection-backup.js
Normal file
125
selection/tm-selection-backup.js
Normal file
@ -0,0 +1,125 @@
|
|||||||
|
{
|
||||||
|
"cyberfeminism": [
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-9706/msg00111.html"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"end2end": [],
|
||||||
|
"net.art": [
|
||||||
|
{
|
||||||
|
"list": "crumb",
|
||||||
|
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=15160"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "crumb",
|
||||||
|
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=40578"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "crumb",
|
||||||
|
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=41637"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "crumb",
|
||||||
|
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=42354"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "crumb",
|
||||||
|
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=43292"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "crumb",
|
||||||
|
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=47543"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "crumb",
|
||||||
|
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=48346"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "crumb",
|
||||||
|
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=54012"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-9708/msg00010.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-9708/msg00009.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-9703/msg00038.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-9703/msg00060.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-9703/msg00096.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-9610/msg00029.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-9705/msg00053.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-9705/msg00003.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-9707/msg00014.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00109.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00117.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00084.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00162.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00153.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00202.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "crumb",
|
||||||
|
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1310&L=new-media-curating&F=&S=&P=2942"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "empyre",
|
||||||
|
"url": "http://lists.cofa.unsw.edu.au/pipermail/empyre/2016-September/009250.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "empyre",
|
||||||
|
"url": "http://lists.cofa.unsw.edu.au/pipermail/empyre/2016-September/009253.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-0009/msg00209.html"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"new media art": [
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-0905/msg00038.html"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
File diff suppressed because one or more lines are too long
@ -1,11 +1,121 @@
|
|||||||
{
|
{
|
||||||
|
"cyberfeminism": [
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-9706/msg00111.html"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"end2end": [],
|
||||||
"net.art": [
|
"net.art": [
|
||||||
{
|
{
|
||||||
"list": "crumb",
|
"list": "crumb",
|
||||||
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=15160"
|
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=15160"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "crumb",
|
||||||
|
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=40578"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "crumb",
|
||||||
|
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=41637"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "crumb",
|
||||||
|
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=42354"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "crumb",
|
||||||
|
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=43292"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "crumb",
|
||||||
|
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=47543"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "crumb",
|
||||||
|
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=48346"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "crumb",
|
||||||
|
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=54012"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-9708/msg00010.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-9708/msg00009.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-9703/msg00038.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-9703/msg00060.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-9703/msg00096.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-9610/msg00029.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-9705/msg00053.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-9705/msg00003.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-9707/msg00014.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00109.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00117.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00084.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00162.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00153.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00202.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "crumb",
|
||||||
|
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1310&L=new-media-curating&F=&S=&P=2942"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "empyre",
|
||||||
|
"url": "http://lists.cofa.unsw.edu.au/pipermail/empyre/2016-September/009250.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "empyre",
|
||||||
|
"url": "http://lists.cofa.unsw.edu.au/pipermail/empyre/2016-September/009253.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"list": "nettime-l",
|
||||||
|
"url": "https://nettime.org/Lists-Archives/nettime-l-0009/msg00209.html"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"end2end": [],
|
|
||||||
"new media art": [
|
"new media art": [
|
||||||
{
|
{
|
||||||
"list": "nettime-l",
|
"list": "nettime-l",
|
||||||
|
|||||||
@ -1,3 +1,3 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
source activate listserv
|
source activate annalen
|
||||||
gunicorn -w 1 --bind 0.0.0.0:6767 www-serve:app
|
gunicorn -w 1 --bind 0.0.0.0:6767 www-serve:app
|
||||||
|
|||||||
@ -1,57 +0,0 @@
|
|||||||
import os, glob, json
|
|
||||||
|
|
||||||
ARCH = "archives/"
|
|
||||||
EXP = "selection/"
|
|
||||||
sel = os.path.join(EXP, "tm-selection.js")
|
|
||||||
|
|
||||||
with open(sel) as f:
|
|
||||||
d = json.load(f)
|
|
||||||
|
|
||||||
def lists():
|
|
||||||
return os.listdir(ARCH)
|
|
||||||
|
|
||||||
def tags():
|
|
||||||
global d
|
|
||||||
return list(d.keys())
|
|
||||||
|
|
||||||
def commit(li, url, tag):
|
|
||||||
global sel
|
|
||||||
|
|
||||||
if tag not in list(d.keys()):
|
|
||||||
print("new tag: " + tag)
|
|
||||||
d[tag] = []
|
|
||||||
|
|
||||||
for i in d[tag]:
|
|
||||||
if i['url'] == url:
|
|
||||||
return False
|
|
||||||
|
|
||||||
d[tag].append({'list': li, 'url': url})
|
|
||||||
|
|
||||||
with open(sel, 'w', encoding='utf-8') as f:
|
|
||||||
json.dump(d, f, ensure_ascii=False, indent=4)
|
|
||||||
|
|
||||||
return True
|
|
||||||
|
|
||||||
def report():
|
|
||||||
|
|
||||||
re = "Report: \n"
|
|
||||||
for k, v in d.items():
|
|
||||||
lre = {}
|
|
||||||
for i in v:
|
|
||||||
if i['list'] not in lre:
|
|
||||||
lre[i['list']] = 0
|
|
||||||
lre[i['list']] += 1
|
|
||||||
re += "<" + k + ">: " + str(len(v)) + " ("
|
|
||||||
for kk, vv in lre.items():
|
|
||||||
re += kk + ": " + str(vv) + " / "
|
|
||||||
re += ")\n"
|
|
||||||
|
|
||||||
return re
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@ -1,21 +1,31 @@
|
|||||||
from flask import render_template, request, jsonify, send_from_directory
|
from flask import render_template, request, jsonify, send_from_directory
|
||||||
from www import app
|
from www import app
|
||||||
import logging
|
import json, logging
|
||||||
from www import archive
|
from selection import sel
|
||||||
|
import urllib.parse
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
LISTSERVS_URL = "http://127.0.0.1:5555"
|
||||||
|
lists_to_serve = []
|
||||||
|
|
||||||
@app.route('/')
|
@app.route('/')
|
||||||
def index():
|
def index():
|
||||||
l = archive.lists()
|
l = sel.lists()
|
||||||
return render_template("index.html", lists=l, tags=archive.tags())
|
return render_template("index.html", lists=l, tags=sel.tags())
|
||||||
|
|
||||||
|
@app.route('/selection')
|
||||||
|
def selection():
|
||||||
|
s = sel.format_selection()
|
||||||
|
return render_template("selection.html", sel=s)
|
||||||
|
|
||||||
|
|
||||||
@app.route('/tags')
|
@app.route('/tags')
|
||||||
def tags():
|
def tags():
|
||||||
return jsonify(result=archive.tags())
|
return jsonify(result=sel.tags())
|
||||||
|
|
||||||
@app.route('/report')
|
@app.route('/report')
|
||||||
def report():
|
def report():
|
||||||
return jsonify(report=archive.report())
|
return jsonify(report=sel.report())
|
||||||
|
|
||||||
# @app.route('/favicon.ico')
|
# @app.route('/favicon.ico')
|
||||||
# def favicon():
|
# def favicon():
|
||||||
@ -29,9 +39,62 @@ def collect():
|
|||||||
li = request.args.get('list')
|
li = request.args.get('list')
|
||||||
|
|
||||||
msg = "Added " + url + " to " + tag
|
msg = "Added " + url + " to " + tag
|
||||||
if not archive.commit(li, url, tag):
|
commited = sel.commit_dump(li, url, tag)
|
||||||
|
if not commited:
|
||||||
msg = url + " already exists..."
|
msg = url + " already exists..."
|
||||||
|
commited = []
|
||||||
|
|
||||||
return jsonify({ "msg": msg, "report": archive.report()})
|
return jsonify({ "msg": msg, "report": sel.report(), "commited": commited, "tag": tag})
|
||||||
|
|
||||||
|
|
||||||
|
def filter_results_selection(res):
|
||||||
|
h = sel.hashmap()
|
||||||
|
for r in res:
|
||||||
|
for rr in r['results']:
|
||||||
|
for rrr in rr['hits']:
|
||||||
|
if rrr['url'] in h:
|
||||||
|
print(rrr['url'] + " in " + h[rrr['url']])
|
||||||
|
rrr['sel'] = h[rrr['url']]
|
||||||
|
|
||||||
|
@app.route('/search')
|
||||||
|
def searh():
|
||||||
|
|
||||||
|
global lists_to_serve
|
||||||
|
|
||||||
|
if len(request.args) < 1:
|
||||||
|
# q: list all table or keep predefined wconfig.lists_to_serve?
|
||||||
|
# wconfig.lists_to_serve = archive.list_tables_db(config.db['database'], config.db['host'], config.db['user'], config.db['password'])
|
||||||
|
r = urllib.request.urlopen(LISTSERVS_URL + '/lists')
|
||||||
|
data = json.loads(r.read().decode('utf8'))
|
||||||
|
lists_to_serve = data['lists']
|
||||||
|
|
||||||
|
return render_template("search.html", archives=lists_to_serve, fields=['content', 'from'])
|
||||||
|
|
||||||
|
k_arg = request.args.get('keyword')
|
||||||
|
l_arg = request.args.get('list')
|
||||||
|
f_arg = request.args.get('field')
|
||||||
|
|
||||||
|
if k_arg is None or k_arg.strip() == '':
|
||||||
|
return "no keyword..."
|
||||||
|
|
||||||
|
if l_arg != "all" and l_arg not in lists_to_serve:
|
||||||
|
return "list '" + l_arg + "' does not exist"
|
||||||
|
|
||||||
|
if f_arg not in ['content', 'from']:
|
||||||
|
return "field '" + f_arg + "' does not exist"
|
||||||
|
|
||||||
|
data = {'keyword': k_arg, 'list': l_arg, 'field': f_arg}
|
||||||
|
val = urllib.parse.urlencode(data)
|
||||||
|
|
||||||
|
URL = LISTSERVS_URL + '/search?' + val
|
||||||
|
|
||||||
|
logging.info("search keyword = " + k_arg)
|
||||||
|
r = urllib.request.urlopen(LISTSERVS_URL + '/search?' + val)
|
||||||
|
data = json.loads(r.read().decode('utf8'))
|
||||||
|
|
||||||
|
filter_results_selection(data['result'])
|
||||||
|
|
||||||
|
return jsonify({'result': data['result'], 'tags': sel.tags()})
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
1
www/static/c3.min.css
vendored
Normal file
1
www/static/c3.min.css
vendored
Normal file
@ -0,0 +1 @@
|
|||||||
|
.c3 svg{font:10px sans-serif;-webkit-tap-highlight-color:transparent}.c3 line,.c3 path{fill:none;stroke:#000}.c3 text{-webkit-user-select:none;-moz-user-select:none;user-select:none}.c3-bars path,.c3-event-rect,.c3-legend-item-tile,.c3-xgrid-focus,.c3-ygrid{shape-rendering:crispEdges}.c3-chart-arc path{stroke:#fff}.c3-chart-arc text{fill:#fff;font-size:13px}.c3-grid line{stroke:#aaa}.c3-grid text{fill:#aaa}.c3-xgrid,.c3-ygrid{stroke-dasharray:3 3}.c3-text.c3-empty{fill:grey;font-size:2em}.c3-line{stroke-width:1px}.c3-circle._expanded_{stroke-width:1px;stroke:#fff}.c3-selected-circle{fill:#fff;stroke-width:2px}.c3-bar{stroke-width:0}.c3-bar._expanded_{fill-opacity:1;fill-opacity:.75}.c3-target.c3-focused{opacity:1}.c3-target.c3-focused path.c3-line,.c3-target.c3-focused path.c3-step{stroke-width:2px}.c3-target.c3-defocused{opacity:.3!important}.c3-region{fill:#4682b4;fill-opacity:.1}.c3-brush .extent{fill-opacity:.1}.c3-legend-item{font-size:12px}.c3-legend-item-hidden{opacity:.15}.c3-legend-background{opacity:.75;fill:#fff;stroke:#d3d3d3;stroke-width:1}.c3-title{font:14px sans-serif}.c3-tooltip-container{z-index:10}.c3-tooltip{border-collapse:collapse;border-spacing:0;background-color:#fff;empty-cells:show;-webkit-box-shadow:7px 7px 12px -9px #777;-moz-box-shadow:7px 7px 12px -9px #777;box-shadow:7px 7px 12px -9px #777;opacity:.9}.c3-tooltip tr{border:1px solid #ccc}.c3-tooltip th{background-color:#aaa;font-size:14px;padding:2px 5px;text-align:left;color:#fff}.c3-tooltip td{font-size:13px;padding:3px 6px;background-color:#fff;border-left:1px dotted #999}.c3-tooltip td>span{display:inline-block;width:10px;height:10px;margin-right:6px}.c3-tooltip td.value{text-align:right}.c3-area{stroke-width:0;opacity:.2}.c3-chart-arcs-title{dominant-baseline:middle;font-size:1.3em}.c3-chart-arcs .c3-chart-arcs-background{fill:#e0e0e0;stroke:none}.c3-chart-arcs .c3-chart-arcs-gauge-unit{fill:#000;font-size:16px}.c3-chart-arcs .c3-chart-arcs-gauge-max{fill:#777}.c3-chart-arcs .c3-chart-arcs-gauge-min{fill:#777}.c3-chart-arc .c3-gauge-value{fill:#000}.c3-chart-arc.c3-target g path{opacity:1}.c3-chart-arc.c3-target.c3-focused g path{opacity:1}
|
||||||
1
www/static/c3.min.js
vendored
Normal file
1
www/static/c3.min.js
vendored
Normal file
File diff suppressed because one or more lines are too long
5
www/static/d3.min.js
vendored
Normal file
5
www/static/d3.min.js
vendored
Normal file
File diff suppressed because one or more lines are too long
BIN
www/static/favicon.ico
Normal file
BIN
www/static/favicon.ico
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 894 B |
@ -13,4 +13,12 @@ input[type="text"] {
|
|||||||
|
|
||||||
button {
|
button {
|
||||||
margin-top: 2em;
|
margin-top: 2em;
|
||||||
|
}
|
||||||
|
|
||||||
|
aaa:hover {
|
||||||
|
background-color: blue;
|
||||||
|
}
|
||||||
|
|
||||||
|
bb {
|
||||||
|
background-color: green;
|
||||||
}
|
}
|
||||||
198
www/static/search.js
Normal file
198
www/static/search.js
Normal file
@ -0,0 +1,198 @@
|
|||||||
|
|
||||||
|
$(document).ready(function(){
|
||||||
|
$('#loading').hide()
|
||||||
|
|
||||||
|
$('#info').click( function() {
|
||||||
|
console.log("click");
|
||||||
|
$('#info-search').toggle();
|
||||||
|
});
|
||||||
|
|
||||||
|
$('#search').submit(function(e) {
|
||||||
|
e.preventDefault();
|
||||||
|
args = $(this).serialize();
|
||||||
|
$('#graph').empty();
|
||||||
|
$('#results').empty();
|
||||||
|
|
||||||
|
console.log('/search?'+ args)
|
||||||
|
|
||||||
|
$('#loading').show()
|
||||||
|
$.get('/search?'+ args, function(data) {
|
||||||
|
$('#loading').hide()
|
||||||
|
console.log(data);
|
||||||
|
// $('#graph').empty();
|
||||||
|
// $('#results').empty();
|
||||||
|
$.each(data.result, function(i, item) {
|
||||||
|
search_result_archive(item, data.tags);
|
||||||
|
});
|
||||||
|
graph(data);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
|
function search_result_archive(a, tags) {
|
||||||
|
|
||||||
|
// aaa = '<aaa class="aaa">[net.art]</aaa> <aaa class="aaa">[net history]</aaa> <aaa class="aaa">[end2end]</aaa>'
|
||||||
|
aaa = ''
|
||||||
|
for(let t of tags) {
|
||||||
|
aaa += '<aaa class="aaa">[' + t + ']</aaa> '
|
||||||
|
}
|
||||||
|
|
||||||
|
$('<div/>', {
|
||||||
|
id: a.archive,
|
||||||
|
class: "archive",
|
||||||
|
}).appendTo('#results');
|
||||||
|
$('#' + a.archive).append("<h3>" + a.archive + "</h3>");
|
||||||
|
$.each(a.results, function(i, r) {
|
||||||
|
$('<ul/>', {
|
||||||
|
id: r.thread + "-" + a.archive,
|
||||||
|
text: r.thread.replace('_', ' ')
|
||||||
|
}).appendTo('#' + a.archive);
|
||||||
|
let hits = "<ul>";
|
||||||
|
|
||||||
|
$.each(r.hits, function(j, h){
|
||||||
|
// console.log(h)
|
||||||
|
let hit = '<li><a href="' + h.url+ '" target="_blank">' + h.subject + '</a> -- <i>' + h.author_name + '</i>';
|
||||||
|
if (h.hasOwnProperty('sel')) {
|
||||||
|
hit += ' <bb class="bb">[' + h.sel + ']</bb>'
|
||||||
|
} else {
|
||||||
|
hit += ' <aa class="aa" data-list="' + a.archive + '" data-url="' + h.url + '"> (+) ' + aaa + '</aa></li>';
|
||||||
|
}
|
||||||
|
hits += hit;
|
||||||
|
});
|
||||||
|
hits += "</ul>";
|
||||||
|
$('#' + r.thread + "-" + a.archive).append(hits);
|
||||||
|
});
|
||||||
|
$(".aa").click(function(e) {
|
||||||
|
e.stopImmediatePropagation();
|
||||||
|
$(this).children().toggle();
|
||||||
|
});
|
||||||
|
$(".aaa").hide();
|
||||||
|
$(".aaa").click(function(e) {
|
||||||
|
e.stopImmediatePropagation();
|
||||||
|
|
||||||
|
let p = $(this).parent();
|
||||||
|
console.log("click : " + p.data("list") + " -- " + p.data("url") + " -- " + $(this).text());
|
||||||
|
tag = $(this).text().replace('[','').replace(']', '');
|
||||||
|
|
||||||
|
// collect (url, tags, list)
|
||||||
|
args = 'url=' + encodeURIComponent(p.data("url")) + '&tags=' + encodeURIComponent(tag) + '&list=' + encodeURIComponent(p.data("list"))
|
||||||
|
|
||||||
|
$.get('/collect?'+ args, function(data) {
|
||||||
|
|
||||||
|
console.log(data)
|
||||||
|
|
||||||
|
$.each(data.commited, function(i, r) {
|
||||||
|
aa = $('aa[data-url="'+r+'"]')
|
||||||
|
p = aa.parent();
|
||||||
|
aa.remove();
|
||||||
|
p.append(' <bb class="bb">[' + data.tag + ']</bb>')
|
||||||
|
});
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
var min_month = new Date(1995, 9);
|
||||||
|
var max_month = new Date();
|
||||||
|
|
||||||
|
function diff_months(d1, d2) {
|
||||||
|
var months;
|
||||||
|
months = (d2.getFullYear() - d1.getFullYear()) * 12;
|
||||||
|
months -= d1.getMonth();
|
||||||
|
months += d2.getMonth();
|
||||||
|
return months <= 0 ? 0 : months;
|
||||||
|
}
|
||||||
|
|
||||||
|
function format(date) {
|
||||||
|
var month_names = [
|
||||||
|
"Jan", "Feb", "Mar",
|
||||||
|
"Apr", "May", "Jun", "Jul",
|
||||||
|
"Aug", "Sep", "Oct",
|
||||||
|
"Nov", "Dec"
|
||||||
|
];
|
||||||
|
return month_names[date.getMonth()] + ' ' + date.getFullYear();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
function graph(data) {
|
||||||
|
var d = diff_months(min_month, max_month);
|
||||||
|
var vec = new Array();
|
||||||
|
for(let ar of data.result) {
|
||||||
|
let ar_vec = new Array(d + 1).fill(0);
|
||||||
|
ar_vec[0] = ar.archive;
|
||||||
|
for(let r of ar.results) {
|
||||||
|
let date = new Date(Date.parse(r.thread.replace("_", " 1, "))); // this may blow...
|
||||||
|
let index = diff_months(min_month, date);
|
||||||
|
ar_vec[index + 1] = r.nbr_hits;
|
||||||
|
}
|
||||||
|
vec.push(ar_vec);
|
||||||
|
}
|
||||||
|
|
||||||
|
var x_axis = new Array(d);
|
||||||
|
for (let i = 0; i < d; i++) {
|
||||||
|
let d = new Date(min_month.getFullYear(), min_month.getMonth());
|
||||||
|
d.setMonth(d.getMonth() + i);
|
||||||
|
x_axis[i] = format(d);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// console.log(vec);
|
||||||
|
|
||||||
|
var chart = c3.generate({
|
||||||
|
bindto: '#graph',
|
||||||
|
data: {
|
||||||
|
columns: vec,
|
||||||
|
type: 'bar'
|
||||||
|
},
|
||||||
|
axis: {
|
||||||
|
x: {
|
||||||
|
type: 'category',
|
||||||
|
categories: x_axis,
|
||||||
|
tick: {
|
||||||
|
culling: {
|
||||||
|
max: 15
|
||||||
|
},
|
||||||
|
multiline:false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
bar: {
|
||||||
|
width: {
|
||||||
|
ratio: 0.8
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
44
www/templates/search.html
Normal file
44
www/templates/search.html
Normal file
@ -0,0 +1,44 @@
|
|||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<title>Times of Nettime</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="{{ url_for('static',filename='c3.min.css') }}">
|
||||||
|
<link rel="stylesheet" type="text/css" href="{{ url_for('static',filename='lestyle.css') }}">
|
||||||
|
<script type=text/javascript src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.2.1/jquery.min.js"></script>
|
||||||
|
<script type="text/javascript" src="{{ url_for('static',filename='d3.min.js') }}" charset="utf-8"></script>
|
||||||
|
<script type="text/javascript" src="{{ url_for('static',filename='c3.min.js') }}"></script>
|
||||||
|
<script type=text/javascript src="{{ url_for('static',filename='search.js') }}"></script>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<form action="/search" method="get" id="search">
|
||||||
|
<label>keyword: </label><input type="search" name="keyword">
|
||||||
|
<select form="search" name="list">
|
||||||
|
<option value="all">all</option>
|
||||||
|
{% for a in archives %}
|
||||||
|
<option value="{{ a }}">{{ a }}</option>
|
||||||
|
{% endfor %}
|
||||||
|
</select>
|
||||||
|
<select form="search" name="field">
|
||||||
|
{% for a in fields %}
|
||||||
|
<option value="{{ a }}">{{ a }}</option>
|
||||||
|
{% endfor %}
|
||||||
|
</select>
|
||||||
|
<input type="submit" value="search" id="submit">
|
||||||
|
<input type="button" value=" ? " id="info">
|
||||||
|
<div id="loading">Loading...</div>
|
||||||
|
</form>
|
||||||
|
<div id="info-search" style="display: none">
|
||||||
|
<table><tbody><tr><th>Operator</th><th> </th></tr>
|
||||||
|
<tr><td>+</td><td>The word is mandatory in all text returned.</td></tr>
|
||||||
|
<tr><td>-</td><td>The word cannot appear in any text returned.</td></tr>
|
||||||
|
<tr><td><</td><td>The word that follows has a lower relevance than other words, although text containing it will still match</td></tr>
|
||||||
|
<tr><td>></td><td>The word that follows has a higher relevance than other words.</td></tr>
|
||||||
|
<tr><td>()</td><td>Used to group words into subexpressions.</td></tr>
|
||||||
|
<tr><td>~</td><td>The word following contributes negatively to the relevance of the text (which is different to the '-' operator, which specifically excludes the word, or the '<' operator, which still causes the word to contribute positively to the relevance of the text.</td></tr>
|
||||||
|
<tr><td>*</td><td>The wildcard, indicating zero or more characters. It can only appear at the end of a word.</td></tr>
|
||||||
|
<tr><td>"</td><td>Anything enclosed in the double quotes is taken as a whole (so you can match phrases, for example).</td></tr>
|
||||||
|
</tbody></table>
|
||||||
|
</div>
|
||||||
|
<div id="graph"></div>
|
||||||
|
<div id="results"></div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
24
www/templates/selection.html
Normal file
24
www/templates/selection.html
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<title>Selection</title>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
{% for k, v in sel.items() %}
|
||||||
|
<b>{{k}}</b>
|
||||||
|
<ul>
|
||||||
|
{% for h in v %}
|
||||||
|
<li><a href="{{h.url}}" target="_blank">{{h.subject}}</a> -- <i>{{h.author_name}}</i> ({{h.len}} chars)</li>
|
||||||
|
{% if 'follow-up' in h %}
|
||||||
|
<ul>
|
||||||
|
{% for f in h['follow-up'] %}
|
||||||
|
<li><a href="{{f.url}}" target="_blank">{{f.subject}}</a> -- <i>{{f.author_name}}</i> ({{f.len}} chars)</li>
|
||||||
|
{% endfor %}
|
||||||
|
</ul>
|
||||||
|
{% endif %}
|
||||||
|
{% endfor %}
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
{% endfor %}
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
Loading…
x
Reference in New Issue
Block a user