tm20 selection [ready for rene]
This commit is contained in:
gauthiier
parent
46a0cc8c0b
commit
abea0f0597
0
selection/__init__.py
Normal file
0
selection/__init__.py
Normal file
@ -1,51 +0,0 @@
|
||||
import os, json, glob, logging
|
||||
|
||||
ARCH = "archives/"
|
||||
EXP = "selection/"
|
||||
sel = os.path.join(EXP, "tm-selection.js")
|
||||
|
||||
def find(li, url):
|
||||
|
||||
d = os.path.join(ARCH, li)
|
||||
|
||||
if not os.path.isdir(d):
|
||||
logging.warning("Invalid archive path: " + d)
|
||||
return None
|
||||
|
||||
dir_files = [f for f in glob.glob(os.path.join(d, "*.json"))]
|
||||
|
||||
for f in dir_files:
|
||||
with open(f) as fp:
|
||||
dj = json.load(fp)
|
||||
for t in dj['threads']:
|
||||
if t['url'] == url:
|
||||
return t
|
||||
|
||||
return None
|
||||
|
||||
def construct():
|
||||
|
||||
dump = {}
|
||||
|
||||
with open(sel) as f:
|
||||
d = json.load(f)
|
||||
|
||||
for k, v in d.items():
|
||||
dump[k] = []
|
||||
for i in v:
|
||||
m = find(i['list'], i['url'])
|
||||
if m is not None:
|
||||
m['list'] = i['list']
|
||||
dump[k].append(m)
|
||||
|
||||
fout = os.path.join(EXP, "tm-selection-dump.js")
|
||||
|
||||
with open(fout, 'w+', encoding='utf-8') as f:
|
||||
json.dump(dump, f, ensure_ascii=False, indent=4)
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
construct()
|
||||
|
||||
|
||||
|
||||
163
selection/sel.py
Normal file
163
selection/sel.py
Normal file
@ -0,0 +1,163 @@
|
||||
import os, json, glob, logging
|
||||
|
||||
ARCH = "archives/"
|
||||
EXP = "selection/"
|
||||
sel = os.path.join(EXP, "tm-selection.js")
|
||||
sel_dump = os.path.join(EXP, "tm-selection-dump.js")
|
||||
|
||||
with open(sel) as f:
|
||||
d = json.load(f)
|
||||
|
||||
def lists():
|
||||
return os.listdir(sel.ARCH)
|
||||
|
||||
def tags():
|
||||
global d
|
||||
return list(d.keys())
|
||||
|
||||
|
||||
def find(li, url):
|
||||
|
||||
d = os.path.join(ARCH, li)
|
||||
|
||||
if not os.path.isdir(d):
|
||||
logging.warning("Invalid archive path: " + d)
|
||||
print("Invalid archive path: " + d)
|
||||
return None
|
||||
|
||||
dir_files = [f for f in glob.glob(os.path.join(d, "*.json"))]
|
||||
|
||||
for f in dir_files:
|
||||
with open(f) as fp:
|
||||
dj = json.load(fp)
|
||||
for t in dj['threads']:
|
||||
if t['url'] == url: # one level..... not recursive
|
||||
return t
|
||||
return None
|
||||
|
||||
def recursive_urls(msg):
|
||||
r = [msg['url']]
|
||||
if 'follow-up' in list(msg.keys()):
|
||||
for m in msg['follow-up']:
|
||||
r += recursive_urls(m)
|
||||
return r
|
||||
|
||||
def commit_selection(li, url, tag):
|
||||
|
||||
if tag not in list(d.keys()):
|
||||
print("new tag: " + tag)
|
||||
d[tag] = []
|
||||
|
||||
for i in d[tag]:
|
||||
if i['url'] == url:
|
||||
return False
|
||||
|
||||
d[tag].append({'list': li, 'url': url})
|
||||
|
||||
with open(sel, 'w', encoding='utf-8') as f:
|
||||
json.dump(d, f, ensure_ascii=False, indent=4)
|
||||
|
||||
return True
|
||||
|
||||
def commit_dump(li, url, tag):
|
||||
|
||||
if not commit_selection(li, url, tag):
|
||||
return None
|
||||
|
||||
m = find(li, url) # <--- time
|
||||
if m is not None:
|
||||
|
||||
with open(sel_dump) as f:
|
||||
dump = json.load(f)
|
||||
|
||||
if tag not in list(dump.keys()):
|
||||
dump[tag] = []
|
||||
|
||||
dump[tag].append(m)
|
||||
|
||||
with open(sel_dump, 'w+', encoding='utf-8') as fout:
|
||||
json.dump(dump, fout, ensure_ascii=False, indent=4)
|
||||
|
||||
commited = recursive_urls(m)
|
||||
return commited
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def commit_from_selection():
|
||||
|
||||
dump = {}
|
||||
with open(sel) as f:
|
||||
d = json.load(f)
|
||||
|
||||
for k, v in d.items():
|
||||
dump[k] = []
|
||||
for i in v:
|
||||
m = find(i['list'], i['url']) # <--- time
|
||||
if m is not None:
|
||||
m['list'] = i['list']
|
||||
dump[k].append(m)
|
||||
|
||||
with open(sel_dump, 'w+', encoding='utf-8') as f:
|
||||
json.dump(dump, f, ensure_ascii=False, indent=4)
|
||||
|
||||
def report():
|
||||
|
||||
re = "Report: \n"
|
||||
for k, v in d.items():
|
||||
lre = {}
|
||||
for i in v:
|
||||
if i['list'] not in lre:
|
||||
lre[i['list']] = 0
|
||||
lre[i['list']] += 1
|
||||
re += "<" + k + ">: " + str(len(v)) + " ("
|
||||
for kk, vv in lre.items():
|
||||
re += kk + ": " + str(vv) + " / "
|
||||
re += ")\n"
|
||||
|
||||
return re
|
||||
|
||||
def recursive_format(msg):
|
||||
msg.pop('id')
|
||||
msg['len'] = len(msg['content'])
|
||||
msg.pop('content')
|
||||
msg.pop('content-type')
|
||||
if 'to' in msg:
|
||||
msg.pop('to')
|
||||
if 'message-id' in msg:
|
||||
msg.pop('message-id')
|
||||
if 'follow-up' in msg:
|
||||
for i in msg['follow-up']:
|
||||
recursive_format(i)
|
||||
|
||||
def format_selection():
|
||||
|
||||
with open(sel_dump) as f:
|
||||
d = json.load(f)
|
||||
|
||||
for k, v in d.items():
|
||||
for i in v:
|
||||
recursive_format(i)
|
||||
|
||||
return d
|
||||
|
||||
def recursive_hashmap(msg, tag, hm):
|
||||
hm[msg['url']] = tag
|
||||
if 'follow-up' in msg:
|
||||
for i in msg['follow-up']:
|
||||
recursive_hashmap(i, tag, hm)
|
||||
|
||||
def hashmap():
|
||||
with open(sel_dump) as f:
|
||||
d = json.load(f)
|
||||
hm = {}
|
||||
for k, v in d.items():
|
||||
for i in v:
|
||||
recursive_hashmap(i, k, hm)
|
||||
return hm
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
d = format_selection()
|
||||
print(json.dumps(d, indent=4, sort_keys=True))
|
||||
|
||||
125
selection/tm-selection-backup.js
Normal file
125
selection/tm-selection-backup.js
Normal file
@ -0,0 +1,125 @@
|
||||
{
|
||||
"cyberfeminism": [
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-9706/msg00111.html"
|
||||
}
|
||||
],
|
||||
"end2end": [],
|
||||
"net.art": [
|
||||
{
|
||||
"list": "crumb",
|
||||
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=15160"
|
||||
},
|
||||
{
|
||||
"list": "crumb",
|
||||
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=40578"
|
||||
},
|
||||
{
|
||||
"list": "crumb",
|
||||
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=41637"
|
||||
},
|
||||
{
|
||||
"list": "crumb",
|
||||
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=42354"
|
||||
},
|
||||
{
|
||||
"list": "crumb",
|
||||
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=43292"
|
||||
},
|
||||
{
|
||||
"list": "crumb",
|
||||
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=47543"
|
||||
},
|
||||
{
|
||||
"list": "crumb",
|
||||
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=48346"
|
||||
},
|
||||
{
|
||||
"list": "crumb",
|
||||
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=54012"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-9708/msg00010.html"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-9708/msg00009.html"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-9703/msg00038.html"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-9703/msg00060.html"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-9703/msg00096.html"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-9610/msg00029.html"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-9705/msg00053.html"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-9705/msg00003.html"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-9707/msg00014.html"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00109.html"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00117.html"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00084.html"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00162.html"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00153.html"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00202.html"
|
||||
},
|
||||
{
|
||||
"list": "crumb",
|
||||
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1310&L=new-media-curating&F=&S=&P=2942"
|
||||
},
|
||||
{
|
||||
"list": "empyre",
|
||||
"url": "http://lists.cofa.unsw.edu.au/pipermail/empyre/2016-September/009250.html"
|
||||
},
|
||||
{
|
||||
"list": "empyre",
|
||||
"url": "http://lists.cofa.unsw.edu.au/pipermail/empyre/2016-September/009253.html"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-0009/msg00209.html"
|
||||
}
|
||||
],
|
||||
"new media art": [
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-0905/msg00038.html"
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@ -1,11 +1,121 @@
|
||||
{
|
||||
"cyberfeminism": [
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-9706/msg00111.html"
|
||||
}
|
||||
],
|
||||
"end2end": [],
|
||||
"net.art": [
|
||||
{
|
||||
"list": "crumb",
|
||||
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=15160"
|
||||
},
|
||||
{
|
||||
"list": "crumb",
|
||||
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=40578"
|
||||
},
|
||||
{
|
||||
"list": "crumb",
|
||||
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=41637"
|
||||
},
|
||||
{
|
||||
"list": "crumb",
|
||||
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=42354"
|
||||
},
|
||||
{
|
||||
"list": "crumb",
|
||||
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=43292"
|
||||
},
|
||||
{
|
||||
"list": "crumb",
|
||||
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=47543"
|
||||
},
|
||||
{
|
||||
"list": "crumb",
|
||||
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=48346"
|
||||
},
|
||||
{
|
||||
"list": "crumb",
|
||||
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=54012"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-9708/msg00010.html"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-9708/msg00009.html"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-9703/msg00038.html"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-9703/msg00060.html"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-9703/msg00096.html"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-9610/msg00029.html"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-9705/msg00053.html"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-9705/msg00003.html"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-9707/msg00014.html"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00109.html"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00117.html"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00084.html"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00162.html"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00153.html"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00202.html"
|
||||
},
|
||||
{
|
||||
"list": "crumb",
|
||||
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1310&L=new-media-curating&F=&S=&P=2942"
|
||||
},
|
||||
{
|
||||
"list": "empyre",
|
||||
"url": "http://lists.cofa.unsw.edu.au/pipermail/empyre/2016-September/009250.html"
|
||||
},
|
||||
{
|
||||
"list": "empyre",
|
||||
"url": "http://lists.cofa.unsw.edu.au/pipermail/empyre/2016-September/009253.html"
|
||||
},
|
||||
{
|
||||
"list": "nettime-l",
|
||||
"url": "https://nettime.org/Lists-Archives/nettime-l-0009/msg00209.html"
|
||||
}
|
||||
],
|
||||
"end2end": [],
|
||||
"new media art": [
|
||||
{
|
||||
"list": "nettime-l",
|
||||
|
||||
@ -1,3 +1,3 @@
|
||||
#!/bin/bash
|
||||
source activate listserv
|
||||
source activate annalen
|
||||
gunicorn -w 1 --bind 0.0.0.0:6767 www-serve:app
|
||||
|
||||
@ -1,57 +0,0 @@
|
||||
import os, glob, json
|
||||
|
||||
ARCH = "archives/"
|
||||
EXP = "selection/"
|
||||
sel = os.path.join(EXP, "tm-selection.js")
|
||||
|
||||
with open(sel) as f:
|
||||
d = json.load(f)
|
||||
|
||||
def lists():
|
||||
return os.listdir(ARCH)
|
||||
|
||||
def tags():
|
||||
global d
|
||||
return list(d.keys())
|
||||
|
||||
def commit(li, url, tag):
|
||||
global sel
|
||||
|
||||
if tag not in list(d.keys()):
|
||||
print("new tag: " + tag)
|
||||
d[tag] = []
|
||||
|
||||
for i in d[tag]:
|
||||
if i['url'] == url:
|
||||
return False
|
||||
|
||||
d[tag].append({'list': li, 'url': url})
|
||||
|
||||
with open(sel, 'w', encoding='utf-8') as f:
|
||||
json.dump(d, f, ensure_ascii=False, indent=4)
|
||||
|
||||
return True
|
||||
|
||||
def report():
|
||||
|
||||
re = "Report: \n"
|
||||
for k, v in d.items():
|
||||
lre = {}
|
||||
for i in v:
|
||||
if i['list'] not in lre:
|
||||
lre[i['list']] = 0
|
||||
lre[i['list']] += 1
|
||||
re += "<" + k + ">: " + str(len(v)) + " ("
|
||||
for kk, vv in lre.items():
|
||||
re += kk + ": " + str(vv) + " / "
|
||||
re += ")\n"
|
||||
|
||||
return re
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@ -1,21 +1,31 @@
|
||||
from flask import render_template, request, jsonify, send_from_directory
|
||||
from www import app
|
||||
import logging
|
||||
from www import archive
|
||||
import json, logging
|
||||
from selection import sel
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
LISTSERVS_URL = "http://127.0.0.1:5555"
|
||||
lists_to_serve = []
|
||||
|
||||
@app.route('/')
|
||||
def index():
|
||||
l = archive.lists()
|
||||
return render_template("index.html", lists=l, tags=archive.tags())
|
||||
l = sel.lists()
|
||||
return render_template("index.html", lists=l, tags=sel.tags())
|
||||
|
||||
@app.route('/selection')
|
||||
def selection():
|
||||
s = sel.format_selection()
|
||||
return render_template("selection.html", sel=s)
|
||||
|
||||
|
||||
@app.route('/tags')
|
||||
def tags():
|
||||
return jsonify(result=archive.tags())
|
||||
return jsonify(result=sel.tags())
|
||||
|
||||
@app.route('/report')
|
||||
def report():
|
||||
return jsonify(report=archive.report())
|
||||
return jsonify(report=sel.report())
|
||||
|
||||
# @app.route('/favicon.ico')
|
||||
# def favicon():
|
||||
@ -29,9 +39,62 @@ def collect():
|
||||
li = request.args.get('list')
|
||||
|
||||
msg = "Added " + url + " to " + tag
|
||||
if not archive.commit(li, url, tag):
|
||||
commited = sel.commit_dump(li, url, tag)
|
||||
if not commited:
|
||||
msg = url + " already exists..."
|
||||
commited = []
|
||||
|
||||
return jsonify({ "msg": msg, "report": sel.report(), "commited": commited, "tag": tag})
|
||||
|
||||
|
||||
def filter_results_selection(res):
|
||||
h = sel.hashmap()
|
||||
for r in res:
|
||||
for rr in r['results']:
|
||||
for rrr in rr['hits']:
|
||||
if rrr['url'] in h:
|
||||
print(rrr['url'] + " in " + h[rrr['url']])
|
||||
rrr['sel'] = h[rrr['url']]
|
||||
|
||||
@app.route('/search')
|
||||
def searh():
|
||||
|
||||
global lists_to_serve
|
||||
|
||||
if len(request.args) < 1:
|
||||
# q: list all table or keep predefined wconfig.lists_to_serve?
|
||||
# wconfig.lists_to_serve = archive.list_tables_db(config.db['database'], config.db['host'], config.db['user'], config.db['password'])
|
||||
r = urllib.request.urlopen(LISTSERVS_URL + '/lists')
|
||||
data = json.loads(r.read().decode('utf8'))
|
||||
lists_to_serve = data['lists']
|
||||
|
||||
return render_template("search.html", archives=lists_to_serve, fields=['content', 'from'])
|
||||
|
||||
k_arg = request.args.get('keyword')
|
||||
l_arg = request.args.get('list')
|
||||
f_arg = request.args.get('field')
|
||||
|
||||
if k_arg is None or k_arg.strip() == '':
|
||||
return "no keyword..."
|
||||
|
||||
if l_arg != "all" and l_arg not in lists_to_serve:
|
||||
return "list '" + l_arg + "' does not exist"
|
||||
|
||||
if f_arg not in ['content', 'from']:
|
||||
return "field '" + f_arg + "' does not exist"
|
||||
|
||||
data = {'keyword': k_arg, 'list': l_arg, 'field': f_arg}
|
||||
val = urllib.parse.urlencode(data)
|
||||
|
||||
URL = LISTSERVS_URL + '/search?' + val
|
||||
|
||||
logging.info("search keyword = " + k_arg)
|
||||
r = urllib.request.urlopen(LISTSERVS_URL + '/search?' + val)
|
||||
data = json.loads(r.read().decode('utf8'))
|
||||
|
||||
filter_results_selection(data['result'])
|
||||
|
||||
return jsonify({'result': data['result'], 'tags': sel.tags()})
|
||||
|
||||
return jsonify({ "msg": msg, "report": archive.report()})
|
||||
|
||||
|
||||
|
||||
1
www/static/c3.min.css
vendored
Normal file
1
www/static/c3.min.css
vendored
Normal file
@ -0,0 +1 @@
|
||||
.c3 svg{font:10px sans-serif;-webkit-tap-highlight-color:transparent}.c3 line,.c3 path{fill:none;stroke:#000}.c3 text{-webkit-user-select:none;-moz-user-select:none;user-select:none}.c3-bars path,.c3-event-rect,.c3-legend-item-tile,.c3-xgrid-focus,.c3-ygrid{shape-rendering:crispEdges}.c3-chart-arc path{stroke:#fff}.c3-chart-arc text{fill:#fff;font-size:13px}.c3-grid line{stroke:#aaa}.c3-grid text{fill:#aaa}.c3-xgrid,.c3-ygrid{stroke-dasharray:3 3}.c3-text.c3-empty{fill:grey;font-size:2em}.c3-line{stroke-width:1px}.c3-circle._expanded_{stroke-width:1px;stroke:#fff}.c3-selected-circle{fill:#fff;stroke-width:2px}.c3-bar{stroke-width:0}.c3-bar._expanded_{fill-opacity:1;fill-opacity:.75}.c3-target.c3-focused{opacity:1}.c3-target.c3-focused path.c3-line,.c3-target.c3-focused path.c3-step{stroke-width:2px}.c3-target.c3-defocused{opacity:.3!important}.c3-region{fill:#4682b4;fill-opacity:.1}.c3-brush .extent{fill-opacity:.1}.c3-legend-item{font-size:12px}.c3-legend-item-hidden{opacity:.15}.c3-legend-background{opacity:.75;fill:#fff;stroke:#d3d3d3;stroke-width:1}.c3-title{font:14px sans-serif}.c3-tooltip-container{z-index:10}.c3-tooltip{border-collapse:collapse;border-spacing:0;background-color:#fff;empty-cells:show;-webkit-box-shadow:7px 7px 12px -9px #777;-moz-box-shadow:7px 7px 12px -9px #777;box-shadow:7px 7px 12px -9px #777;opacity:.9}.c3-tooltip tr{border:1px solid #ccc}.c3-tooltip th{background-color:#aaa;font-size:14px;padding:2px 5px;text-align:left;color:#fff}.c3-tooltip td{font-size:13px;padding:3px 6px;background-color:#fff;border-left:1px dotted #999}.c3-tooltip td>span{display:inline-block;width:10px;height:10px;margin-right:6px}.c3-tooltip td.value{text-align:right}.c3-area{stroke-width:0;opacity:.2}.c3-chart-arcs-title{dominant-baseline:middle;font-size:1.3em}.c3-chart-arcs .c3-chart-arcs-background{fill:#e0e0e0;stroke:none}.c3-chart-arcs .c3-chart-arcs-gauge-unit{fill:#000;font-size:16px}.c3-chart-arcs .c3-chart-arcs-gauge-max{fill:#777}.c3-chart-arcs .c3-chart-arcs-gauge-min{fill:#777}.c3-chart-arc .c3-gauge-value{fill:#000}.c3-chart-arc.c3-target g path{opacity:1}.c3-chart-arc.c3-target.c3-focused g path{opacity:1}
|
||||
1
www/static/c3.min.js
vendored
Normal file
1
www/static/c3.min.js
vendored
Normal file
File diff suppressed because one or more lines are too long
5
www/static/d3.min.js
vendored
Normal file
5
www/static/d3.min.js
vendored
Normal file
File diff suppressed because one or more lines are too long
BIN
www/static/favicon.ico
Normal file
BIN
www/static/favicon.ico
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 894 B |
@ -14,3 +14,11 @@ input[type="text"] {
|
||||
button {
|
||||
margin-top: 2em;
|
||||
}
|
||||
|
||||
aaa:hover {
|
||||
background-color: blue;
|
||||
}
|
||||
|
||||
bb {
|
||||
background-color: green;
|
||||
}
|
||||
198
www/static/search.js
Normal file
198
www/static/search.js
Normal file
@ -0,0 +1,198 @@
|
||||
|
||||
$(document).ready(function(){
|
||||
$('#loading').hide()
|
||||
|
||||
$('#info').click( function() {
|
||||
console.log("click");
|
||||
$('#info-search').toggle();
|
||||
});
|
||||
|
||||
$('#search').submit(function(e) {
|
||||
e.preventDefault();
|
||||
args = $(this).serialize();
|
||||
$('#graph').empty();
|
||||
$('#results').empty();
|
||||
|
||||
console.log('/search?'+ args)
|
||||
|
||||
$('#loading').show()
|
||||
$.get('/search?'+ args, function(data) {
|
||||
$('#loading').hide()
|
||||
console.log(data);
|
||||
// $('#graph').empty();
|
||||
// $('#results').empty();
|
||||
$.each(data.result, function(i, item) {
|
||||
search_result_archive(item, data.tags);
|
||||
});
|
||||
graph(data);
|
||||
});
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
function search_result_archive(a, tags) {
|
||||
|
||||
// aaa = '<aaa class="aaa">[net.art]</aaa> <aaa class="aaa">[net history]</aaa> <aaa class="aaa">[end2end]</aaa>'
|
||||
aaa = ''
|
||||
for(let t of tags) {
|
||||
aaa += '<aaa class="aaa">[' + t + ']</aaa> '
|
||||
}
|
||||
|
||||
$('<div/>', {
|
||||
id: a.archive,
|
||||
class: "archive",
|
||||
}).appendTo('#results');
|
||||
$('#' + a.archive).append("<h3>" + a.archive + "</h3>");
|
||||
$.each(a.results, function(i, r) {
|
||||
$('<ul/>', {
|
||||
id: r.thread + "-" + a.archive,
|
||||
text: r.thread.replace('_', ' ')
|
||||
}).appendTo('#' + a.archive);
|
||||
let hits = "<ul>";
|
||||
|
||||
$.each(r.hits, function(j, h){
|
||||
// console.log(h)
|
||||
let hit = '<li><a href="' + h.url+ '" target="_blank">' + h.subject + '</a> -- <i>' + h.author_name + '</i>';
|
||||
if (h.hasOwnProperty('sel')) {
|
||||
hit += ' <bb class="bb">[' + h.sel + ']</bb>'
|
||||
} else {
|
||||
hit += ' <aa class="aa" data-list="' + a.archive + '" data-url="' + h.url + '"> (+) ' + aaa + '</aa></li>';
|
||||
}
|
||||
hits += hit;
|
||||
});
|
||||
hits += "</ul>";
|
||||
$('#' + r.thread + "-" + a.archive).append(hits);
|
||||
});
|
||||
$(".aa").click(function(e) {
|
||||
e.stopImmediatePropagation();
|
||||
$(this).children().toggle();
|
||||
});
|
||||
$(".aaa").hide();
|
||||
$(".aaa").click(function(e) {
|
||||
e.stopImmediatePropagation();
|
||||
|
||||
let p = $(this).parent();
|
||||
console.log("click : " + p.data("list") + " -- " + p.data("url") + " -- " + $(this).text());
|
||||
tag = $(this).text().replace('[','').replace(']', '');
|
||||
|
||||
// collect (url, tags, list)
|
||||
args = 'url=' + encodeURIComponent(p.data("url")) + '&tags=' + encodeURIComponent(tag) + '&list=' + encodeURIComponent(p.data("list"))
|
||||
|
||||
$.get('/collect?'+ args, function(data) {
|
||||
|
||||
console.log(data)
|
||||
|
||||
$.each(data.commited, function(i, r) {
|
||||
aa = $('aa[data-url="'+r+'"]')
|
||||
p = aa.parent();
|
||||
aa.remove();
|
||||
p.append(' <bb class="bb">[' + data.tag + ']</bb>')
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
|
||||
}
|
||||
|
||||
var min_month = new Date(1995, 9);
|
||||
var max_month = new Date();
|
||||
|
||||
function diff_months(d1, d2) {
|
||||
var months;
|
||||
months = (d2.getFullYear() - d1.getFullYear()) * 12;
|
||||
months -= d1.getMonth();
|
||||
months += d2.getMonth();
|
||||
return months <= 0 ? 0 : months;
|
||||
}
|
||||
|
||||
function format(date) {
|
||||
var month_names = [
|
||||
"Jan", "Feb", "Mar",
|
||||
"Apr", "May", "Jun", "Jul",
|
||||
"Aug", "Sep", "Oct",
|
||||
"Nov", "Dec"
|
||||
];
|
||||
return month_names[date.getMonth()] + ' ' + date.getFullYear();
|
||||
}
|
||||
|
||||
|
||||
function graph(data) {
|
||||
var d = diff_months(min_month, max_month);
|
||||
var vec = new Array();
|
||||
for(let ar of data.result) {
|
||||
let ar_vec = new Array(d + 1).fill(0);
|
||||
ar_vec[0] = ar.archive;
|
||||
for(let r of ar.results) {
|
||||
let date = new Date(Date.parse(r.thread.replace("_", " 1, "))); // this may blow...
|
||||
let index = diff_months(min_month, date);
|
||||
ar_vec[index + 1] = r.nbr_hits;
|
||||
}
|
||||
vec.push(ar_vec);
|
||||
}
|
||||
|
||||
var x_axis = new Array(d);
|
||||
for (let i = 0; i < d; i++) {
|
||||
let d = new Date(min_month.getFullYear(), min_month.getMonth());
|
||||
d.setMonth(d.getMonth() + i);
|
||||
x_axis[i] = format(d);
|
||||
}
|
||||
|
||||
|
||||
// console.log(vec);
|
||||
|
||||
var chart = c3.generate({
|
||||
bindto: '#graph',
|
||||
data: {
|
||||
columns: vec,
|
||||
type: 'bar'
|
||||
},
|
||||
axis: {
|
||||
x: {
|
||||
type: 'category',
|
||||
categories: x_axis,
|
||||
tick: {
|
||||
culling: {
|
||||
max: 15
|
||||
},
|
||||
multiline:false
|
||||
}
|
||||
}
|
||||
},
|
||||
bar: {
|
||||
width: {
|
||||
ratio: 0.8
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
44
www/templates/search.html
Normal file
44
www/templates/search.html
Normal file
@ -0,0 +1,44 @@
|
||||
<html>
|
||||
<head>
|
||||
<title>Times of Nettime</title>
|
||||
<link rel="stylesheet" type="text/css" href="{{ url_for('static',filename='c3.min.css') }}">
|
||||
<link rel="stylesheet" type="text/css" href="{{ url_for('static',filename='lestyle.css') }}">
|
||||
<script type=text/javascript src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.2.1/jquery.min.js"></script>
|
||||
<script type="text/javascript" src="{{ url_for('static',filename='d3.min.js') }}" charset="utf-8"></script>
|
||||
<script type="text/javascript" src="{{ url_for('static',filename='c3.min.js') }}"></script>
|
||||
<script type=text/javascript src="{{ url_for('static',filename='search.js') }}"></script>
|
||||
</head>
|
||||
<body>
|
||||
<form action="/search" method="get" id="search">
|
||||
<label>keyword: </label><input type="search" name="keyword">
|
||||
<select form="search" name="list">
|
||||
<option value="all">all</option>
|
||||
{% for a in archives %}
|
||||
<option value="{{ a }}">{{ a }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
<select form="search" name="field">
|
||||
{% for a in fields %}
|
||||
<option value="{{ a }}">{{ a }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
<input type="submit" value="search" id="submit">
|
||||
<input type="button" value=" ? " id="info">
|
||||
<div id="loading">Loading...</div>
|
||||
</form>
|
||||
<div id="info-search" style="display: none">
|
||||
<table><tbody><tr><th>Operator</th><th> </th></tr>
|
||||
<tr><td>+</td><td>The word is mandatory in all text returned.</td></tr>
|
||||
<tr><td>-</td><td>The word cannot appear in any text returned.</td></tr>
|
||||
<tr><td><</td><td>The word that follows has a lower relevance than other words, although text containing it will still match</td></tr>
|
||||
<tr><td>></td><td>The word that follows has a higher relevance than other words.</td></tr>
|
||||
<tr><td>()</td><td>Used to group words into subexpressions.</td></tr>
|
||||
<tr><td>~</td><td>The word following contributes negatively to the relevance of the text (which is different to the '-' operator, which specifically excludes the word, or the '<' operator, which still causes the word to contribute positively to the relevance of the text.</td></tr>
|
||||
<tr><td>*</td><td>The wildcard, indicating zero or more characters. It can only appear at the end of a word.</td></tr>
|
||||
<tr><td>"</td><td>Anything enclosed in the double quotes is taken as a whole (so you can match phrases, for example).</td></tr>
|
||||
</tbody></table>
|
||||
</div>
|
||||
<div id="graph"></div>
|
||||
<div id="results"></div>
|
||||
</body>
|
||||
</html>
|
||||
24
www/templates/selection.html
Normal file
24
www/templates/selection.html
Normal file
@ -0,0 +1,24 @@
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>Selection</title>
|
||||
</head>
|
||||
<body>
|
||||
{% for k, v in sel.items() %}
|
||||
<b>{{k}}</b>
|
||||
<ul>
|
||||
{% for h in v %}
|
||||
<li><a href="{{h.url}}" target="_blank">{{h.subject}}</a> -- <i>{{h.author_name}}</i> ({{h.len}} chars)</li>
|
||||
{% if 'follow-up' in h %}
|
||||
<ul>
|
||||
{% for f in h['follow-up'] %}
|
||||
<li><a href="{{f.url}}" target="_blank">{{f.subject}}</a> -- <i>{{f.author_name}}</i> ({{f.len}} chars)</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</ul>
|
||||
|
||||
{% endfor %}
|
||||
</body>
|
||||
</html>
|
||||
Loading…
x
Reference in New Issue
Block a user