gauthiier/List_server_busy
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

tm20 selection [ready for rene]

Browse Source
This commit is contained in:
gauthiier
2019-12-09 13:45:24 +01:00
parent 46a0cc8c0b
commit abea0f0597
17 changed files with 3421 additions and 1238 deletions
Download Patch File Download Diff File Expand all files Collapse all files
selection/__init__.py
View File
selection/construct.py
-51
View File
@@ -1,51 +0,0 @@
import os, json, glob, logging
ARCH = "archives/"
EXP = "selection/"
sel = os.path.join(EXP, "tm-selection.js")
def find(li, url):
d = os.path.join(ARCH, li)
if not os.path.isdir(d):
logging.warning("Invalid archive path: " + d)
return None
dir_files = [f for f in glob.glob(os.path.join(d, "*.json"))]
for f in dir_files:
with open(f) as fp:
dj = json.load(fp)
for t in dj['threads']:
if t['url'] == url:
return t
return None
def construct():
dump = {}
with open(sel) as f:
d = json.load(f)
for k, v in d.items():
dump[k] = []
for i in v:
m = find(i['list'], i['url'])
if m is not None:
m['list'] = i['list']
dump[k].append(m)
fout = os.path.join(EXP, "tm-selection-dump.js")
with open(fout, 'w+', encoding='utf-8') as f:
json.dump(dump, f, ensure_ascii=False, indent=4)
if __name__ == "__main__":
construct()
selection/sel.py
+163
View File
@@ -0,0 +1,163 @@
import os, json, glob, logging
ARCH = "archives/"
EXP = "selection/"
sel = os.path.join(EXP, "tm-selection.js")
sel_dump = os.path.join(EXP, "tm-selection-dump.js")
with open(sel) as f:
d = json.load(f)
def lists():
return os.listdir(sel.ARCH)
def tags():
global d
return list(d.keys())
def find(li, url):
d = os.path.join(ARCH, li)
if not os.path.isdir(d):
logging.warning("Invalid archive path: " + d)
print("Invalid archive path: " + d)
return None
dir_files = [f for f in glob.glob(os.path.join(d, "*.json"))]
for f in dir_files:
with open(f) as fp:
dj = json.load(fp)
for t in dj['threads']:
if t['url'] == url: # one level..... not recursive
return t
return None
def recursive_urls(msg):
r = [msg['url']]
if 'follow-up' in list(msg.keys()):
for m in msg['follow-up']:
r += recursive_urls(m)
return r
def commit_selection(li, url, tag):
if tag not in list(d.keys()):
print("new tag: " + tag)
d[tag] = []
for i in d[tag]:
if i['url'] == url:
return False
d[tag].append({'list': li, 'url': url})
with open(sel, 'w', encoding='utf-8') as f:
json.dump(d, f, ensure_ascii=False, indent=4)
return True
def commit_dump(li, url, tag):
if not commit_selection(li, url, tag):
return None
m = find(li, url) # <--- time
if m is not None:
with open(sel_dump) as f:
dump = json.load(f)
if tag not in list(dump.keys()):
dump[tag] = []
dump[tag].append(m)
with open(sel_dump, 'w+', encoding='utf-8') as fout:
json.dump(dump, fout, ensure_ascii=False, indent=4)
commited = recursive_urls(m)
return commited
return None
def commit_from_selection():
dump = {}
with open(sel) as f:
d = json.load(f)
for k, v in d.items():
dump[k] = []
for i in v:
m = find(i['list'], i['url']) # <--- time
if m is not None:
m['list'] = i['list']
dump[k].append(m)
with open(sel_dump, 'w+', encoding='utf-8') as f:
json.dump(dump, f, ensure_ascii=False, indent=4)
def report():
re = "Report: \n"
for k, v in d.items():
lre = {}
for i in v:
if i['list'] not in lre:
lre[i['list']] = 0
lre[i['list']] += 1
re += "<" + k + ">: " + str(len(v)) + " ("
for kk, vv in lre.items():
re += kk + ": " + str(vv) + " / "
re += ")\n"
return re
def recursive_format(msg):
msg.pop('id')
msg['len'] = len(msg['content'])
msg.pop('content')
msg.pop('content-type')
if 'to' in msg:
msg.pop('to')
if 'message-id' in msg:
msg.pop('message-id')
if 'follow-up' in msg:
for i in msg['follow-up']:
recursive_format(i)
def format_selection():
with open(sel_dump) as f:
d = json.load(f)
for k, v in d.items():
for i in v:
recursive_format(i)
return d
def recursive_hashmap(msg, tag, hm):
hm[msg['url']] = tag
if 'follow-up' in msg:
for i in msg['follow-up']:
recursive_hashmap(i, tag, hm)
def hashmap():
with open(sel_dump) as f:
d = json.load(f)
hm = {}
for k, v in d.items():
for i in v:
recursive_hashmap(i, k, hm)
return hm
if __name__ == "__main__":
d = format_selection()
print(json.dumps(d, indent=4, sort_keys=True))
selection/tm-selection-backup.js
+125
View File
@@ -0,0 +1,125 @@
{
"cyberfeminism": [
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9706/msg00111.html"
}
],
"end2end": [],
"net.art": [
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=15160"
},
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=40578"
},
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=41637"
},
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=42354"
},
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=43292"
},
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=47543"
},
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=48346"
},
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=54012"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9708/msg00010.html"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9708/msg00009.html"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9703/msg00038.html"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9703/msg00060.html"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9703/msg00096.html"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9610/msg00029.html"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9705/msg00053.html"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9705/msg00003.html"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9707/msg00014.html"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00109.html"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00117.html"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00084.html"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00162.html"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00153.html"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00202.html"
},
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1310&L=new-media-curating&F=&S=&P=2942"
},
{
"list": "empyre",
"url": "http://lists.cofa.unsw.edu.au/pipermail/empyre/2016-September/009250.html"
},
{
"list": "empyre",
"url": "http://lists.cofa.unsw.edu.au/pipermail/empyre/2016-September/009253.html"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-0009/msg00209.html"
}
],
"new media art": [
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-0905/msg00038.html"
}
]
}
selection/tm-selection-dump.js
+2669 -1120
View File
File diff suppressed because one or more lines are too long
selection/tm-selection.js
+111 -1
View File
@@ -1,11 +1,121 @@
{
"cyberfeminism": [
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9706/msg00111.html"
}
],
"end2end": [],
"net.art": [
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=15160"
},
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=40578"
},
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=41637"
},
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=42354"
},
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=43292"
},
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=47543"
},
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=48346"
},
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1406&L=new-media-curating&F=&S=&P=54012"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9708/msg00010.html"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9708/msg00009.html"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9703/msg00038.html"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9703/msg00060.html"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9703/msg00096.html"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9610/msg00029.html"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9705/msg00053.html"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9705/msg00003.html"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-9707/msg00014.html"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00109.html"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00117.html"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00084.html"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00162.html"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00153.html"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-0102/msg00202.html"
},
{
"list": "crumb",
"url": "https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind1310&L=new-media-curating&F=&S=&P=2942"
},
{
"list": "empyre",
"url": "http://lists.cofa.unsw.edu.au/pipermail/empyre/2016-September/009250.html"
},
{
"list": "empyre",
"url": "http://lists.cofa.unsw.edu.au/pipermail/empyre/2016-September/009253.html"
},
{
"list": "nettime-l",
"url": "https://nettime.org/Lists-Archives/nettime-l-0009/msg00209.html"
}
],
"end2end": [],
"new media art": [
{
"list": "nettime-l",