gauthiier/listservs
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

MEGA -- DB

Browse Source
This commit is contained in:
gauthiier
2019-07-11 13:21:42 +02:00
parent 3703dcc169
commit 4197cd4d32
25 changed files with 663 additions and 1657 deletions
Download Patch File Download Diff File Expand all files Collapse all files
www/routes.py
+42 -117
View File
@@ -1,144 +1,46 @@
from flask import render_template, request, jsonify
from www import app
from www import archives
import search.archive
import archive.archive as archive
import config
import www.config as wconfig
from datetime import datetime
import logging
logging.info(' ------- arch = Archives() -------- ')
arch = archives.Archives()
arch.load()
archives_data = arch.data
@app.route('/')
def index():
k = archives_data.keys()
return render_template("index.html", archives=k)
# def get_key(kv_tuple):
# k = kv_tuple[0]
# # k is of the form "Month_Year" - ex.: "January_2001"
# try:
# return datetime.strptime(k, "%B_%Y")
# except Exception:
# pass
# # k is of the form "Month(abv)_Year(abv)" - ex.: "Jan_01"
# try:
# return datetime.strptime(k, "%b_%y")
# except Exception:
# pass
# # k is of the form "Year" - ex.: "2001"
# try:
# return datetime.strptime(k, "%Y")
# except Exception:
# pass
# return None
@app.route('/<list>')
def get_list(list):
if list in archives_data:
d = []
for k, v in sorted(archives_data[list].archive.items(), key=search.archive.get_key, reverse=True):
d.append({"name": k, "url": v['url'], "nbr_threads": len(v['threads'])})
return render_template("list.html", list_name=list, list=d)
else:
return 'nee nee'
@app.route('/<list>/<sublist>')
def get_sublist(list, sublist):
print(list)
print(sublist)
sublist = sublist.replace(' ', '_')
if list in archives_data and sublist in archives_data[list].archive:
return render_template("threads.html", sublist_name=sublist, threads=archives_data[list].archive[sublist]['threads'])
else:
return 'na na'
@app.route('/<list>/<sublist>/<int:index>')
def get_message(list, sublist, index):
sublist = sublist.replace(' ', '_')
index = int(index)
if list in archives_data and sublist in archives_data[list].archive and index < len(archives_data[list].archive[sublist]['threads']):
return render_template("message.html", message=archives_data[list].archive[sublist]['threads'][index])
else:
'non non'
@app.route('/<list>/<sublist>/<int:index>/<path:follow_ups>')
def get_follow_ups(list, sublist, index, follow_ups):
sublist = sublist.replace(' ', '_')
index = int(index)
ups = follow_ups.split('/')
follow = []
for u in ups:
follow.append(int(u))
if list in archives_data and sublist in archives_data[list].archive and index < len(archives_data[list].archive[sublist]['threads']):
message = archives_data[list].archive[sublist]['threads'][index]
for f in follow:
message = message['follow-up'][f]
return render_template("message.html", message=message)
else:
'nope nope'
return render_template("index.html")
@app.route('/search')
def searh():
if len(request.args) < 1:
k = archives_data.keys()
return render_template("search.html", archives=k, fields=['content', 'from(name)', 'from(email)'], hits=['n/a', '2', '3', '4', '5', '6', '7', '8', '9'])
return render_template("search.html", archives=wconfig.lists_to_serve, fields=['content', 'from'])
k_arg = request.args.get('keyword')
l_arg = request.args.get('list')
sl_arg = request.args.get('sublist')
f_arg = request.args.get('field')
h_arg = request.args.get('hits')
if k_arg is None or k_arg.strip() == '':
return "no keyword..."
if l_arg is None:
return "no list..."
if not (l_arg == "all") and not (l_arg in archives_data):
if l_arg != "all" and l_arg not in wconfig.lists_to_serve:
return "list '" + l_arg + "' does not exist"
if sl_arg is not None:
if not sl_arg in archives_data[l]:
return "sublist '" + sl_arg + "' does not exist in list '" + l_arg + "'"
if f_arg not in ['content', 'from']:
return "field '" + f_arg + "' does not exist"
if f_arg == "from(name)":
f_arg = 'author_name'
elif f_arg == "from(email)":
f_arg = 'from'
lists = []
if l_arg == "all":
for k in archives_data.keys():
lists.append(k)
lists = wconfig.lists_to_serve
else:
lists.append(l_arg)
nbr_hits = 0
if h_arg in ['2', '3', '4', '5', '6', '7', '8', '9']:
nbr_hits = int(h_arg)
################################
##
## need to cache all the below
## need to cache all the below.....
##
################################
@@ -147,18 +49,41 @@ def searh():
logging.info("search keyword = " + k_arg)
for l in lists:
if k_arg == "rank":
logging.info(" ranking " + l)
s = archives_data[l].threads_ranking()
else:
s = archives_data[l].search(keyword=k_arg, field=f_arg, min_hits=nbr_hits)
with archive.Archive(l, config=config.db) as a:
if f_arg == 'content':
r = a.content_search(k_arg)
else:
r = a.from_search(k_arg)
results.append(s)
# format data to return
search_results = { "keyword": k_arg, "field": f_arg, "archive": a.archive_name, "results": [] }
month_year_results = {}
## -- sort results?
search_results = sorted(results, key=get_result_key)
for (from_, author_name_, subject_, date_, url_) in r:
m_y = date_.strftime("%B_%Y")
if m_y not in month_year_results:
month_year_results[m_y] = []
month_year_results[m_y].append({ 'url': url_, 'subject': subject_, 'author_name': author_name_})
return jsonify(result=search_results)
for k, v in sorted(month_year_results.items(), key=get_key, reverse=True):
search_results['results'].append({ 'thread': k, 'nbr_hits': len(v), 'hits': v})
# search_results['results'].append({ 'thread': k, 'nbr_hits': nbr_hits, 'hits': hits})
# where:
# 'thread' = "%B_%Y" aka. January 2001
# 'nbr_hits' = nbr hits for that month
# 'hits' = [{ 'url': h['url'], 'subject': h['subject'], 'author_name': h['from']}]
results.append(search_results)
sorted_results = sorted(results, key=get_result_key)
return jsonify(result=sorted_results)
def get_key(kv):
return datetime.strptime(kv[0], "%B_%Y")
def get_result_key(r):
return r['archive']
www/templates/index.html
+1 -3
View File
@@ -1,8 +1,6 @@
<html>
<head></head>
<body>
{% for a in archives %}
<a href="/{{ a }}"><h3>{{ a }}</h3></a>
{% endfor %}
<a href="/search"><h3>---> SEARCH <---</h3></a>
</body>
</html>
www/templates/list.html
-10
View File
@@ -1,10 +0,0 @@
<html>
<head></head>
<body>
<ul>
{% for t in list %}
<li><a href="{{ list_name }}/{{ t.name }}"><h3>{{ t.name }} -- {{ t.nbr_threads }}</h3></a></li>
{% endfor %}
</ul>
</body>
</html>
www/templates/message.html
-11
View File
@@ -1,11 +0,0 @@
<html>
<head>
<meta charset="UTF-8">
</head>
<body>
<h3>{{ message.subject }}</h3>
<h4>{{ message.author_name }}</h4>
<h4>{{ message.date }}</h4>
<p>{{ message.content }} </p>
</body>
</html>
www/templates/search.html
-5
View File
@@ -20,11 +20,6 @@
<option value="{{ a }}">{{ a }}</option>
{% endfor %}
</select>
<select form="search" name="hits">
{% for a in hits %}
<option value="{{ a }}">{{ a }}</option>
{% endfor %}
</select>
<input type="submit" value="search" id="submit">
<div id="loading">Loading...</div>
</form>
www/templates/threads.html
-25
View File
@@ -1,25 +0,0 @@
<html>
<head></head>
<body>
{% macro message(m, index, urlpath)-%}
{% set path = urlpath + '/' + index|string %}
<li>
{{ index }}. <a href="{{ path }}">{{ m.subject }}</a> <i>{{ m.author_name }}</i>
{% if m.get('follow-up') %}
<ul>
{% for msg in m.get('follow-up') %}
{{ message(m=msg, index=loop.index - 1, urlpath=path) }}
{% endfor %}
</ul>
{% endif %}
</li>
{%- endmacro %}
<ul>
{% for m in threads recursive %}
{{ message(m=m, index=loop.index - 1, urlpath=sublist_name) }}
{% endfor %}
</ul>
</body>
</html>