Le-Club-des-Sans-Sujets/pppadump
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

updates

Browse Source
This commit is contained in:
Michael Murtaugh 2015-03-05 15:10:16 +01:00
parent 8acb66f08a
commit 8b35430b43
2 changed files with 76 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
etherdump
View File

@ -58,15 +58,16 @@ class PadServer (object):
return json.load(urlopen(url))['data']['groupIDs'] return json.load(urlopen(url))['data']['groupIDs']
def getPadText (self, padID): def getPadText (self, padID):
data = {'apikey': self.apikey, 'padID': padID} data = {'apikey': self.apikey, 'padID': padID.encode("utf-8")}
return json.load(urlopen(self.apiurl+'getText?'+urlencode(data)))['data']['text'] return json.load(urlopen(self.apiurl+'getText?'+urlencode(data)))['data']['text']
def getPadHTML (self, padID): def getPadHTML (self, padID):
data = {'apikey': self.apikey, 'padID': padID} data = {'apikey': self.apikey, 'padID': padID.encode("utf-8")}
return json.load(urlopen(self.apiurl+'getHTML?'+urlencode(data)))['data']['html'] return json.load(urlopen(self.apiurl+'getHTML?'+urlencode(data)))['data']['html']
def getPadLastEdited (self, padID): def getPadLastEdited (self, padID):
raw = json.load(urlopen(self.apiurl+'getLastEdited?'+urlencode({'apikey': self.apikey, 'padID': padID})))['data']['lastEdited'] data = {'apikey': self.apikey, 'padID': padID.encode("utf-8")}
raw = json.load(urlopen(self.apiurl+'getLastEdited?'+urlencode(data)))['data']['lastEdited']
return datetime.fromtimestamp(int(raw)/1000) return datetime.fromtimestamp(int(raw)/1000)
def getPadURL (self, padID): def getPadURL (self, padID):
@ -88,7 +89,7 @@ def get_template_env (tpath=None):
# template = env.get_template('pad.html') # template = env.get_template('pad.html')
# print template.render(the='variables', go='here').encode("utf-8") # print template.render(the='variables', go='here').encode("utf-8")
def dumpPads (padserver, padids, pub_path, group_path, sleeptime=0.01, skip_existing=False, templates=None): def dumpPads (padserver, padids, pub_path, group_path, sleeptime=0.01, force=False, templates=None):
template_env = get_template_env(templates) template_env = get_template_env(templates)
pad_template = template_env.get_template("pad.html") pad_template = template_env.get_template("pad.html")
numpads = len(padids) numpads = len(padids)
@ -117,21 +118,31 @@ def dumpPads (padserver, padids, pub_path, group_path, sleeptime=0.01, skip_exis
sys.stderr.write("\rDumping pads... [{0}/{1}]".format(i+1, numpads)) sys.stderr.write("\rDumping pads... [{0}/{1}]".format(i+1, numpads))
sys.stderr.flush() sys.stderr.flush()
if skip_existing:
if os.path.exists(fp+".json"):
continue
# Write Metadata textpath = fp + ".txt"
htmlpath = fp+".html"
metapath = fp+".json"
last_edited = padserver.getPadLastEdited(padid).isoformat()
if os.path.exists(metapath):
with open(metapath) as f:
meta = json.load(f)
if not force and meta.get("last_edited") == last_edited:
if verbose:
print("Up to date, skipping", file=sys.stderr)
continue
meta = { meta = {
'pad_id': padid, 'pad_id': padid,
'group_id': group_id, 'group_id': group_id,
'pad_name': pad_name 'pad_name': pad_name
} }
meta['last_edited'] = padserver.getPadLastEdited(padid).isoformat() meta['last_edited'] = last_edited
# Write Text # Write Text
textpath = fp + ".txt"
with open(textpath, "w") as f: with open(textpath, "w") as f:
try: try:
text = padserver.getPadText(padid) text = padserver.getPadText(padid)
@ -143,7 +154,6 @@ def dumpPads (padserver, padids, pub_path, group_path, sleeptime=0.01, skip_exis
except (TypeError, HTTPError, ValueError) as e: except (TypeError, HTTPError, ValueError) as e:
print(u"Warning: unable to load text for pad {0}, {1}".format(padid, e).encode("utf-8"), file=sys.stderr) print(u"Warning: unable to load text for pad {0}, {1}".format(padid, e).encode("utf-8"), file=sys.stderr)
htmlpath = fp+".html"
with open(htmlpath, "w") as f: with open(htmlpath, "w") as f:
html = padserver.getPadHTML(padid) html = padserver.getPadHTML(padid)
meta['html_path'] = htmlpath meta['html_path'] = htmlpath
@ -167,7 +177,7 @@ def dumpPads (padserver, padids, pub_path, group_path, sleeptime=0.01, skip_exis
# except (TypeError, HTTPError, ValueError) as e: # except (TypeError, HTTPError, ValueError) as e:
# print(u"Warning: unable to load HTML for pad {0}, {1}".format(padid, e).encode("utf-8"), file=sys.stderr) # print(u"Warning: unable to load HTML for pad {0}, {1}".format(padid, e).encode("utf-8"), file=sys.stderr)
with open(fp+".json", "w") as f: with open(metapath, "w") as f:
f.write(json.dumps(meta)) f.write(json.dumps(meta))
if sleeptime: if sleeptime:
@ -231,8 +241,10 @@ def padids_from_path (path):
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
# command
parser.add_argument('command', default="", help='command to perform: listpads, listgroups, dump, createindex') parser.add_argument('command', default="", help='command to perform: listpads, listgroups, dump, createindex')
# padinfo
parser.add_argument('--padinfo', default="padinfo.json", help='path to JSON file containing all pad login data, default padinfo.json, alternatively specify hostname, port, etc as separate options') parser.add_argument('--padinfo', default="padinfo.json", help='path to JSON file containing all pad login data, default padinfo.json, alternatively specify hostname, port, etc as separate options')
parser.add_argument('--hostname', default="", help='the hostname of the etherpad server') parser.add_argument('--hostname', default="", help='the hostname of the etherpad server')
parser.add_argument('--port', type=int, help='port of etherpad server') parser.add_argument('--port', type=int, help='port of etherpad server')
@ -243,16 +255,20 @@ if __name__ == "__main__":
parser.add_argument('--verbose', action="store_true", default=False, help='debug mode, verbose output') parser.add_argument('--verbose', action="store_true", default=False, help='debug mode, verbose output')
parser.add_argument('--pubpath', default="pub", help='path to dump public pads') parser.add_argument('--pubpath', default="pub", help='path to dump public pads')
parser.add_argument('--grouppath', default="priv", help='path to dump group pads') parser.add_argument('--grouppath', default="priv", help='path to dump group pads')
parser.add_argument('--lines', default=False, action="store_true", help='output one per line instead of JSON')
parser.add_argument('--skip-existing', default=False, action="store_true", help='skip existing files on dump')
parser.add_argument('--limit', default=None, type=int, help='(dump) stop after limit items')
parser.add_argument('--templates', default=os.path.join(os.getcwd(), "templates"), help='(addition) templates path, default: ./templates') parser.add_argument('--templates', default=os.path.join(os.getcwd(), "templates"), help='(addition) templates path, default: ./templates')
# INDEX-specific opts # listpads/groups-specific
parser.add_argument('--lines', default=False, action="store_true", help='(listpads/groups) output one per line instead of JSON')
# dump-specific
parser.add_argument('--force', default=False, action="store_true", help='(dump) force dump even if up to date')
parser.add_argument('--skip', default=None, type=int, help='(dump) skip this many (start at index)')
parser.add_argument('--limit', default=None, type=int, help='(dump) stop after limit items')
# index-specific
parser.add_argument('--title', default="etherpad index & archive", help='(index) title') parser.add_argument('--title', default="etherpad index & archive", help='(index) title')
parser.add_argument('--exclude-groups', default=False, action="store_true", help='(index) ignore groups') parser.add_argument('--exclude-groups', default=False, action="store_true", help='(index) ignore groups')
parser.add_argument('--groupinfo', default=None, help='(createindex) groupinfo json file') parser.add_argument('--groupinfo', default=None, help='(index) groupinfo json file')
parser.add_argument('--output', default=None, help='(index) path for output (default stdout)') parser.add_argument('--output', default=None, help='(index) path for output (default stdout)')
@ -318,6 +334,8 @@ if __name__ == "__main__":
elif cmd == "dump": elif cmd == "dump":
start = time.time() start = time.time()
padids = padserver.listAllPads() padids = padserver.listAllPads()
if args.skip:
padids = padids[args.skip:]
if args.limit: if args.limit:
padids = padids[:args.limit] padids = padids[:args.limit]
@ -326,7 +344,7 @@ if __name__ == "__main__":
padids, padids,
args.pubpath, args.pubpath,
args.grouppath, args.grouppath,
args.skip_existing, force=args.force,
templates=args.templates) templates=args.templates)
if verbose: if verbose:
@ -334,8 +352,24 @@ if __name__ == "__main__":
elif cmd == "index": elif cmd == "index":
def augment_info(info, groupinfo):
if info.get("last_edited") != None:
dt = datetime.strptime( info.get("last_edited"), "%Y-%m-%dT%H:%M:%S" )
info['last_edited_parsed'] = dt
info['last_edited_str'] = str(dt)
if groupinfo:
gid = info.get("group_id")
if gid.startswith("g."):
gid = gid[2:]
if gid in groupinfo:
info[u"group_name"] = groupinfo[gid].get("name")
# print (info, file=sys.stderr)
return info
def get_pads(groupinfo=None): def get_pads(groupinfo=None):
pads = padids_from_path(args.pubpath) pads = padids_from_path(args.pubpath)
pads = [augment_info(x, groupinfo) for x in pads]
# print (("padids_from_path", args.pubpath, pads), file=sys.stderr) # print (("padids_from_path", args.pubpath, pads), file=sys.stderr)
if not args.exclude_groups and os.path.exists(args.grouppath): if not args.exclude_groups and os.path.exists(args.grouppath):
groups = [os.path.join(args.grouppath, x) for x in os.listdir(args.grouppath)] groups = [os.path.join(args.grouppath, x) for x in os.listdir(args.grouppath)]
@ -347,7 +381,9 @@ if __name__ == "__main__":
if b not in groupinfo: if b not in groupinfo:
continue continue
try: try:
pads.extend(padids_from_path(gp)) pad_infos = padids_from_path(gp)
pad_infos = [augment_info(x, groupinfo) for x in pad_infos]
pads.extend(pad_infos)
except OSError: except OSError:
pass pass
return pads return pads
@ -357,6 +393,9 @@ if __name__ == "__main__":
with open(args.groupinfo) as gif: with open(args.groupinfo) as gif:
groupinfo = json.load(gif) groupinfo = json.load(gif)
if verbose:
print ("Using groupinfo", file=sys.stderr)
pads = get_pads(groupinfo) pads = get_pads(groupinfo)
padids = [(x.get("pad_name").lower(), x) for x in pads] padids = [(x.get("pad_name").lower(), x) for x in pads]
padids.sort() padids.sort()
@ -372,7 +411,8 @@ if __name__ == "__main__":
out.write(index_template.render( out.write(index_template.render(
pads = pads, pads = pads,
title = args.title title = args.title,
timestamp = datetime.now()
).encode("utf-8")) ).encode("utf-8"))
if args.output: if args.output:

23
templates/index.html
View File

@ -5,16 +5,17 @@
<title>{% block title %}{{title}}{% endblock %}</title> <title>{% block title %}{{title}}{% endblock %}</title>
<link rel="stylesheet" type="text/css" href="{%block css %}styles.css{%endblock%}"> <link rel="stylesheet" type="text/css" href="{%block css %}styles.css{%endblock%}">
{% block scripts %} {% block scripts %}
<script src="jquery.js"></script> <script src="jquery-latest.js"></script>
<script src="jquery.tablesorter.js"></script> <script src="jquery.tablesorter.min.js"></script>
<script src="script.js"></script> <script src="script.js"></script>
{% endblock scripts %} {% endblock scripts %}
</head> </head>
<body> <body>
{% block header %}<h1>{{title}}</h1>{% endblock %} {% block header %}<h1>{{title}}</h1>{% endblock %}
{% block info %}<p class="info">Last updated {{timestamp}}</p>{% endblock %}
{% block namefilter %} {% block namefilter %}
<div id="namefilter"> <div id="namefilter">
<input type="text" id="namefilterinput" placeholder="name filter" autofocus > <input type="text" id="namefilterinput" value="" placeholder="name filter" autofocus >
</div> </div>
<script> <script>
@ -46,16 +47,22 @@ var namefilter = (function (opts) {
return ret; return ret;
})(); })();
$("#namefilterinput").on("keyup", function (e) { namefilter($(this).val()); }) $("#namefilterinput").bind("keyup", function (e) { namefilter($(this).val()); })
$(document).ready(function()
{
$("table.listing").tablesorter();
}
);
</script> </script>
{% endblock %} {% endblock %}
<table class="listing"> <table class="listing tablesorter">
<thead> <thead>
<tr> <tr>
<th>link</th> <th>link</th>
<th>pad name</th> <th>pad name (click to view archived page)</th>
<th>group</th> <th>group</th>
<th>last edited</th> <th>last edited</th>
<th>size</th> <th>size</th>
@ -66,8 +73,8 @@ $("#namefilterinput").on("keyup", function (e) { namefilter($(this).val()); })
<tr> <tr>
<td class="pad_url"><a class="edit" href="{{ pad.url }}">edit</a></td> <td class="pad_url"><a class="edit" href="{{ pad.url }}">edit</a></td>
<td class="pad_name"><a href="{{ pad.html_path }}">{{ pad.pad_name }}</a></td> <td class="pad_name"><a href="{{ pad.html_path }}">{{ pad.pad_name }}</a></td>
<td class="pad_group">{{ pad.group_id }}</td> <td class="pad_group">{{ pad.group_name|default(pad.group_id) }}</td>
<td class="pad_last_edited">{{ pad.last_edited }}</td> <td class="pad_last_edited">{{ pad.last_edited_str }}</td>
<td class="pad_size">{{ pad.text_length_human }}</td> <td class="pad_size">{{ pad.text_length_human }}</td>
</tr> </tr>
{% endfor %} {% endfor %}