Le-Club-des-Sans-Sujets/pppadump
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

continued tweaks

Browse Source
This commit is contained in:
Michael Murtaugh 2015-09-19 11:43:16 +02:00
parent 3957c44faa
commit 0f340433ae
3 changed files with 271 additions and 190 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

108
etherdump/commands/dump.py
View File

@ -1,6 +1,11 @@
#!/usr/bin/env python
# License: AGPL
#
#
# todo:
# Capture exceptions... add HTTP status errors (502) to meta!!
# so that an eventual index can show the problematic pages!
# Also: provide links to text only / html versions when diff HTML fails
from __future__ import print_function
from etherdump import DATAPATH
@ -12,6 +17,7 @@ from datetime import datetime
from xml.etree import cElementTree as ET
from urllib import urlencode
from urllib2 import urlopen, HTTPError, URLError
from time import sleep
# external dependencies (use pip to install these)
import html5lib, jinja2
@ -102,6 +108,11 @@ def get_template_env (tpath=None):
env = jinja2.Environment(loader=loader)
return env
def get_group_info(gid, info):
if 'groups' in info:
if gid in info['groups']:
return info['groups'][gid]
def main(args):
p = ArgumentParser("""
_ _ _
@ -117,7 +128,6 @@ def main(args):
p.add_argument("--verbose", default=False, action="store_true", help="flag for verbose output")
p.add_argument("--limit", type=int, default=None)
p.add_argument("--allpads", default=False, action="store_true", help="flag to process all pads")
p.add_argument("--spider", default=False, action="store_true", help="flag to spider pads")
p.add_argument("--templatepath", default=os.path.join(DATAPATH, "templates"), help="directory with templates (override default files)")
p.add_argument("--colors-template", default="pad_colors.html", help="pad with authorship colors template name: pad_colors.html")
p.add_argument("--padlink", default=[], action="append", help="give a pad link pattern, example: 'http\:\/\/10\.1\.10\.1/p/(.*)'")
@ -126,8 +136,11 @@ def main(args):
p.add_argument("--showurls", default=False, action="store_true", help="flag to display API URLs that are used (to stderr)")
p.add_argument("--hidepaths", default=False, action="store_true", help="flag to not display paths")
p.add_argument("--pretend", default=False, action="store_true", help="flag to not actually save")
p.add_argument("--linkify", default=False, action="store_true", help="flag to process [[link]] forms (and follow when --spider is used)")
p.add_argument("--spider", default=False, action="store_true", help="flag to spider pads (requires --linkify)")
p.add_argument("--add-images", default=False, action="store_true", help="flag to add image tags")
p.add_argument("--authors-css", default="authors.css", help="filename to save collected authorship css (nb: etherdump will overwrite this file!)")
p.add_argument("--force", default=False, action="store_true", help="force dump (even if not updated since last dump)")
p.add_argument("--authors-css", default=None, help="filename to save collected authorship css (nb: any existing file will be mercilessly overwritten), default: don't accumulate css")
# TODO css from pad --- ie specify a padid for a stylesheet!!!!!!
# p.add_argument("--css", default="styles.css", help="padid of stylesheet")
@ -175,14 +188,37 @@ def main(args):
done.add(padid)
data['padID'] = padid.encode("utf-8")
if args.verbose:
print ("PADID \"{0}\"".format(padid).encode("utf-8"), file=sys.stderr)
print (u"PADID \"{0}\"".format(padid).encode("utf-8"), file=sys.stderr)
# g.yIRLMysh0PMsCMHc$
grouppat = re.compile(ur"^g\.(\w+)\$(.+)$")
m = grouppat.search(padid)
if m:
group = m.group(1)
ginfo = get_group_info(group, info)
if not ginfo:
print ("No info for group '{0}', skipping".format(group), file=sys.stderr)
continue
padid = m.group(2)
else:
group = None
ginfo = None
if not args.pretend:
try:
if ginfo:
os.makedirs(os.path.join(args.path, ginfo['name']))
else:
os.makedirs(args.path)
except OSError:
pass
retry = True
tries = 1
while retry:
retry = False
try:
# _
@ -192,14 +228,18 @@ def main(args):
# |_| |_| |_|\___|\__\__,_|
meta_url = urlify(padid, ext=".json")
meta_out = "{0}/{1}".format(args.path, meta_url.encode("utf-8"))
raw_url = urlify(padid, ext=".txt")
raw_out = "{0}/{1}".format(args.path, raw_url.encode("utf-8"))
colors_url = urlify(padid, ext=".html")
if ginfo:
meta_out = "{0}/{1}/{2}".format(args.path, ginfo['name'], meta_url.encode("utf-8"))
raw_out = "{0}/{1}/{2}".format(args.path, ginfo['name'], raw_url.encode("utf-8"))
colors_out = "{0}/{1}/{2}".format(args.path, ginfo['name'], colors_url.encode("utf-8"))
else:
meta_out = "{0}/{1}".format(args.path, meta_url.encode("utf-8"))
raw_out = "{0}/{1}".format(args.path, raw_url.encode("utf-8"))
colors_out = "{0}/{1}".format(args.path, colors_url.encode("utf-8"))
if not args.hidepaths:
print (meta_out, file=sys.stderr)
if not args.pretend:
meta = {}
meta['padid'] = padid
@ -208,6 +248,19 @@ def main(args):
print (revisions_url, file=sys.stderr)
meta['total_revisions'] = json.load(urlopen(revisions_url))['data']['revisions']
# CHECK REVISIONS (against existing meta)
if meta['total_revisions'] == 0:
if args.verbose:
print (" pad has no revisions, skipping", file=sys.stderr)
continue
if os.path.exists(meta_out):
with open(meta_out) as f:
old_meta = json.load(f)
if not args.force and old_meta['total_revisions'] == meta['total_revisions']:
if args.verbose:
print (" skipping (up to date)", file=sys.stderr)
continue
lastedited_url = apiurl+'getLastEdited?'+urlencode(data)
if args.showurls:
print (lastedited_url, file=sys.stderr)
@ -223,21 +276,24 @@ def main(args):
meta['colors'] = colors_url
meta['raw'] = raw_url
meta['meta'] = meta_url
with open(meta_out, "w") as f:
json.dump(meta, f)
# defer output to LAST STEP (as confirmation)
# _ __ __ ___ __
# | '__/ _` \ \ /\ / /
# | | | (_| |\ V V /
# |_| \__,_| \_/\_/
if not args.hidepaths:
print (raw_out, file=sys.stderr)
text_url = apiurl+"getText?"+urlencode(data)
if args.showurls:
print (text_url, file=sys.stderr)
if not args.pretend:
rawText = json.load(urlopen(text_url))['data']['text']
if rawText.strip() == "":
if args.verbose:
print (" empty text, skipping", file=sys.stderr)
continue
if not args.hidepaths:
print (raw_out, file=sys.stderr)
with open(raw_out, "w") as f:
f.write(rawText.encode("utf-8"))
@ -269,6 +325,7 @@ def main(args):
# Stage 1: Process as text
# Process [[wikilink]] style links
# and (optionally) add linked page names to spider todo list
if args.linkify:
html, links = linkify(html)
if args.spider:
for l in links:
@ -338,8 +395,8 @@ def main(args):
if len(i):
selector, rule = i.split(' ',1)
authors_css_rules[selector] = rule
style = '' # strip the individual style tag from each page (only exports to authors-css file)
# nb: it's up to the template to refer to the authors-css file
# replace individual style with a ref to the authors-css
style = '<link rel="stylesheet" type="text/css" href="{0}">'.format(args.authors_css)
else:
style = ET.tostring(style, method="html")
else:
@ -362,6 +419,11 @@ def main(args):
lastedited = meta['lastedited']
).encode("utf-8"))
# OUTPUT METADATA (finally)
if not args.hidepaths:
print (meta_out, file=sys.stderr)
with open(meta_out, "w") as f:
json.dump(meta, f)
# _
# | | ___ ___ _ __
# | |/ _ \ / _ \| '_ \
@ -372,8 +434,24 @@ def main(args):
count += 1
if args.limit and count >= args.limit:
break
except TypeError:
print ("ERROR, skipping!", file=sys.stderr)
# except HTTPError as e:
# retry = True
# except TypeError as e:
# print ("TypeError, skipping!", file=sys.stderr)
except Exception as e:
print ("[{0}] Exception: {1}".format(tries, e), file=sys.stderr)
sleep(3)
retry = True
if retry:
tries += 1
if tries > 5:
print (" GIVING UP", file=sys.stderr)
retry = False
# Write the unified CSS with authors
if args.authors_css:

1
etherdump/data/templates/pad_colors.html
View File

@ -5,7 +5,6 @@
<meta charset="utf-8">
<meta revision="{{revision}}">
<link rel="stylesheet" type="text/css" href="pad.css">
<link rel="stylesheet" type="text/css" href="authors.css">
{{ style }}
</head>
<body>

6
padinfo.sample.json
View File

@ -4,5 +4,9 @@
"hostname": "localhost",
"apiversion": "1.2.9",
"apiurl": "/api/",
"apikey": "8f55f9ede1b3f5d88b3c54eb638225a7bb71c64867786b608abacfdb7d418be1"
"apikey": "8f55f9ede1b3f5d88b3c54eb638225a7bb71c64867786b608abacfdb7d418be1",
"groups": {
"71FpVh4MZBvl8VZ6": {"name": "Transmediale", "id": 43},
"HyYfoX3Q6S5utxs5": {"name": "test", "id": 42 }
}
}