42 lines
1.5 KiB
Python
42 lines
1.5 KiB
Python
import sys, logging, argparse
|
|
import lists.crawl, config
|
|
|
|
logging.basicConfig(level=logging.DEBUG)
|
|
|
|
def run(list_archive_name_url, archive_dir):
|
|
|
|
## check valid url?... hmm... nej
|
|
for l in list_archive_name_url:
|
|
logging.info("+++++ Crawling list: " + l['name'] + " +++++")
|
|
lists.crawl.crawl(url=l['url'], name=l['name'], sublist_name=l['subname'], archive_dir=archive_dir)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
p = argparse.ArgumentParser(description='Mailinglists are dead. Long live mailinglists!')
|
|
p.add_argument('url', metavar="url", help="mailinglist urls to archive", nargs="+")
|
|
p.add_argument('--names', '-n', help="mailinglists' names (for the archive)", nargs="+")
|
|
p.add_argument('--subnames', '-s', help="mailinglists' subnames (on the webpage)", nargs="+")
|
|
p.add_argument('--archives', '-a', help="path to archives directory", default=config.archives)
|
|
|
|
args = p.parse_args()
|
|
|
|
l = []
|
|
if len(args.url) == 1 and args.url[0] == 'all':
|
|
l = config.lists
|
|
elif args.names == None and args.subnames == None:
|
|
for u in args.url:
|
|
for li in config.lists:
|
|
if u == li['name']:
|
|
l.append(li)
|
|
elif len(args.url) == len(args.names) == len(args.subnames):
|
|
i = 0
|
|
for u in args.url:
|
|
l.append({'name': args.names[i], 'url': u, 'subname': args.subnames[i]})
|
|
i += 1
|
|
else:
|
|
sys.exit("Inconcistent list of names, urls, and subnames. Aborting.")
|
|
|
|
|
|
run(l, args.archives)
|