List_server_busy/export_xml.py

import argparse, os, glob, sys, json, email.utils
import xml.etree.ElementTree as et
from datetime import datetime
import regex as re

def format_subject(s):
	return ' '.join(s.split())

def format_content(c):
	return c.strip().replace("\n\n\n", "\n")

# See for Nevejan?s research
def remove_control_characters(s):
    return re.sub(r'\p{C}-[\r\n\t]+', '', s)
    # return re.sub(r'\p{Cc}-[\r\n\t]+', '', s)

def parse_date_file(fname):
	return datetime.strptime(fname, '%B_%Y.json')

# returns utc timestamp
def parse_date_msg(msg):
	date_str = msg['date']
	time_tz = None
	try:
		date_tz = email.utils.parsedate_tz(date_str)
		time_tz = email.utils.mktime_tz(date_tz) #utc timestamp
	except TypeError:
		print("Format Date TypeError")
		print("  > " + date_str)
		return None
	except ValueError:
		print("Format Date ValueError")
		print("  > " + date_str)
		return None
	except Exception as ex:
		print(ex)
	finally:
		return time_tz

# recursive
def index_follow_up(msg):
	r = []
	if 'follow-up' in msg:
		for m in msg['follow-up']:
			d = parse_date_msg(m)
			r.append((d, m))
			r += index_follow_up(m)
	return r

nn = 0

def emit_mail_xml(msg, xmlel):

	global nn
	nn += 1

	mail = et.SubElement(xmlel, 'mail')

	subject = et.SubElement(mail, 'subject')
	subject.text = format_subject(msg['subject'])

	to = et.SubElement(mail, 'to')
	if 'to' in msg: 
		to.text = msg['to']
	else:
		to.text = 'n/a'

	from_ = et.SubElement(mail, 'from')
	from_.text = msg['from']

	date = et.SubElement(mail, 'date')
	date.text = msg['date']

	content = et.SubElement(mail, 'content')
	content.text = format_content(msg['content'])

	# recursuve "follow-up"
	if 'follow-up' in msg:
		followups = index_follow_up(msg)
		followups.sort(key=lambda tup: tup[0])
		for d, f in followups:
			emit_mail_xml(f, xmlel)


def export_file(f, fout):

	with open(f) as fp:
		d = json.load(fp)

	all_mail = et.Element('all')
	for t in d['threads']:
		emit_mail_xml(t, all_mail)		

	fout.write(et.tostring(all_mail))		

def export_year(d, dt, fout):

	dir_files = [f for f in glob.glob(os.path.join(d, "*.json"))]

	chapter = et.Element('chapter')
	year = et.SubElement(chapter, 'year')
	year.text = dt.strftime('%Y')

	# SORT MONTHS BEFORE WRITING TO XML
	dates = []
	for f in dir_files:

		fdt = parse_date_file(os.path.basename(f))
		if dt.year != fdt.year:
			continue

		dates.append((fdt, f))

	dates.sort(key=lambda tup: tup[0])

	for d, f in dates:

		print(f)

		section = et.SubElement(chapter, 'section')
		month = et.SubElement(section, 'month')
		month.text = d.strftime('%B')

		with open(f) as fp:
			dj = json.load(fp)

		mails = et.SubElement(section, 'mails')
		for t in dj['threads']:
			emit_mail_xml(t, mails)		

	# write utf8 to file (et.tostring are bytes)
	fout.write(et.tostring(chapter).decode('utf-8', 'ignore'))
	# fout.write(remove_control_characters(et.tostring(chapter).decode('utf-8', 'ignore')))

if __name__ == "__main__":

	p = argparse.ArgumentParser(description='Mailinglists are dead. Long live mailinglists!')
	p.add_argument('file', metavar="f", help="mailinglist file(s) or dir(s) to export", nargs="+")
	p.add_argument('--year', '-y', metavar='y', type=str, help='year of archive')

	args = p.parse_args()

	if args.year:
		dt = datetime.strptime(args.year, '%Y')		

	if not args.file:
		sys.exit('No file(s). Aborting.')

# with open("out.xml", "w") as fout:
	for f in args.file:
		if args.year:
			if not os.path.isdir(f):
				sys.exit(f + ' is not a valid directory. Aborting.')
			foutname = os.path.basename(f) + "_" + dt.strftime('%Y') + ".xml"
			with open(foutname, "w") as fout:
				export_year(f, dt, fout)
				print("nbr of message exported: " + str(nn))
		else:
			if not os.path.isfile(f):
				sys.exit(f + ' is not a valid file. Aborting.')			
			export_file(f, fout)