har html

firebug-netexp/har_parser.py

@@ -1,15 +1,114 @@
-import sys, json
+#!/usr/bin/python2.7
+
+import sys, csv, json, os, re
+from optparse import OptionParser
+
+# matches a given url to all possible bugs
+def match(url, bugs):
+	for b in bugs:
+		pattern = re.compile(b['pattern'])
+		if(pattern.search(url)):
+			return {'name': b['name'], 'type': b['type'], 'class': b['classification']}
+	return None
+
+def format(e, b):
+	r = {}
+	# bug
+	r['bug'] = b
+	# request
+	r['request'] = {}
+	r['request']['date_time'] = e['startedDateTime']
+	r['request']['time'] = e['time']
+	r['request']['method'] = e['request']['method']
+	r['request']['url'] = e['request']['url']
+	headers = e['request']['headers']
+	for h in headers:
+		if h['name'] == 'User-Agent':
+			r['request']['user_agent'] = h['value']
+		elif h['name'] == 'Referer':
+			r['request']['referer'] = h['value']
+	# response
+	r['response'] = {}
+	r['response']['status'] = e['response']['status']
+	r['response']['server_ip'] = e['serverIPAddress']
+	headers = e['response']['headers']
+	for h in headers:
+		if h['name'] == 'Server':
+			r['response']['server_agent'] = h['value']
+		if h['name'] == 'Last-Modified':
+			r['response']['last_modified'] = h['value']
+		if h['name'] == 'Date':
+			r['response']['date_time'] = h['value']
+	r['response']['header_size'] = e['response']['headersSize']
+	r['response']['body_size'] = e['response']['bodySize']
+	# content
+	r['content'] = {}
+	r['content']['mime_type'] = e['response']['content']['mimeType']
+	r['content']['size'] = e['response']['content']['size']
+	r['content']['data'] = e['response']['content']['text']
+	# timings
+	r['timing'] = e['timings']
+
+	return r
+
+def run(options):
+
+	if not options.file:
+		sys.exit('No har input file specified. Aborting.')
+
+	har_file = options.file
+
+	try:
+		with open(har_file) as har_data_file:
+			har_data = json.load(har_data_file)
+	except Exception, ee:	
+		sys.exit('Error loading har input file... Aborting.')
+
+
+	if not options.bugs:
+		sys.exit('No ghostery bugs input file. Aborting.')
+
+	bugs_file = options.bugs
+
+	try:
+		with open(bugs_file) as bugs_data_file:
+			bugs_data = json.load(bugs_data_file)
+	except Exception, ee:	
+		sys.exit('Error loading bugs data... Aborting.')
+
+	transactions = []
+	bugs = bugs_data['bugs']
+	tags = bugs_data['tags']
+	entries = har_data['log']['entries']
+	i = 0
+	for e in entries:
+		i += 1
+		#print str(i) + '/' + str(len(entries))
+		url = e['request']['url']
+		bug = match(url, bugs) 
+		if bug:
+			info = format(e, bug)
+			transactions.append(info)
+
+	#we leave the ghost tags section... 
+
+	output = {}
+	output['browser'] = har_data['log']['browser']
+	output['exporter'] = har_data['log']['creator']
+	output['date_time'] = har_data['log']['pages'][0]['startedDateTime']
+	output['data'] = transactions
+
+	return output
+
 
 if __name__ == '__main__':
 
-	fp = sys.stdin
+	p = OptionParser();
-	try:
+	p.add_option('-f', '--file', action="store", help="har input file")
-		sdata = fp.read()
+	p.add_option('-b', '--bugs', action="store", help="ghostery bugs input file")
-		data = json.loads(sdata)
-	except Exception, ee:	
-		sys.exit('Error loading data... Aborting.')
 
-	entries = data['log']['entries']
+	options, args = p.parse_args()
-	for e in entries:
+
-		req = e['request']
+	result = run(options)
-		print req['url']
+
+	print json.dumps(result, indent=2, separators=(',',':'))

firebug-netexp/html/+++/har_lestyle.css

@@ -0,0 +1,70 @@
+body {
+	width: 100%;
+	height: 100%;
+	margin: 0;
+	padding: 0;
+}
+
+.agent {
+	padding: 1.5em;
+	float: left;
+	margin: 1em;
+	width: 20em;
+	background-color: #eeeeee;
+}
+
+.card {
+	padding: 1.5em;
+	float: left;
+	display: inline;
+	margin: 1em;
+	width: 45em;
+	background-color: #eeeeee;
+	border-radius: 6px;
+}
+
+.bug {
+	padding: 0.5em;
+	border-radius: 6px;
+}
+
+.data {
+	margin-left: 0.5em;
+	margin-right: 0.5em;
+	word-wrap: break-word;
+}
+
+.name {
+	font-weight: bold;
+}
+
+.code {
+	word-wrap: break-word;
+	padding-top: 1.5em;
+	padding-bottom: 1.5em;
+}
+
+.ad {
+	background-color: rgba(0, 191, 243, 0.9);
+}
+
+.analytics {
+	background-color: rgba(163, 170, 64, 0.9);
+}
+
+.tracker {
+	background-color: rgba(208, 18, 49, 0.9);
+}
+.widget {
+	background-color: rgba(0, 154, 138, 0.9);
+}
+.privacy {
+	background-color: rgba(251, 88, 37, 0.9);
+}
+
+
+table {
+	width: 100%;
+}
+
+

firebug-netexp/html/index_template.html

@@ -0,0 +1,10 @@
+<html lang="en">
+<head>
+	<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+	<link rel="stylesheet" type="text/css" href="+++/har_lestyle.css"/>
+	<title>Bugs Forensics Catalogue</title>
+</head>
+<body>
+		[[content]]
+</body>
+</html>

firebug-netexp/html/netexp_to_htmlcards.py

@@ -0,0 +1,116 @@
+import sys, csv, json, os, re
+
+def emit_header(browser, exporter, data_time):
+	str_s = '<div class="agent">'
+	str_s += '<b>Date</b>:<br>'
+	str_s += data_time + '\n'
+	str_s += '<br>\n'	
+	str_s += '<b>Browser</b>:<br>'
+	str_s += browser['name']+ '-' + browser['version'] + '\n'
+	str_s += '<br>\n'
+	str_s += '<b>Exporter</b>:<br>'
+	str_s += exporter['name']+ '-' + exporter['version'] + '\n'
+	str_s += '<br>\n'	
+	str_s += '</div>\n' 
+	return str_s
+
+
+def emit_bug(b):
+
+	bug = b['bug']
+	str_s = '<div class="bug ' + bug['type'] + '">'
+
+	# bug
+	str_s += '<b>Bug</b>'
+	str_s += '<br>'
+	str_s += '<div class="data">\n'
+	str_s += 'type: ' + bug['type'] + '<br>' + 'property: ' + bug['name']
+	str_s += '</div>'
+
+	str_s += '<br>'
+
+	# request
+	req = b['request']
+	str_s += '<b>Request</b>'
+	str_s += '<br>'
+	str_s += '<div class="data">\n'
+	str_s += req['method'] + '<br>' + req['url'] + '<br>' + req['user_agent']
+	str_s += '<br>'
+	if 'referer' in req:
+		str_s += '<br>'
+		str_s += 'Ref: ' + req['referer']
+		str_s += '<br>'
+	str_s += '</div>'
+
+	str_s += '<br>'
+
+	# response
+	res = b['response']
+	str_s += '<b>Response</b>'
+	str_s += '<br>'
+	str_s += '<div class="data">\n'
+	if 'server_agent' in res:
+		str_s += str(res['status']) + '<br>' + str(res['server_ip']) + '<br>' + res['server_agent'] + '<br>'
+	else:
+		str_s += str(res['status']) + '<br>' + str(res['server_ip']) + '<br>'
+	str_s += '</div>'
+
+	str_s += '<br>'
+
+	# content
+	c = b['content']
+	str_s += '<b>Content</b>'
+	str_s += '<br>'
+	str_s += '<div class="data">\n'
+	str_s += 'mime type: ' + c['mime_type'] + '<br>'
+	str_s += 'size: ' + str(c['size']) + '<br>'
+
+	if len(c['data']) > 500:
+		str_s += '<div class="code">\n'
+		str_s += html_escape(c['data']) + '\n'
+		str_s += '</div>\n'
+
+	str_s += '</div>'
+	str_s += '</div>'
+	return str_s
+
+def html_escape(text):
+	html_escape_table = {
+	 "&": "&amp;",
+	 '"': "&quot;",
+	 "'": "&apos;",
+	 ">": "&gt;",
+	 "<": "&lt;",
+	 "[": "&#91",
+	 "]": "&#93"
+	 }
+	return "".join(html_escape_table.get(c,c) for c in text)
+
+if __name__ == '__main__':
+
+	fp = sys.stdin
+	try:
+		data = json.loads(fp.read())
+	except Exception, ee:	
+		sys.exit('Error loading data... Aborting.')
+
+	try:
+		template = open(os.path.join('.', 'index_template.html'), 'r+');
+	except:
+		print('error opening template file. aborting...');
+		sys.exit(0);		
+
+	content = ""
+	content += emit_header(data['browser'], data['exporter'], data['date_time'])
+
+	bugs = data['data']
+
+	for b in bugs:				
+		card = "<div class='card'>\n"
+		card += emit_bug(b)
+		card += "</div>\n"
+		content += card
+
+	html = template.read().replace('[[content]]', content);
+
+	print (html).encode("utf-8")

ghost-bugs/bugs_pretty.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python2.7
+import sys, json
+if __name__ == '__main__':
+	fp = sys.stdin
+	data = json.loads(fp.read())
+	print json.dumps(data, indent=2, separators=(',',':'))