diff --git a/firebug-netexp/har_parser.py b/firebug-netexp/har_parser.py index e10eda2..e35bb06 100644 --- a/firebug-netexp/har_parser.py +++ b/firebug-netexp/har_parser.py @@ -1,15 +1,114 @@ -import sys, json +#!/usr/bin/python2.7 + +import sys, csv, json, os, re +from optparse import OptionParser + +# matches a given url to all possible bugs +def match(url, bugs): + for b in bugs: + pattern = re.compile(b['pattern']) + if(pattern.search(url)): + return {'name': b['name'], 'type': b['type'], 'class': b['classification']} + return None + +def format(e, b): + r = {} + # bug + r['bug'] = b + # request + r['request'] = {} + r['request']['date_time'] = e['startedDateTime'] + r['request']['time'] = e['time'] + r['request']['method'] = e['request']['method'] + r['request']['url'] = e['request']['url'] + headers = e['request']['headers'] + for h in headers: + if h['name'] == 'User-Agent': + r['request']['user_agent'] = h['value'] + elif h['name'] == 'Referer': + r['request']['referer'] = h['value'] + # response + r['response'] = {} + r['response']['status'] = e['response']['status'] + r['response']['server_ip'] = e['serverIPAddress'] + headers = e['response']['headers'] + for h in headers: + if h['name'] == 'Server': + r['response']['server_agent'] = h['value'] + if h['name'] == 'Last-Modified': + r['response']['last_modified'] = h['value'] + if h['name'] == 'Date': + r['response']['date_time'] = h['value'] + r['response']['header_size'] = e['response']['headersSize'] + r['response']['body_size'] = e['response']['bodySize'] + # content + r['content'] = {} + r['content']['mime_type'] = e['response']['content']['mimeType'] + r['content']['size'] = e['response']['content']['size'] + r['content']['data'] = e['response']['content']['text'] + # timings + r['timing'] = e['timings'] + + return r + +def run(options): + + if not options.file: + sys.exit('No har input file specified. Aborting.') + + har_file = options.file + + try: + with open(har_file) as har_data_file: + har_data = json.load(har_data_file) + except Exception, ee: + sys.exit('Error loading har input file... Aborting.') + + + if not options.bugs: + sys.exit('No ghostery bugs input file. Aborting.') + + bugs_file = options.bugs + + try: + with open(bugs_file) as bugs_data_file: + bugs_data = json.load(bugs_data_file) + except Exception, ee: + sys.exit('Error loading bugs data... Aborting.') + + transactions = [] + bugs = bugs_data['bugs'] + tags = bugs_data['tags'] + entries = har_data['log']['entries'] + i = 0 + for e in entries: + i += 1 + #print str(i) + '/' + str(len(entries)) + url = e['request']['url'] + bug = match(url, bugs) + if bug: + info = format(e, bug) + transactions.append(info) + + #we leave the ghost tags section... + + output = {} + output['browser'] = har_data['log']['browser'] + output['exporter'] = har_data['log']['creator'] + output['date_time'] = har_data['log']['pages'][0]['startedDateTime'] + output['data'] = transactions + + return output + if __name__ == '__main__': - fp = sys.stdin - try: - sdata = fp.read() - data = json.loads(sdata) - except Exception, ee: - sys.exit('Error loading data... Aborting.') + p = OptionParser(); + p.add_option('-f', '--file', action="store", help="har input file") + p.add_option('-b', '--bugs', action="store", help="ghostery bugs input file") - entries = data['log']['entries'] - for e in entries: - req = e['request'] - print req['url'] \ No newline at end of file + options, args = p.parse_args() + + result = run(options) + + print json.dumps(result, indent=2, separators=(',',':')) \ No newline at end of file diff --git a/firebug-netexp/html/+++/har_lestyle.css b/firebug-netexp/html/+++/har_lestyle.css new file mode 100644 index 0000000..6fb53cd --- /dev/null +++ b/firebug-netexp/html/+++/har_lestyle.css @@ -0,0 +1,70 @@ +body { + width: 100%; + height: 100%; + margin: 0; + padding: 0; +} + +.agent { + padding: 1.5em; + float: left; + margin: 1em; + width: 20em; + background-color: #eeeeee; +} + +.card { + padding: 1.5em; + float: left; + display: inline; + margin: 1em; + width: 45em; + background-color: #eeeeee; + border-radius: 6px; +} + +.bug { + padding: 0.5em; + border-radius: 6px; +} + +.data { + margin-left: 0.5em; + margin-right: 0.5em; + word-wrap: break-word; +} + +.name { + font-weight: bold; +} + +.code { + word-wrap: break-word; + padding-top: 1.5em; + padding-bottom: 1.5em; +} + +.ad { + background-color: rgba(0, 191, 243, 0.9); +} + +.analytics { + background-color: rgba(163, 170, 64, 0.9); +} + +.tracker { + background-color: rgba(208, 18, 49, 0.9); +} +.widget { + background-color: rgba(0, 154, 138, 0.9); +} +.privacy { + background-color: rgba(251, 88, 37, 0.9); +} + + +table { + width: 100%; +} + + diff --git a/firebug-netexp/html/index_template.html b/firebug-netexp/html/index_template.html new file mode 100644 index 0000000..0061994 --- /dev/null +++ b/firebug-netexp/html/index_template.html @@ -0,0 +1,10 @@ + + + + + Bugs Forensics Catalogue + + + [[content]] + + diff --git a/firebug-netexp/html/netexp_to_htmlcards.py b/firebug-netexp/html/netexp_to_htmlcards.py new file mode 100644 index 0000000..e9ebbd1 --- /dev/null +++ b/firebug-netexp/html/netexp_to_htmlcards.py @@ -0,0 +1,116 @@ +import sys, csv, json, os, re + +def emit_header(browser, exporter, data_time): + str_s = '
' + str_s += 'Date:
' + str_s += data_time + '\n' + str_s += '
\n' + str_s += 'Browser:
' + str_s += browser['name']+ '-' + browser['version'] + '\n' + str_s += '
\n' + str_s += 'Exporter:
' + str_s += exporter['name']+ '-' + exporter['version'] + '\n' + str_s += '
\n' + str_s += '
\n' + return str_s + + +def emit_bug(b): + + bug = b['bug'] + str_s = '
' + + # bug + str_s += 'Bug' + str_s += '
' + str_s += '
\n' + str_s += 'type: ' + bug['type'] + '
' + 'property: ' + bug['name'] + str_s += '
' + + str_s += '
' + + # request + req = b['request'] + str_s += 'Request' + str_s += '
' + str_s += '
\n' + str_s += req['method'] + '
' + req['url'] + '
' + req['user_agent'] + str_s += '
' + if 'referer' in req: + str_s += '
' + str_s += 'Ref: ' + req['referer'] + str_s += '
' + str_s += '
' + + str_s += '
' + + # response + res = b['response'] + str_s += 'Response' + str_s += '
' + str_s += '
\n' + if 'server_agent' in res: + str_s += str(res['status']) + '
' + str(res['server_ip']) + '
' + res['server_agent'] + '
' + else: + str_s += str(res['status']) + '
' + str(res['server_ip']) + '
' + str_s += '
' + + str_s += '
' + + # content + c = b['content'] + str_s += 'Content' + str_s += '
' + str_s += '
\n' + str_s += 'mime type: ' + c['mime_type'] + '
' + str_s += 'size: ' + str(c['size']) + '
' + + if len(c['data']) > 500: + str_s += '
\n' + str_s += html_escape(c['data']) + '\n' + str_s += '
\n' + + str_s += '
' + str_s += '
' + return str_s + +def html_escape(text): + html_escape_table = { + "&": "&", + '"': """, + "'": "'", + ">": ">", + "<": "<", + "[": "[", + "]": "]" + } + return "".join(html_escape_table.get(c,c) for c in text) + +if __name__ == '__main__': + + fp = sys.stdin + try: + data = json.loads(fp.read()) + except Exception, ee: + sys.exit('Error loading data... Aborting.') + + try: + template = open(os.path.join('.', 'index_template.html'), 'r+'); + except: + print('error opening template file. aborting...'); + sys.exit(0); + + content = "" + content += emit_header(data['browser'], data['exporter'], data['date_time']) + + bugs = data['data'] + + for b in bugs: + card = "
\n" + card += emit_bug(b) + card += "
\n" + content += card + + html = template.read().replace('[[content]]', content); + + print (html).encode("utf-8") \ No newline at end of file diff --git a/ghost-bugs/bugs_pretty.py b/ghost-bugs/bugs_pretty.py new file mode 100644 index 0000000..31ee586 --- /dev/null +++ b/ghost-bugs/bugs_pretty.py @@ -0,0 +1,6 @@ +#!/usr/bin/python2.7 +import sys, json +if __name__ == '__main__': + fp = sys.stdin + data = json.loads(fp.read()) + print json.dumps(data, indent=2, separators=(',',':')) \ No newline at end of file