new bugs list and filter

ghost-bugs/bugs_parser.py

@@ -1,5 +1,13 @@
 import sys, json
 
+def reorder_ids(bugs):
+	result = {}
+	for key, info in bugs.iteritems():
+		id = info['aid']
+		result[str(id)] = key
+	return result
+
+
 if __name__ == '__main__':
 
 	fp = sys.stdin
@@ -10,6 +18,7 @@ if __name__ == '__main__':
 		sys.exit('Error loading data... Aborting.')
 
 	apps = data['apps']
+	bugs = reorder_ids(data['bugs'])
 
 	result = {}
 	result['ads'] = []
@@ -18,13 +27,13 @@ if __name__ == '__main__':
 	result['widgets'] = []
 	result['privacy'] = []
 
-
 	hosts = data['firstPartyExceptions']
 
 	for key, info in apps.iteritems():
-		if key in hosts:
+		k = bugs[key] 
+		if k in hosts:
 			urls = []
-			for u in hosts[key]:
+			for u in hosts[k]:
 				if info['cat'] == 'tracker':
 					result['trackers'].append(u)
 				elif info['cat'] == 'ad':

webpagetest/wpt_csv_filter.py

@@ -86,8 +86,12 @@ def run(options):
 	writer = csv.DictWriter(out_csv, fieldnames=csv_fields)
 	writer.writeheader()
 
-
+	last_seq = 0
 	for r in wpt_data:
+		seq = int(r['Sequence Number'])
+		if  seq < last_seq:
+			break
+		last_seq = seq
 		stats['total'] += 1
 		host = r['Host']
 		if any(a in host for a in bugs_data['ads']):

webpagetest/wpt_csv_filter_batch.py

@@ -32,8 +32,9 @@ def run(options):
 
 	for f in csv_files:
 		options.file = f
+		print "processing - " + f
 		wptf.run(options)
-		print "..............."
+		print ".......done........"
 
 if __name__ == '__main__':