stat aggregation + some other stuff

2 files changed, 201 insertions, 66 deletions

diff --git a/README.md b/README.md
index 609abd6..fe146c6 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-Script to help make sense of log output from popular SSH honeypot [Cowrie](https://github.com/cowrie/cowrie/). I think I got carried away with the specialized filtering settings. Only tested on my setup. Untested with Telnet. YMMV
+Script to help make sense of log output from popular SSH honeypot [Cowrie](https://github.com/cowrie/cowrie/). I think I got carried away with the specialized filtering settings. Untested with Telnet or proxy mode. YMMV
 
 # Usage
 
@@ -12,24 +12,22 @@ cowrieder.py <options> /var/log/cowrie/
 Options:
 
 ```
-  -h, --help            show this help message and exit
-  -v, --verbose         Make the output more verbose (print abridged logs for all logins where commands are run, not just the interesting ones)
-  -V, --verboser        Make the output even more verbose (print timestamp, event id, and message (if applicable) for all events)
-  -W, --verbosest       Make the output even more verboserer (just group the input log by session id and dump it to pretty-printed JSON)
-  -P, --pings           Include only connections where no authentication attempt was made
-  -F, --fails           Include only failed logins
-  -L, --logins          Include only successful logins where no commands were run
-  -C, --commands        Include only logins where only this list of basic test commands were run: ['uname', 'nproc']
-  -E, --eventful        Include only the cool interesting sessions that aren't any of that other stuff
-  -r, --redact-ip       Redact IP addresses by converting them to 16-character hashes
-  -i, --filter-ip FILTER_IP
-                        Filter data to a specific IP or censored hash thereof
-  -c, --no-color        Don't color-code terminal output
-  -s, --stats           Print aggregated statistics
-  -S, --only-stats      Print only aggregated statistics
+-h, --help            show this help message and exit
+-v, --verbose         Make the output more verbose (print abridged logs for all logins where commands are run, not just the interesting ones)
+-V, --verboser        Make the output even more verbose (print timestamp, event id, and message (if applicable) for all events)
+-W, --verbosest       Make the output even more verboserer (dump as much information as possible as pretty-printed JSON)
+-P, --pings           Include only connections where no authentication attempt was made
+-F, --fails           Include only failed logins
+-L, --logins          Include only successful logins where no commands were run
+-C, --commands        Include only logins where only this list of basic test commands were run: ['uname', 'nproc']
+-E, --eventful        Include only the cool interesting sessions that aren't any of that other stuff
+-r, --redact-ip       Redact IP addresses by converting them to 16-character hashes
+-i, --filter-ip FILTER_IP
+                      Filter data to a specific IP, censored hash thereof, or netmask (127.0.0.1 / 0b39f7265d91e7c3 / 127.0.0.0/16)
+-c, --no-color        Don't color-code terminal output
+-s, --stats           Print aggregated statistics
+-S, --only-stats      Print only aggregated statistics
 ```
 
 # TODO
-- Aggregate more stats
-- Print stats on verbosity levels 2 and 3
 - HTML output option
\ No newline at end of file
diff --git a/cowrieder.py b/cowrieder.py
index 24c189c..f0450eb 100644
--- a/cowrieder.py
+++ b/cowrieder.py
@@ -4,6 +4,8 @@ import random
 import os
 import argparse
 import hashlib
+import ipaddress
+from pprint import pprint
 
 cmdping_commands = ['uname', 'nproc'] # List of commands that, if used exclusively, qualify a session as a "command ping"
 
@@ -16,12 +18,12 @@ cmdping_commands = ['uname', 'nproc'] # List of commands that, if used exclusive
 # ArgParse crap coming in hot
 
 parser = argparse.ArgumentParser(
-  description="Script to parse JSON logs from popular SSH honeypot Cowrie and print them in a more readable format"
+  description="Script to parse JSON logs from popular SSH honeypot Cowrie, print them in a more readable format, and gather some statistics about them"
 )
 
-parser.add_argument('filepath')
+parser.add_argument('filepath', help="The path to the cowrie JSON log file you want to analyze, or a folder of them. If given a folder, looks for any .json file or any file starting with \"cowrie.json\" and ingests them sorted by file modification timestamp")
 
-parser.add_argument('-v', '--verbose', action='store_true', help="Make the output more verbose (print abridged logs for all logins where commands are run, not just the interesting ones)")
+parser.add_argument('-v', '--verbose', action='store_true', help="Make the output more verbose (do less to collapse less-interesting information)")
 parser.add_argument('-V', '--verboser', action='store_true', help="Make the output even more verbose (print timestamp, event id, and message (if applicable) for all events)")
 parser.add_argument('-W', '--verbosest', action='store_true', help="Make the output even more verboserer (dump as much information as possible as pretty-printed JSON)")
 
@@ -32,7 +34,7 @@ parser.add_argument('-C', '--commands', action='store_true', help='Include only
 parser.add_argument('-E', '--eventful', action='store_true', help='Include only the cool interesting sessions that aren\'t any of that other stuff')
 
 parser.add_argument('-r', '--redact-ip', action='store_true', help='Redact IP addresses by converting them to 16-character hashes')
-parser.add_argument('-i', '--filter-ip', help='Filter data to a specific IP or censored hash thereof')
+parser.add_argument('-i', '--filter-ip', help='Filter data to a specific IP, censored hash thereof, or netmask (127.0.0.1 / 0b39f7265d91e7c3 / 127.0.0.0/16)')
 
 parser.add_argument('-c', '--no-color', action='store_true', help="Don't color-code terminal output")
 
@@ -141,7 +143,7 @@ def session_cred_combos(session): # List passwords used in login attempt
         if('password' in event):
           ra.append((event['username'], event['password']))
         elif('key' in event):
-          ra.append((event['username'], []))
+          ra.append((event['username'], "[SSH passkey attempt]")) # This means that the login attempt was made using a pubkey instead of a password
   return ra
 
 def session_ran_command(session): # Was a command run during the session
@@ -179,7 +181,7 @@ def session_pingcmds(session): # Return array of all "command ping" commands use
           argv0 = event['input'][:event['input'].index(" ")]
 
         if(argv0 in cmdping_commands):
-          ra.append(argv0)
+          ra.append(event['input'])
   return ra
 
 # Loading da file(s) into da master array
@@ -190,7 +192,7 @@ def load_json_to_array(fp):
       entry = json.loads(line)
 
       if(
-        (not args.filter_ip or entry['src_ip'] == args.filter_ip or hashgen(entry['src_ip']) == args.filter_ip) # If it's not the IP we're filtering for don't even bother man
+        (not args.filter_ip or hashgen(entry['src_ip']) == args.filter_ip or ipaddress.ip_address(entry['src_ip']) in ipaddress.ip_network(args.filter_ip)) # If it's not the IP we're filtering for don't even bother man
         ):
         session_id = entry['session']
 
@@ -213,6 +215,10 @@ elif(os.path.isdir(filepath)):
 else:
   raise FileNotFoundError
 
+if(len(sessions) < 1):
+  print("No logs were loaded")
+  exit(1)
+
 # Filtering
 
 newsessions = sessions.copy()
@@ -241,12 +247,23 @@ sessions = newsessions
 
 # Statistics Aggregation
 
-login_combos = {}
+def percent_of_all_sessions(num):
+  if(verbosity < 2):
+    return int(num / len(sessions) * 100)
+  return num / len(sessions) * 100
+
+login_combos = []
+attempt_counts_by_username = {}
+attempt_counts_by_password = {}
 
-# Stats pertaining to source IPs
+cnt_pings = 0
+cnt_fails = 0
+cnt_lping = 0
+cnt_cmdping = 0
+cnt_eventful = 0
+cnt_filessaved = 0
 
-attempt_counts_by_ip = {}
-ips_by_attempt_count = {}
+attempt_counts_by_ip = {} # Stats pertaining to source IPs
 def top_n_percent_of_ips_session_percent(n): # pretty self explanatory. Pass a float between 0 and 1
   i = 0
   maxi = len(attempt_counts_by_ip) * n
@@ -256,52 +273,153 @@ def top_n_percent_of_ips_session_percent(n): # pretty self explanatory. Pass a f
       break
     c += attempt_counts_by_ip[ip]
     i += 1
-  return int(c / len(sessions) * 100)
+  return percent_of_all_sessions(c)
+
+def update_cnt_by(item, cnt_by):
+  if(item not in cnt_by):
+    cnt_by.update({item: 1})
+  else:
+    cnt_by[item] += 1
 
 for s_id in sessions:
   s = sessions[s_id]
   ip = session_ip(s)
-  if(ip not in attempt_counts_by_ip):
-    attempt_counts_by_ip.update({ip: 1})
-  else:
-    attempt_counts_by_ip[ip] += 1
+  # Count how many attempts this source IP has made
+  update_cnt_by(ip, attempt_counts_by_ip)
+  # Create list of credential combos used
+  for a in session_cred_combos(s):
+    login_combos.append(a)
+  # Count sessions by type
+  if(not session_attempted_login(s)):
+    cnt_pings += 1
+  elif(session_attempted_login(s) and not session_login_succeeded(s)):
+    cnt_fails += 1
+  elif(session_login_succeeded(s) and not session_ran_command(s)):
+    cnt_lping += 1
+  elif(session_is_cmdping(s)):
+    cnt_cmdping += 1
+  elif(session_ran_command(s) and not session_is_cmdping(s)):
+    cnt_eventful += 1
+  if(session_saved_file(s)):
+    cnt_filessaved += 1
+
+for a in login_combos:
+  update_cnt_by(a[0], attempt_counts_by_username)
+  update_cnt_by(a[1], attempt_counts_by_password)
+
+def cnt_by_to_by_cnt(cnt_by):
+  by_cnt = {}
+  for item in cnt_by:
+    if(cnt_by[item] not in by_cnt):
+      by_cnt.update({cnt_by[item]: []})
+    by_cnt[cnt_by[item]].append(item)
+  return by_cnt.copy()
 
 attempt_counts_by_ip = dict(sorted(attempt_counts_by_ip.items(), key=lambda item: item[1], reverse=True))
-for ip in attempt_counts_by_ip:
-  if(attempt_counts_by_ip[ip] not in ips_by_attempt_count):
-    ips_by_attempt_count.update({attempt_counts_by_ip[ip]: []})
-  ips_by_attempt_count[attempt_counts_by_ip[ip]].append(ip)
-
-def printstats():
-  print("Source IP addresses by attempt count:")
-  for num in ips_by_attempt_count:
-    out = (" " * (len(str(next(iter(ips_by_attempt_count)))) - len(str(num)))) + str(num) + ": "
-    ipcount = 0
-    for ip in ips_by_attempt_count[num]:
-      if(verbosity == 0 and ipcount >= 3):
-        out += str(len(ips_by_attempt_count[num])) + " more"
+attempt_counts_by_username = dict(sorted(attempt_counts_by_username.items(), key=lambda item: item[1], reverse=True))
+attempt_counts_by_password = dict(sorted(attempt_counts_by_password.items(), key=lambda item: item[1], reverse=True))
+
+ips_by_attempt_count = cnt_by_to_by_cnt(attempt_counts_by_ip)
+usernames_by_attempt_count = cnt_by_to_by_cnt(attempt_counts_by_username)
+passwords_by_attempt_count = cnt_by_to_by_cnt(attempt_counts_by_password)
+
+# Printing logic coming up
+
+def print_by_cnt(by_cnt, is_ip = False, truncate = 3):
+  for num in by_cnt:
+    out = (" " * (len(str(next(iter(by_cnt)))) - len(str(num)))) + str(num) + ": "
+    cnt = 0
+    for item in by_cnt[num]:
+      if(verbosity == 0 and cnt >= truncate):
+        out += str(len(by_cnt[num])) + " more"
         break
-      out += ppip(ip, ttycolor=True)
-      if(ipcount < len(ips_by_attempt_count[num]) - 1):
+      if(is_ip):
+        out += ppip(item, ttycolor=True)
+      else:
+        out += str(item)
+      if(cnt < len(by_cnt[num]) - 1):
         out += ", "
-      ipcount += 1
+      cnt += 1
     print(out)
-  print("Of a total " + str(len(sessions)) + " sessions:")
-  print("The top 50% of IPs were responsible for " + str(top_n_percent_of_ips_session_percent(0.5)) + "%")
-  print("The top 10% of IPs were responsible for " + str(top_n_percent_of_ips_session_percent(0.1)) + "%")
-  print("The top 1% of IPs were responsible for " + str(top_n_percent_of_ips_session_percent(0.01)) + "%")
-  print("The top single IP was responsible for " + str(int(attempt_counts_by_ip[next(iter(attempt_counts_by_ip))] / len(sessions) * 100)) + "%")
 
-# Printing logic coming up
+def print_percentage_stats():
+  if(len(attempt_counts_by_ip) > 1):
+    print("\n--- Of a total " + str(len(sessions)) + " sessions from " + str(len(attempt_counts_by_ip)) + " IPs: ---")
+    print()
+    if(verbosity > 0):
+      print("The top 75% of IPs were responsible for " + str(top_n_percent_of_ips_session_percent(0.75)) + "%")
+    print("The top 50% of IPs were responsible for " + str(top_n_percent_of_ips_session_percent(0.5)) + "%")
+    if(verbosity > 0):
+      print("The top 25% of IPs were responsible for " + str(top_n_percent_of_ips_session_percent(0.25)) + "%")
+    print("The top 10% of IPs were responsible for " + str(top_n_percent_of_ips_session_percent(0.1)) + "%")
+    print("The top 1% of IPs were responsible for " + str(top_n_percent_of_ips_session_percent(0.01)) + "%")
+    print("The top single IP was responsible for " + str(percent_of_all_sessions(attempt_counts_by_ip[next(iter(attempt_counts_by_ip))])) + "%")
+  else:
+    num = next(iter(ips_by_attempt_count))
+    print("\n--- Of a total " + str(num) + " sessions from " + ips_by_attempt_count[num][0] + ": ---")
+  print()
+  if(include_empty_pings):
+    print(str(percent_of_all_sessions(cnt_pings)) + "% (" + str(cnt_pings) + ") made no attempt to log in")
+  if(include_failed_logins):
+    print(str(percent_of_all_sessions(cnt_fails)) + "% (" + str(cnt_fails) + ") attempted to log in and failed")
+  if(include_login_pings):
+    print(str(percent_of_all_sessions(cnt_lping)) + "% (" + str(cnt_lping) + ") logged in and then left without doing anything")
+  if(include_command_pings):
+    print(str(percent_of_all_sessions(cnt_cmdping)) + "% (" + str(cnt_cmdping) + ") logged in and then ran only a subset of these commands: " + str(cmdping_commands))
+  if(include_eventful):
+    print(str(percent_of_all_sessions(cnt_eventful)) + "% (" + str(cnt_eventful) + ") logged in and then did something else")
+    print(str(percent_of_all_sessions(cnt_filessaved)) + "% (" + str(cnt_filessaved) + ") saved a file")
+
+if(verbosity == 3): # Maximum verbosity: literally just dump everything we have as pretty-printed JSON
+  outjson = {}
 
-if(verbosity == 3): # Maximum verbosity: literally just dump the now-sorted-by-session logs as pretty-printed JSON
-  percentiles = {
-    "50%": top_n_percent_of_ips_session_percent(0.5),
-    "10%": top_n_percent_of_ips_session_percent(0.1),
-    "1%": top_n_percent_of_ips_session_percent(0.01),
-    "1": int(attempt_counts_by_ip[next(iter(attempt_counts_by_ip))] / len(sessions) * 100),
-  }
-  print(json.dumps([sessions, ips_by_attempt_count, attempt_counts_by_ip, percentiles], indent=4))
+  if(not args.only_stats):
+    outjson.update({'sessions': sessions})
+  if(print_stats):
+    percentages = {
+      'top_90_percent_ips': top_n_percent_of_ips_session_percent(0.9),
+      'top_80_percent_ips': top_n_percent_of_ips_session_percent(0.8),
+      'top_75_percent_ips': top_n_percent_of_ips_session_percent(0.75),
+      'top_70_percent_ips': top_n_percent_of_ips_session_percent(0.7),
+      'top_60_percent_ips': top_n_percent_of_ips_session_percent(0.6),
+      'top_50_percent_ips': top_n_percent_of_ips_session_percent(0.5),
+      'top_40_percent_ips': top_n_percent_of_ips_session_percent(0.4),
+      'top_30_percent_ips': top_n_percent_of_ips_session_percent(0.3),
+      'top_25_percent_ips': top_n_percent_of_ips_session_percent(0.25),
+      'top_20_percent_ips': top_n_percent_of_ips_session_percent(0.2),
+      'top_10_percent_ips': top_n_percent_of_ips_session_percent(0.1),
+      'top_5_percent_ips': top_n_percent_of_ips_session_percent(0.05),
+      'top_1_percent_ips': top_n_percent_of_ips_session_percent(0.01),
+      'top_ip': percent_of_all_sessions(attempt_counts_by_ip[next(iter(attempt_counts_by_ip))]),
+      'no_login': percent_of_all_sessions(cnt_pings),
+      'failed_login': percent_of_all_sessions(cnt_fails),
+      'login_no_action': percent_of_all_sessions(cnt_lping),
+      'login_uneventful': percent_of_all_sessions(cnt_cmdping),
+      'eventful': percent_of_all_sessions(cnt_eventful),
+      'saved_file': percent_of_all_sessions(cnt_filessaved)
+    }
+    counts = {
+      'no_login': cnt_pings,
+      'failed_login': cnt_fails,
+      'login_no_action': cnt_lping,
+      'login_uneventful': cnt_cmdping,
+      'eventful': cnt_eventful,
+      'saved_file': cnt_filessaved
+    }
+    all_stats = {
+      'percentages': percentages,
+      'counts': counts,
+      'login_combos': login_combos,
+      'attempt_counts_by_ip': attempt_counts_by_ip,
+      'attempt_counts_by_username': attempt_counts_by_username,
+      'attempt_counts_by_password': attempt_counts_by_password,
+      'ips_by_attempt_count': ips_by_attempt_count,
+      'usernames_by_attempt_count': usernames_by_attempt_count,
+      'passwords_by_attempt_count': passwords_by_attempt_count
+    }
+    outjson.update({'stats': all_stats})
+
+  print(json.dumps(outjson, indent=4))
   exit()
 
 elif(verbosity == 2): # High verbosity mode: print timestamp, event type and message (if applicable) for every event. I could probably do better but I'm lazy
@@ -317,7 +435,17 @@ elif(verbosity == 2): # High verbosity mode: print timestamp, event type and mes
         else:
           print(entry["timestamp"] + " > " + eventid + (" " * (29 - len(eventid))) + ": " + entry['message'])
   if(print_stats):
-    print("TODO -- this shit's gonna be verbose as fuck man")
+    print("\n--- Credential combinations used: ---")
+    pprint(login_combos, sort_dicts=False)
+    print("\n--- Usernames and number of times they were tried: ---")
+    pprint(attempt_counts_by_username, sort_dicts=False)
+    print("\n--- Passwords and number of times they were tried: ---")
+    pprint(attempt_counts_by_password, sort_dicts=False)
+    print("\n--- Source IPs and number of connections originating from them: ---")
+    pprint(attempt_counts_by_ip, sort_dicts=False)
+    print_percentage_stats()
+
+
 
 else: # Default (low) verbosity: just the good stuff
   if(not args.only_stats):
@@ -370,4 +498,13 @@ else: # Default (low) verbosity: just the good stuff
       else: # Printing logic for successful logins in which no commands were run
         print(timestamp + " > " + ppip(ip, True, True) + " > " + ccgen("GREEN") + "Login" + ccgen() + " with credentials: " + str(session_cred_combos(session)))
   if(print_stats):
-    printstats()
+    print("\n--- Source IP addresses by attempt count: ---")
+    print_by_cnt(ips_by_attempt_count, True)
+    if(verbosity > 0):
+      print("\n--- Credential combos used: ---")
+      print(str(login_combos))
+    print("\n--- Usernames by attempt count: ---")
+    print_by_cnt(usernames_by_attempt_count, truncate=10)
+    print("\n--- Passwords by attempt count: ---")
+    print_by_cnt(passwords_by_attempt_count, truncate=10)
+    print_percentage_stats()