import json import sys import random import os import argparse import hashlib import ipaddress from pprint import pprint import time start_time = time.time() cmdping_commands = ['uname', 'nproc'] # List of commands that, if used exclusively, qualify a session as a "command ping" pubkey_attempt_str = "[SSH_PUBKEY_ATTEMPT]" # Definitions if you're reading this code for some reason: # - An empty ping is when a client connects and then does nothing, not even a log in attempt, presumably just to check if an SSH server exists # - A login ping is when a client logs in successfully and leaves without doing anything # - A "command ping" is when a client logs in and then only runs the handful of common test/query commands listed above. These make up such a # large percentage of sessions where any command is run that I felt the need to create a distinction in order to make finding the most interesting log entires easier # ArgParse crap coming in hot parser = argparse.ArgumentParser( description="Script to parse JSON logs from popular SSH honeypot Cowrie, print them in a more readable format, and gather some statistics about them" ) parser.add_argument('filepath', help="The path to the cowrie JSON log file you want to analyze, or a folder of them. If given a folder, looks for any .json file or any file starting with \"cowrie.json\" and ingests them sorted by file modification timestamp") parser.add_argument('-v', '--verbose', action='store_true', help="Make the output more verbose (do less to collapse less-interesting information)") parser.add_argument('-V', '--verboser', action='store_true', help="Make the output even more verbose (one line per event)") parser.add_argument('-W', '--verbosest', action='store_true', help="Make the output even more verboserer (dump as much information as possible as pretty-printed JSON)") parser.add_argument('-P', '--pings', action='store_true', help='Include only connections where no authentication attempt was made') parser.add_argument('-F', '--fails', action='store_true', help='Include only failed logins') parser.add_argument('-L', '--logins', action='store_true', help='Include only successful logins where no commands were run') parser.add_argument('-C', '--commands', action='store_true', help='Include only logins where only this list of basic test commands were run: ' + str(cmdping_commands)) parser.add_argument('-E', '--eventful', action='store_true', help='Include only the cool interesting sessions that aren\'t any of that other stuff') parser.add_argument('-u', '--filter-username', help='Filter data to uses of a specific username') parser.add_argument('-p', '--filter-password', help='Filter data to uses of a specific password. Pass "'+pubkey_attempt_str+'" to filter for attempts to log in using an SSH public key instead of a password, which does happen') parser.add_argument('-z', '--filter-sensor', help='Filter data to a certain \"sensor\". Useful if Cowrie is running in proxy mode to multiple endpoints') parser.add_argument('-i', '--filter-ip', help='Filter data to a specific IP, censored hash thereof, or netmask (127.0.0.1 / 0b39f7265d91e7c3 / 127.0.0.0/16)') parser.add_argument('-r', '--redact-ip', action='store_true', help='Redact IP addresses by converting them to 16-character hashes') parser.add_argument('-c', '--no-color', action='store_true', help="Don't color-code terminal output") parser.add_argument('-s', '--stats', action='store_true', help='Print aggregated statistics') parser.add_argument('-S', '--only-stats', action='store_true', help='Print only aggregated statistics') def ccgen(seed=""): # Color code generator if(not os.isatty(1) or args.no_color): # Do not output color code at all if output is not a TTY that can accept them return "" match(seed): # Match color keywords to control codes case "RED": return "\x1b[31m" case "GREEN": return "\x1b[32m" case "YELLOW": return "\x1b[33m" case "BLUE": return "\x1b[34m" case "PURPLE": return "\x1b[35m" case "CYAN": return "\x1b[36m" case "GREY": return "\x1b[37m" case "RESET": return "\x1b[0m" if(seed): # If seed is provided, use it to generate a color random.seed(seed) return "\x1b[38;2;{r};{g};{b}m".format(r=random.randrange(50, 256), g=random.randrange(50, 256), b=random.randrange(50, 256)) return "\x1b[0m" def ttylog(mystring): #mystring if(os.isatty(1)): print(mystring) def ppip(addr, spacing = False, ttycolor = False): # Pretty-print an address with colour and, if it's IPv4 or hashed, a consistent width rs = addr if(ttycolor): rs = ccgen(rs) + rs + ccgen() if(len(addr) <= 16 and spacing): rs = rs + (" " * (15 - len(addr))) return rs def hashgen(seed, length=16): if(length % 2): raise Warning("Due to strange behaviour on the part of the hexdigest() function, length should be a multiple of 2") return hashlib.shake_256(str(seed).encode("utf-8")).hexdigest(int(length / 2)) args = parser.parse_args() verbosity = 0 if(args.verbosest): verbosity = 3 elif(args.verboser): verbosity = 2 elif(args.verbose): verbosity = 1 filepath = args.filepath blankmode = (args.pings or args.fails or args.logins or args.commands or args.eventful) include_empty_pings = (not blankmode or args.pings) include_failed_logins = (not blankmode or args.fails) include_login_pings = (not blankmode or args.logins) include_command_pings = (not blankmode or args.commands) include_eventful = (not blankmode or args.eventful) print_stats = args.stats or args.only_stats sessions = {} # Dict - session_id : list of events in session def session_ip(session): # Grab IP from session return session[0]['src_ip'] def session_timestamp(session): # Grab initial timestamp of session return session[0]['timestamp'] def session_duration(session): # Grab duration of session -- if session never closed return -1 if(session[-1]['eventid'] == "cowrie.session.closed"): return float(session[-1]['duration']) return -1 def session_sensor(session): # Grab sensor from session return session[0]['sensor'] def session_attempted_login(session): # Determine whether a login was attempted for event in session: if(event['eventid'] == "cowrie.login.failed" or event['eventid'] == "cowrie.login.success"): return True return False def session_login_succeeded(session): # Determine whether a login attempt was successful for event in session: if(event['eventid'] == "cowrie.login.success"): return True return False def session_cred_combos(session): # List passwords used in login attempt ra = [] if(session_attempted_login(session)): for event in session: if(event['eventid'] == "cowrie.login.failed" or event['eventid'] == "cowrie.login.success"): if('password' in event): ra.append((event['username'], event['password'])) elif('key' in event): ra.append((event['username'], pubkey_attempt_str)) # This means that the login attempt was made using a pubkey instead of a password return ra def session_ran_command(session): # Was a command run during the session for event in session: if(event['eventid'] == "cowrie.command.input"): return True return False def session_saved_file(session): # Were any files saved to disk during the session for event in session: if(event['eventid'] == "cowrie.session.file_download"): return True return False def session_is_cmdping(session): # Determine if a session qualifies as a "command ping" if(session_ran_command(session)): for event in session: if(event['eventid'] == "cowrie.command.input"): argv0 = event['input'] # Isolate the name of the command if(" " in argv0): argv0 = event['input'][:event['input'].index(" ")] if(argv0 not in cmdping_commands): return False return True return False def session_pingcmds(session): # Return array of all "command ping" commands used in a session ra = [] if(session_ran_command(session)): for event in session: if(event['eventid'] == "cowrie.command.input"): argv0 = event['input'] # Isolate the name of the command if(" " in argv0): argv0 = event['input'][:event['input'].index(" ")] if(argv0 in cmdping_commands): ra.append(event['input']) return ra # Loading da file(s) into da master array def load_json_to_array(fp): with open(fp, "r") as logfile: for line in logfile: entry = json.loads(line) if( (not args.filter_ip or hashgen(entry['src_ip']) == args.filter_ip or ipaddress.ip_address(entry['src_ip']) in ipaddress.ip_network(args.filter_ip)) and (not args.filter_sensor or args.filter_sensor == entry['sensor']) # If it's not the IP/sensor we're filtering for don't even bother man ): session_id = entry['session'] if(session_id not in sessions): sessions.update({session_id: [entry]}) else: sessions[session_id].append(entry) if(os.path.isfile(filepath)): load_json_to_array(filepath) elif(os.path.isdir(filepath)): # run load_json_to_array for every json file in the directory files = {} for f in os.listdir(filepath): if(f[:11] == "cowrie.json" or f[-5:] == ".json"): fi = os.path.join(filepath, f) files.update({fi: os.path.getmtime(fi)}) for g in dict(sorted(files.items(), key=lambda item: item[1])): load_json_to_array(g) else: raise FileNotFoundError if(len(sessions) < 1): print("No logs were loaded") exit(1) # Filtering newsessions = sessions.copy() for session_id in sessions: session = sessions[session_id] if(args.redact_ip): src_ip = session_ip(session) dst_ip = session[0]['dst_ip'] for i, event in enumerate(session): for a in event: if(isinstance(event[a], str)): newsessions[session_id][i][a] = event[a].replace('127.0.0.1', 'localhost').replace(src_ip, hashgen(src_ip)).replace(dst_ip, 'Host IP') s_u = [] s_p = [] for a in session_cred_combos(session): s_u.append(a[0]) s_p.append(a[1]) if( (blankmode and ( (not include_empty_pings and not session_attempted_login(session)) or (not include_failed_logins and session_attempted_login(session) and not session_login_succeeded(session)) or (not include_login_pings and session_login_succeeded(session) and not session_ran_command(session)) or (not include_command_pings and session_is_cmdping(session)) or (not include_eventful and session_ran_command(session) and not session_is_cmdping(session)))) or (args.filter_username and args.filter_username not in s_u) or (args.filter_password and args.filter_password not in s_p)): newsessions.pop(session_id) sessions = newsessions # Statistics Aggregation def percent_of_all_sessions(num): if(verbosity < 2): return int(num / len(sessions) * 100) return num / len(sessions) * 100 login_combos = [] attempt_counts_by_username = {} attempt_counts_by_password = {} passwords_by_username = {} cnt_pings = 0 cnt_fails = 0 cnt_lping = 0 cnt_cmdping = 0 cnt_eventful = 0 cnt_filessaved = 0 login_combos_by_ip = {} attempt_counts_by_ip = {} # Stats pertaining to source IPs def top_n_percent_of_ips_session_percent(n): # pretty self explanatory. Pass a float between 0 and 1 i = 0 maxi = len(attempt_counts_by_ip) * n c = 0 for ip in attempt_counts_by_ip: if(i > maxi): break c += attempt_counts_by_ip[ip] i += 1 return percent_of_all_sessions(c) def update_cnt_by(item, cnt_by): if(item not in cnt_by): cnt_by.update({item: 1}) else: cnt_by[item] += 1 for s_id in sessions: s = sessions[s_id] ip = session_ip(s) # Count how many attempts this source IP has made update_cnt_by(ip, attempt_counts_by_ip) # Create list of credential combos used for a in session_cred_combos(s): login_combos.append(a) # Group credential combos by IP if(ip not in login_combos_by_ip): login_combos_by_ip.update({ip: session_cred_combos(s)}) else: for a in session_cred_combos(s): login_combos_by_ip[ip].append(a) # Count sessions by type if(not session_attempted_login(s)): cnt_pings += 1 elif(session_attempted_login(s) and not session_login_succeeded(s)): cnt_fails += 1 elif(session_login_succeeded(s) and not session_ran_command(s)): cnt_lping += 1 elif(session_is_cmdping(s)): cnt_cmdping += 1 elif(session_ran_command(s) and not session_is_cmdping(s)): cnt_eventful += 1 if(session_saved_file(s)): cnt_filessaved += 1 for a in login_combos: update_cnt_by(a[0], attempt_counts_by_username) update_cnt_by(a[1], attempt_counts_by_password) if(a[0] not in passwords_by_username): passwords_by_username.update({a[0]: [a[1]]}) elif(a[1] not in passwords_by_username[a[0]]): passwords_by_username[a[0]].append(a[1]) def cnt_by_to_by_cnt(cnt_by): by_cnt = {} for item in cnt_by: if(cnt_by[item] not in by_cnt): by_cnt.update({cnt_by[item]: []}) by_cnt[cnt_by[item]].append(item) return by_cnt.copy() attempt_counts_by_ip = dict(sorted(attempt_counts_by_ip.items(), key=lambda item: item[1], reverse=True)) attempt_counts_by_username = dict(sorted(attempt_counts_by_username.items(), key=lambda item: item[1], reverse=True)) attempt_counts_by_password = dict(sorted(attempt_counts_by_password.items(), key=lambda item: item[1], reverse=True)) passwords_by_username = dict(sorted(passwords_by_username.items(), key=lambda item: len(item[1]), reverse=True)) login_combos_by_ip = dict(sorted(login_combos_by_ip.items(), key=lambda item: len(item[1]), reverse=True)) ips_by_attempt_count = cnt_by_to_by_cnt(attempt_counts_by_ip) usernames_by_attempt_count = cnt_by_to_by_cnt(attempt_counts_by_username) passwords_by_attempt_count = cnt_by_to_by_cnt(attempt_counts_by_password) log_parsing_time = time.time() - start_time # Printing logic coming up def print_dict_of_lists(by_cnt, is_ip = False, key_is_ip = False, truncate = 3, limit = -1): name_max_length = 0 for item in by_cnt: if(name_max_length < len(str(item))): name_max_length = len(str(item)) entries_printed = 0 for num in by_cnt: if(limit > 0 and entries_printed >= limit): break if(key_is_ip): out = (" " * (name_max_length - len(str(num)))) + ppip(str(num), ttycolor=True) + ": " else: out = (" " * (name_max_length - len(str(num)))) + str(num) + ": " cnt = 0 for item in by_cnt[num]: if(verbosity == 0 and cnt >= truncate): out += str(len(by_cnt[num])) + " more" break if(is_ip): out += ppip(item, ttycolor=True) else: out += str(item) if(cnt < len(by_cnt[num]) - 1): out += ", " cnt += 1 print(out) entries_printed += 1 def print_percentage_stats(): if(len(attempt_counts_by_ip) > 1): print("\n--- Of a total " + str(len(sessions)) + " sessions from " + str(len(attempt_counts_by_ip)) + " IPs: ---") print() if(verbosity > 0): print("The top 75% of IPs were responsible for " + str(top_n_percent_of_ips_session_percent(0.75)) + "%") print("The top 50% of IPs were responsible for " + str(top_n_percent_of_ips_session_percent(0.5)) + "%") if(verbosity > 0): print("The top 25% of IPs were responsible for " + str(top_n_percent_of_ips_session_percent(0.25)) + "%") print("The top 10% of IPs were responsible for " + str(top_n_percent_of_ips_session_percent(0.1)) + "%") print("The top 1% of IPs were responsible for " + str(top_n_percent_of_ips_session_percent(0.01)) + "%") print("The top single IP was responsible for " + str(percent_of_all_sessions(attempt_counts_by_ip[next(iter(attempt_counts_by_ip))])) + "%") print() else: num = next(iter(ips_by_attempt_count)) print("\n--- Of a total " + str(num) + " sessions from " + ips_by_attempt_count[num][0] + ": ---") if(include_empty_pings): print(str(percent_of_all_sessions(cnt_pings)) + "% (" + str(cnt_pings) + ") made no attempt to log in") if(include_failed_logins): print(str(percent_of_all_sessions(cnt_fails)) + "% (" + str(cnt_fails) + ") attempted to log in and failed") if(include_login_pings): print(str(percent_of_all_sessions(cnt_lping)) + "% (" + str(cnt_lping) + ") logged in and then left without doing anything") if(include_command_pings): print(str(percent_of_all_sessions(cnt_cmdping)) + "% (" + str(cnt_cmdping) + ") logged in and then ran only a subset of these commands: " + str(cmdping_commands)) if(include_eventful): print(str(percent_of_all_sessions(cnt_eventful)) + "% (" + str(cnt_eventful) + ") logged in and then did something else") print(str(percent_of_all_sessions(cnt_filessaved)) + "% (" + str(cnt_filessaved) + ") saved a file") if(verbosity == 3): # Maximum verbosity: literally just dump everything we have as pretty-printed JSON outjson = {} if(not args.only_stats): outjson.update({'sessions': sessions}) if(print_stats): percentages = { 'top_90_percent_ips': top_n_percent_of_ips_session_percent(0.9), 'top_80_percent_ips': top_n_percent_of_ips_session_percent(0.8), 'top_75_percent_ips': top_n_percent_of_ips_session_percent(0.75), 'top_70_percent_ips': top_n_percent_of_ips_session_percent(0.7), 'top_60_percent_ips': top_n_percent_of_ips_session_percent(0.6), 'top_50_percent_ips': top_n_percent_of_ips_session_percent(0.5), 'top_40_percent_ips': top_n_percent_of_ips_session_percent(0.4), 'top_30_percent_ips': top_n_percent_of_ips_session_percent(0.3), 'top_25_percent_ips': top_n_percent_of_ips_session_percent(0.25), 'top_20_percent_ips': top_n_percent_of_ips_session_percent(0.2), 'top_10_percent_ips': top_n_percent_of_ips_session_percent(0.1), 'top_5_percent_ips': top_n_percent_of_ips_session_percent(0.05), 'top_1_percent_ips': top_n_percent_of_ips_session_percent(0.01), 'top_ip': percent_of_all_sessions(attempt_counts_by_ip[next(iter(attempt_counts_by_ip))]), 'no_login': percent_of_all_sessions(cnt_pings), 'failed_login': percent_of_all_sessions(cnt_fails), 'login_no_action': percent_of_all_sessions(cnt_lping), 'login_uneventful': percent_of_all_sessions(cnt_cmdping), 'eventful': percent_of_all_sessions(cnt_eventful), 'saved_file': percent_of_all_sessions(cnt_filessaved) } counts = { 'no_login': cnt_pings, 'failed_login': cnt_fails, 'login_no_action': cnt_lping, 'login_uneventful': cnt_cmdping, 'eventful': cnt_eventful, 'saved_file': cnt_filessaved } all_stats = { 'percentages': percentages, 'counts': counts, 'login_combos': login_combos, 'attempt_counts_by_ip': attempt_counts_by_ip, 'attempt_counts_by_username': attempt_counts_by_username, 'attempt_counts_by_password': attempt_counts_by_password, 'ips_by_attempt_count': ips_by_attempt_count, 'usernames_by_attempt_count': usernames_by_attempt_count, 'passwords_by_attempt_count': passwords_by_attempt_count, 'passwords_by_username': passwords_by_username, 'login_combos_by_ip': login_combos_by_ip, 'log_parsing_time': log_parsing_time } outjson.update({'stats': all_stats}) print(json.dumps(outjson, indent=4)) exit() elif(verbosity == 2): # High verbosity mode: print timestamp, event type and message (if applicable) for every event. I could probably do better but I'm lazy if(not args.only_stats): for session_id in sessions: session = sessions[session_id] ip = session_ip(session) print("\n--- Session " + ccgen(session_id) + session_id + ccgen() + " from " + ppip(ip, ttycolor=True) + " to " + session_sensor(session) + " ---") for entry in session: eventid = entry['eventid'] if(entry['eventid'] == "cowrie.session.params"): print(entry["timestamp"] + " > " + eventid + (" " * (29 - len(eventid))) + ": ARCH: " + entry["arch"]) else: print(entry["timestamp"] + " > " + eventid + (" " * (29 - len(eventid))) + ": " + entry['message']) if(print_stats): print("\n--- Source IPs and number of connections originating from them: ---") pprint(attempt_counts_by_ip, sort_dicts=False) print("\n--- Credential combinations used: ---") pprint(login_combos, sort_dicts=False) print("\n--- Usernames and number of times they were tried: ---") pprint(attempt_counts_by_username, sort_dicts=False) print("\n--- Passwords and number of times they were tried: ---") pprint(attempt_counts_by_password, sort_dicts=False) print("\n--- Passwords grouped by corresponding username: ---") pprint(passwords_by_username, sort_dicts=False) print("\n--- Credential combos grouped by source IP: ---") pprint(login_combos_by_ip, sort_dicts=False) print_percentage_stats() print("Log took {}s to parse".format(str(log_parsing_time)[:4])) else: # Default (low) verbosity: just the good stuff if(not args.only_stats): for session_id in sessions: # Declaring initial variables session = sessions[session_id] ip = session_ip(session) timestamp = session_timestamp(session)[:22].replace("T", " ") # clean up the timestamp text all pretty like durationstring = "closed after " + str(session_duration(session)) + "s" if(session_duration(session) < 0): durationstring = "active as of processing" if(not session_attempted_login(session)): # Printing empty pings print(timestamp + " > " + ppip(ip, True, True) + " > Connection opened and " + durationstring) elif(session_attempted_login(session) and not session_login_succeeded(session)): # Printing failed logins print(timestamp + " > " + ppip(ip, True, True) + " > " + ccgen("RED") +"Failed login" + ccgen() + " with credentials: " + str(session_cred_combos(session))) elif(session_ran_command(session)): # Printing logic for sessions in which a command was run if(not verbosity == 1 and session_is_cmdping(session)): # If -v is not enabled, collapse "command pings" to one-liners print(timestamp + " > " + ppip(ip, True, True) + " > " + ccgen("GREEN") + "Login" + ccgen() + " with credentials: " + str(session_cred_combos(session)) + ccgen("BLUE") + " ran commands " + ccgen() + str(session_pingcmds(session))) else: # Printing logic for un-collapsed sessions where commands were run print(timestamp + " > " + ppip(ip, True, True) + " > " + ccgen("CYAN") + "Session" + ccgen() + ":") for entry in session: match(entry["eventid"]): case "cowrie.login.failed": pw = [] if("password" in entry): pw = entry["password"] print(" | Failed login: " + entry["username"] + "/" + str(pw)) case "cowrie.login.success": print(" | Successful login: " + entry["username"] + "/" + entry["password"]) case "cowrie.command.input": print(" | CMD: " + entry["input"]) case "cowrie.session.file_download": df = "" if("url" in entry): df = entry["url"] elif("destfile" in entry): df = entry["destfile"] if(verbosity == 1): print(" | Saved file " + df + " to " + entry["outfile"]) else: print(" | Saved file " + df + " to " + entry["outfile"][:40] + "...") print(" X Session " + durationstring) else: # Printing logic for successful logins in which no commands were run print(timestamp + " > " + ppip(ip, True, True) + " > " + ccgen("GREEN") + "Login" + ccgen() + " with credentials: " + str(session_cred_combos(session))) if(print_stats): if(verbosity > 0): print("\n--- Source IP addresses by attempt count: ---") print_dict_of_lists(ips_by_attempt_count, True) print("\n--- Credential combos used: ---") print(str(login_combos)) print("\n--- Usernames by attempt count: ---") print_dict_of_lists(usernames_by_attempt_count) print("\n--- Passwords by attempt count: ---") print_dict_of_lists(passwords_by_attempt_count) print("\n--- Passwords by username: ---") print_dict_of_lists(passwords_by_username) print("\n--- IPs with credential combos tried: ---") print_dict_of_lists(login_combos_by_ip, key_is_ip=True) else: print("\n--- Top source IP addresses by attempt count: ---") print_dict_of_lists(ips_by_attempt_count, True, limit = 20) print("\n--- Top usernames by attempt count: ---") print_dict_of_lists(usernames_by_attempt_count, truncate=10, limit = 20) print("\n--- Top passwords by attempt count: ---") print_dict_of_lists(passwords_by_attempt_count, truncate=10, limit = 20) print("\n--- Top usernames with passwords tried: ---") print_dict_of_lists(passwords_by_username, truncate=10, limit = 20) print("\n--- Top IPs with credential combos tried: ---") print_dict_of_lists(login_combos_by_ip, truncate=7, limit = 20, key_is_ip=True) print_percentage_stats() print("Log took {}s to parse".format(str(log_parsing_time)[:4]))