total reorganization of the way session information is stored and retrieved + some other stuff

2 files changed, 156 insertions, 124 deletions

diff --git a/README.md b/README.md
index 76c0d22..f0fe695 100644
--- a/README.md
+++ b/README.md
@@ -11,13 +11,13 @@ Options:
 ```
 -h, --help            show this help message and exit
 -v, --verbose         Make the output more verbose (print abridged logs for all logins where commands are run, not just the interesting ones)
--vv, --verboser       Make the output even more verbose (print timestamp, event id, and message (if applicable) for all events)
--vvv, --verbosest     Make the output even more verboserer (just group the input log by session id and dump it to pretty-printed JSON)
+-V, --verboser        Make the output even more verbose (print timestamp, event id, and message (if applicable) for all events)
+-W, --verbosest       Make the output even more verboserer (just group the input log by session id and dump it to pretty-printed JSON)
 -P, --pings           Print only "empty pings"
 -F, --fails           Print only failed logins
 -L, --logins          Print only "login pings"
 -C, --commands        Print only "command pings"
--S, --sessions        Print only the cool interesting sessions that aren't any of that other stuff.
+-E, --eventful        Print only the cool interesting sessions that aren't any of that other stuff.
 -r, --redact-ip       Redact IP addresses by converting them to 16-character hashes
 -i, --filter-ip FILTER_IP
                       Filter data to a specific IP or censored hash thereof
diff --git a/cowrieder.py b/cowrieder.py
index 44051af..db549bc 100644
--- a/cowrieder.py
+++ b/cowrieder.py
@@ -40,7 +40,7 @@ def ppip(addr, spacing = False, ttycolor = False): # Pretty-print an address wit
   rs = addr
   if(ttycolor):
     rs = ccgen(rs) + rs + ccgen()
-  if(len(rs) <= 16 and spacing):
+  if(len(addr) <= 16 and spacing):
     rs = rs + (" " * (15 - len(addr)))
   return rs
 
@@ -49,6 +49,8 @@ def hashgen(seed, length=16):
     raise Warning("Due to strange behaviour on the part of the hexdigest() function, length should be a multiple of 2")
   return hashlib.shake_256(str(seed).encode("utf-8")).hexdigest(int(length / 2))
 
+# ArgParse crap coming in hot
+
 parser = argparse.ArgumentParser(
   description="Script to parse JSON logs from popular SSH honeypot Cowrie and print them in a more readable format"
 )
@@ -56,168 +58,202 @@ parser = argparse.ArgumentParser(
 parser.add_argument('filepath')
 
 parser.add_argument('-v', '--verbose', action='store_true', help="Make the output more verbose (print abridged logs for all logins where commands are run, not just the interesting ones)")
-parser.add_argument('-vv', '--verboser', action='store_true', help="Make the output even more verbose (print timestamp, event id, and message (if applicable) for all events)")
-parser.add_argument('-vvv', '--verbosest', action='store_true', help="Make the output even more verboserer (just group the input log by session id and dump it to pretty-printed JSON)")
+parser.add_argument('-V', '--verboser', action='store_true', help="Make the output even more verbose (print timestamp, event id, and message (if applicable) for all events)")
+parser.add_argument('-W', '--verbosest', action='store_true', help="Make the output even more verboserer (just group the input log by session id and dump it to pretty-printed JSON)")
 
 parser.add_argument('-P', '--pings', action='store_true', help='Print only "empty pings"')
 parser.add_argument('-F', '--fails', action='store_true', help='Print only failed logins')
 parser.add_argument('-L', '--logins', action='store_true', help='Print only "login pings"')
 parser.add_argument('-C', '--commands', action='store_true', help='Print only "command pings"')
-parser.add_argument('-S', '--sessions', action='store_true', help='Print only the cool interesting sessions that aren\'t any of that other stuff.')
+parser.add_argument('-E', '--eventful', action='store_true', help='Print only the cool interesting sessions that aren\'t any of that other stuff.')
 
 parser.add_argument('-r', '--redact-ip', action='store_true', help='Redact IP addresses by converting them to 16-character hashes')
 parser.add_argument('-i', '--filter-ip', help='Filter data to a specific IP or censored hash thereof')
 
-parser.add_argument('-a', '--stats', action='store_true', help='Print aggregated statistics')
-parser.add_argument('-j', '--json-stats', action='store_true', help='Dump aggregated statistics as JSON')
+parser.add_argument('-s', '--stats', action='store_true', help='Print aggregated statistics')
+parser.add_argument('-S', '--only-stats', action='store_true', help='Print only aggregated statistics')
 
 
 args = parser.parse_args()
-print_stats = args.stats or args.json_stats
+print_stats = args.stats or args.only_stats
+
+verbosity = 0
+if(args.verbosest):
+  verbosity = 3
+elif(args.verboser):
+  verbosity = 2
+elif(args.verbose):
+  verbosity = 1
 
 filepath = args.filepath
 
-blankmode = (args.pings or args.fails or args.logins or args.commands or args.sessions)
+blankmode = (args.pings or args.fails or args.logins or args.commands or args.eventful or args.only_stats)
 
-print_emptypings = (not blankmode or args.pings)
-print_fails      = (not blankmode or args.fails)
-print_loginpings = (not blankmode or args.logins)
-print_cmdpings   = (not blankmode or args.commands)
-print_sessions   = (not blankmode or args.sessions)
+include_empty_pings   = (not blankmode or args.pings)
+include_failed_logins = (not blankmode or args.fails)
+include_login_pings   = (not blankmode or args.logins)
+include_command_pings = (not blankmode or args.commands)
+include_eventful      = (not blankmode or args.eventful)
 
-entries = []
 sessions = {} # Dict - session_id : list of events in session
-session_ips = {} # Dict - session_id : source IP
-session_timestamps = {} # Dict - session_id : timestamp of initial connection
-session_durations = {} # Dict - session_id : duration
-session_ran_command = [] # List of sessions where a command was run 
-session_successful_login = {} # Dict - session_id : bool: was the login attempt successful
-                              # Sessions where no login attempt was made will not be present
+
+def session_ip(session): # Grab IP from session
+  return session[0]['src_ip']
+
+def session_timestamp(session): # Grab initial timestamp of session
+  return session[0]['timestamp']
+
+def session_duration(session): # Grab duration of session -- if session never closed return -1
+  if(session[-1]['eventid'] == "cowrie.session.closed"):
+    return float(session[-1]['duration'])
+  return -1
+
+def session_sensor(session): # Grab sensor from session
+  return session[0]['sensor']
+
+def session_attempted_login(session): # Determine whether a login was attempted
+  for event in session:
+    if(event['eventid'] == "cowrie.login.failed" or event['eventid'] == "cowrie.login.success"):
+      return True
+  return False
+
+def session_login_succeeded(session): # Determine whether a login attempt was successful
+  for event in session:
+    if(event['eventid'] == "cowrie.login.success"):
+      return True
+  return False
+
+def session_cred_combos(session): # List passwords used in login attempt
+  ra = []
+  if(session_attempted_login(session)):
+    for event in session:
+      if(event['eventid'] == "cowrie.login.failed" or event['eventid'] == "cowrie.login.success"):
+        if('password' in event):
+          ra.append([event['username'], event['password']])
+        elif('key' in event):
+          ra.append([event['username'], []])
+  return ra
+
+def session_ran_command(session): # Was a command run during the session
+  for event in session:
+    if(event['eventid'] == "cowrie.command.input"):
+      return True
+  return False
+
+def session_saved_file(session):
+  for event in session:
+    if(event['eventid'] == "cowrie.session.file_download"):
+      return True
+  return False
+
 cmdping_commands = ['uname', 'nproc', 'echo'] # List of commands that, if used exclusively, qualify a session as a "command ping"
-session_not_cmdpings = [] # List of sessions where a command not in cmdping_commands was run
-session_cmdping_commands = {} # Dict - session_id : list of "command ping" commands used in session
+
+def session_is_cmdping(session): # Determine if a session qualifies as a "command ping"
+  if(session_ran_command(session)):
+    for event in session:
+      if(event['eventid'] == "cowrie.command.input"):
+        argv0 = event['input'] # Isolate the name of the command
+        if(" " in argv0):
+          argv0 = event['input'][:event['input'].index(" ")]
+
+        if(argv0 not in cmdping_commands):
+          return False
+    return True
+  return False
+
+def session_pingcmds(session): # Return array of all "command ping" commands used in a session
+  ra = []
+  if(session_ran_command(session)):
+    for event in session:
+      if(event['eventid'] == "cowrie.command.input"):
+        argv0 = event['input'] # Isolate the name of the command
+        if(" " in argv0):
+          argv0 = event['input'][:event['input'].index(" ")]
+
+        if(argv0 in cmdping_commands):
+          ra.append(argv0)
+  return ra
 
 
 
-with open(filepath, "r") as logfile:
+
+with open(filepath, "r") as logfile: # Load da file and collect all possible relevant data about it
   for line in logfile:
     entry = json.loads(line)
     
     if(
-      (not args.filter_ip or entry["src_ip"] == args.filter_ip or hashgen(entry["src_ip"]) == args.filter_ip) # If it's not the IP we're filtering for don't even bother man
+      (not args.filter_ip or entry['src_ip'] == args.filter_ip or hashgen(entry['src_ip']) == args.filter_ip) # If it's not the IP we're filtering for don't even bother man
       ):
-      entries.append(entry)
       session_id = entry['session']
 
+      if(args.redact_ip):
+        entry['src_ip'] = hashgen(entry['src_ip'])
+
       if(session_id not in sessions):
         sessions.update({session_id: [entry]})
-        if(args.redact_ip):
-          session_ips.update({session_id: hashgen(entry['src_ip'])})
-        else:
-          session_ips.update({session_id: entry['src_ip']})
       else:
         sessions[session_id].append(entry)
 
-      if(entry["eventid"] == "cowrie.session.connect" and session_id not in session_timestamps): # Save the timestamp of the session's initial connection
-        session_timestamps.update({session_id: entry['timestamp']})
-
-      if(entry["eventid"] == "cowrie.session.closed" and session_id not in session_durations): # Save the session's duration
-        session_durations.update({session_id: entry['duration']})
-
-      if(entry["eventid"] == "cowrie.command.input"): 
-        if(session_id not in session_ran_command): # Save whether any commands were run during the session
-          session_ran_command.append(session_id)
-
-        argv0 = entry["input"] # Isolate the name of the command
-        if(" " in argv0):
-          argv0 = entry["input"][:entry["input"].index(" ")]
-
-        if(argv0 in cmdping_commands): # Determine whether this event qualifies the session as not a command ping
-          if(session_id not in session_cmdping_commands):
-            session_cmdping_commands.update({session_id: [argv0]})
-          else:
-            session_cmdping_commands[session_id].append(argv0)
-        else:
-          if(session_id not in session_not_cmdpings):
-            session_not_cmdpings.append(session_id)
+# Filtering
 
-      if(entry["eventid"] == "cowrie.login.success" and session_id not in session_successful_login): # Save if login was a success
-        session_successful_login.update({session_id: True})
+newsessions = sessions.copy()
 
-      if(entry["eventid"] == "cowrie.login.failed" and session_id not in session_successful_login): # Save if login failed
-        session_successful_login.update({session_id: False})
+for session_id in sessions:
+  session = sessions[session_id]
+  
+  if(blankmode and (
+       (not include_empty_pings   and not session_attempted_login(session))
+    or (not include_failed_logins and     session_attempted_login(session) and not session_login_succeeded(session))
+    or (not include_login_pings   and     session_login_succeeded(session) and not session_ran_command(session))
+    or (not include_command_pings and     session_is_cmdping(session))
+    or (not include_eventful      and     session_ran_command(session)     and not session_is_cmdping(session))
+  )):
+    newsessions.pop(session_id)
 
-      if(entry["eventid"] == "cowrie.login.success" and session_id in session_successful_login): # Save if login succeeded after failing
-        session_successful_login[session_id] = True
-
-# Blank Mode logic
-
-if(blankmode):
-  newsessions = sessions.copy()
-
-  for session_id in sessions:
-
-    if(
-         (not print_emptypings and session_id not in session_successful_login)                                                                                        # Is empty ping and don't print those
-      or (not print_fails      and session_id in     session_successful_login and not session_successful_login[session_id])                                           # Is failed login and don't print those
-      or (not print_loginpings and session_id in     session_successful_login and     session_successful_login[session_id] and session_id not in session_ran_command) # is login ping and don't print those
-      or (not print_cmdpings   and session_id in     session_ran_command      and     session_id not in session_not_cmdpings)                                         # is command ping and don't print those
-      or (not print_sessions   and session_id in     session_ran_command      and     session_id in session_not_cmdpings)                                             # is session and don't print those
-    ):
-      newsessions.pop(session_id)
-
-  sessions = newsessions
+sessions = newsessions
 
 # Printing logic coming up
 
-if(args.verboser): # High verbosity mode: print timestamp, event type and message (if applicable) for every event. I could probably do better but I'm lazy
+if(verbosity == 2): # High verbosity mode: print timestamp, event type and message (if applicable) for every event. I could probably do better but I'm lazy
   for session_id in sessions:
-    ip = session_ips[session_id]     
-    print("\n--- Session " + ccgen(session_id) + session_id + ccgen() + " from " + ppip(ip, ttycolor=True) +" ---")
-    for entry in sessions[session_id]:
-      if(entry['message']):
-        eventid = entry['eventid']
-        print(entry["timestamp"] + " > " + eventid + (" " * (29 - len(eventid))) + ": " + entry['message'])
+    session = sessions[session_id]
+    ip = session_ip(session_id)
+    print("\n--- Session " + ccgen(session_id) + session_id + ccgen() + " from " + ppip(ip, ttycolor=True) + " to " + session_sensor(session) + " ---")
+    for entry in session:
+      eventid = entry['eventid']
+      if(entry['eventid'] == "cowrie.session.params"):
+        print(entry["timestamp"] + " > " + eventid + (" " * (29 - len(eventid))) + ": ARCH: " + entry["arch"])
       else:
-        print(entry["timestamp"] + " > " + entry['eventid'])
+        print(entry["timestamp"] + " > " + eventid + (" " * (29 - len(eventid))) + ": " + entry['message'])
 
-elif(args.verbosest): # Maximum verbosity: literally just pretty print the now-sorted-by-session logs
+elif(verbosity == 3): # Maximum verbosity: literally just pretty print the now-sorted-by-session logs
   print(json.dumps(sessions, indent=4))
 
 else: # Default (low) verbosity: just the good stuff
   for session_id in sessions:
+    
+    # Declaring initial variables
+    session = sessions[session_id]
+    ip = session_ip(session)
+    timestamp = session_timestamp(session)[:22].replace("T", " ") # clean up the timestamp text all pretty like
+    durationstring = " closed after " + str(session_duration(session)) + "s"
+    if(session_duration(session) < 0):
+      durationstring = "active as of processing"
 
-    ip = session_ips[session_id]
-    timestamp = session_timestamps[session_id][:22].replace("T", " ") # clean up the timestamp text all pretty like
+    if(not session_attempted_login(session)): # Printing empty pings
+      print(timestamp + " > " + ppip(ip, True, True) + " > " + ccgen("PURPLE") + "Empty ping" + ccgen() + durationstring)
 
-    if(session_id not in session_successful_login): # Printing logic for empty pings
-      if(session_id in session_durations):
-        print(timestamp + " > " + ppip(ip, True, True) + " > " + ccgen("PURPLE") + "Empty ping" + ccgen() + " lasting " + session_durations[session_id] + "s")
-      else:
-        print(timestamp + " > " + ppip(ip, True, True) + " > " + ccgen("PURPLE") + "Empty ping" + ccgen() + " active as of processing")
-
-    elif(not session_successful_login[session_id]): # Printing logic for failed logins
-      cred_combos = []
-      for entry in sessions[session_id]:
-        if(entry["eventid"] == "cowrie.login.failed"):
-          pw = []
-          if("password" in entry):
-            pw = entry["password"]
-          cred_combos.append((entry["username"], pw))
-      print(timestamp + " > " + ppip(ip, True, True) + " > " + ccgen("RED") +"Failed login" + ccgen() + ": " + str(cred_combos))
-
-    elif(session_id in session_ran_command): # Printing logic for sessions in which a command was run
-
-      if(not args.verbose and session_id not in session_not_cmdpings): # If -v is not enabled, collapse "command pings" to one-liners
-        cred_combos = []
-        for entry in sessions[session_id]:
-          if(entry["eventid"] == "cowrie.login.failed" or entry["eventid"] == "cowrie.login.success"):
-            cred_combos.append((entry["username"], entry["password"]))
-        print(timestamp + " > " + ppip(ip, True, True) + " > " + ccgen("BLUE") + "Command ping" + ccgen() + ": " + str(cred_combos) + " > " + str(session_cmdping_commands[session_id]))
-
-      else: # Printing logic for un-collapsed sessions
+    elif(session_attempted_login(session) and not session_login_succeeded(session)): # Printing failed logins
+      print(timestamp + " > " + ppip(ip, True, True) + " > " + ccgen("RED") +"Failed login" + ccgen() + ": " + str(session_cred_combos(session)))
+
+    elif(session_ran_command(session)): # Printing logic for sessions in which a command was run
+
+      if(not verbosity == 1 and session_is_cmdping(session)): # If -v is not enabled, collapse "command pings" to one-liners
+        print(timestamp + " > " + ppip(ip, True, True) + " > " + ccgen("BLUE") + "Command ping" + ccgen() + ": " + str(session_cred_combos(session)) + " > " + str(session_pingcmds(session)))
+
+      else: # Printing logic for un-collapsed sessions where commands were run
         print(timestamp + " > " + ppip(ip, True, True) + " > " + ccgen("CYAN") + "Session" + ccgen() + ":")
-        for entry in sessions[session_id]:
+        for entry in session:
           match(entry["eventid"]):
             case "cowrie.login.failed":
               pw = []
@@ -234,18 +270,14 @@ else: # Default (low) verbosity: just the good stuff
                 df = entry["url"]
               elif("destfile" in entry):
                 df = entry["destfile"]
-              if(args.verbose):
+              if(verbosity == 1):
                 print(" | Saved file " + df + " to " + entry["outfile"])
               else:
                 print(" | Saved file " + df + " to " + entry["outfile"][:40] + "...")
-        print(" X Disconnected after " + session_durations[session_id] + "s")
+        print(" X Session " + durationstring)
         
     else: # Printing logic for successful logins in which no commands were run
-      cred_combos = []
-      for entry in sessions[session_id]:
-        if(entry["eventid"] == "cowrie.login.failed" or entry["eventid"] == "cowrie.login.success"):
-          cred_combos.append((entry["username"], entry["password"]))
-      print(timestamp + " > " + ppip(ip, True, True) + " > " + ccgen("GREEN") + "Login ping" + ccgen() + ": " + str(cred_combos))
+      print(timestamp + " > " + ppip(ip, True, True) + " > " + ccgen("GREEN") + "Login ping" + ccgen() + ": " + str(session_cred_combos(session)))
 
 if(args.stats):
-  print("Aggregate statistics")
\ No newline at end of file
+  print("Aggregate statistics TODO")
\ No newline at end of file