diff options
| -rw-r--r-- | config__.default.py | 11 | ||||
| -rw-r--r-- | generate.py | 10 | ||||
| -rw-r--r-- | import-misskey.py | 27 |
3 files changed, 18 insertions, 30 deletions
diff --git a/config__.default.py b/config__.default.py index 2a19083..a4a7a15 100644 --- a/config__.default.py +++ b/config__.default.py @@ -12,13 +12,4 @@ character_limit = 80 log_length = 168 # How many of the most recently generated posts to log, so that the bot won't repeat them -# The default value of 168 logs a week of hourly posts - -emojis = True -# The markov chain strips colons, which removes emojis from posts but leaves their names. -# If set to True, a list of emoji shortcodes (eg. :neofox_floof:) present in the dataset -# will be stored and any occurences of the emojis' names will be replaced with the code to -# add the emoji back in. This may cause problems with emojis whose names are common words, -# eg. :cat: -# If set to false, emoji names will be removed as well. -# Reimport after changing this setting.
\ No newline at end of file +# The default value of 168 logs a week of hourly posts
\ No newline at end of file diff --git a/generate.py b/generate.py index 9e94910..041e436 100644 --- a/generate.py +++ b/generate.py @@ -3,6 +3,7 @@ import sys import requests import json import os +import re import secrets__ import config__ @@ -16,7 +17,7 @@ log_path = dir_path + "/posts_log.json" try: with open(log_path, "r") as infile: posts_log = json.loads(infile.read()) -except FileNotFoundError: +except (FileNotFoundError, json.decoder.JSONDecodeError): with open(log_path, "w") as outfile: outfile.write('[""]') posts_log = [] @@ -44,10 +45,9 @@ posts_log.append(text) with open(log_path, "w") as outfile: outfile.write(json.dumps(posts_log[-config__.log_length:])) -if(config__.emojis == True): - emojis = json.load(open(dir_path + "/emojis.json")) - for emoji in emojis: - text = text.replace(emoji, ":"+emoji+":") +emojis = json.load(open(dir_path + "/emojis.json")) +for emoji in emojis: + text = re.sub(r'\b'+emoji+r'\b', ":"+emoji+":", text) text = text.replace('@','@â').replace('#','#â') print(text) diff --git a/import-misskey.py b/import-misskey.py index 29b63bb..e5acfd3 100644 --- a/import-misskey.py +++ b/import-misskey.py @@ -43,27 +43,24 @@ for note in export_json: text = re.sub(HTML, '', text) text = re.sub(SPACE, ' ', text) text = re.sub(CONTRACTION, r'\1\2', text) - if(config__.emojis == False): - text = re.sub(EMOJI, '', text) text = text.strip() print(f" - {text}") corpus.append(text) - if(config__.emojis == True): - noteemojis = EMOJI.findall(text) - if(noteemojis): - emojis = emojis + noteemojis + noteemojis = EMOJI.findall(text) + if(noteemojis): + emojis = emojis + noteemojis -if(config__.emojis == True): - print('[+] preserving emojis:') - emojis = list(set(emojis)) - s_emo = [] - for emo in emojis: - s_emo.append(emo.replace(":", "")) - print(s_emo) - with open("emojis.json", "w") as f: - json.dump(s_emo, f) + +print('[+] preserving emojis:') +emojis = list(set(emojis)) +s_emo = [] +for emo in emojis: + s_emo.append(emo.replace(":", "")) +print(s_emo) +with open("emojis.json", "w") as f: + json.dump(s_emo, f) print('[+] building markov chain') model = chain.Text("@@note@@".join(corpus), well_formed=False) |