1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
|
import re
import json
import sys
import config__
print('[+] loading nlp enhanced markov chain')
import chain
MENTION = re.compile(r'@[a-zA-Z0-9.-_]+(@[a-zA-Z0-9.]+-_)?')
MFM_BEGIN = re.compile(r'\$\[[a-z0-9.,=]+')
MFM_END = re.compile(r'\]+')
HTML = re.compile(r'</?[a-z]+>')
SPACE = re.compile(r'[ \n]+')
CONTRACTION = re.compile(r"(\w+)'(\w+)")
EMOJI = re.compile(r':[a-zA-Z0-9-_]+:')
print('[+] loading note json')
export_f = sys.argv[1]
export = open(export_f)
export_json = json.load(export)
corpus = []
emojis = []
for note in export_json:
if note.get('visibility') not in ['public', 'unlisted']:
continue
if note.get('localOnly'):
continue
if note.get('cw'):
continue
text = note.get('text')
if not text:
continue
text = text.lower()
text = re.sub(MENTION, '', text)
text = re.sub(MFM_BEGIN, '', text)
text = re.sub(MFM_END, '', text)
text = re.sub(HTML, '', text)
text = re.sub(SPACE, ' ', text)
text = re.sub(CONTRACTION, r'\1\2', text)
if(config__.emojis == False):
text = re.sub(EMOJI, '', text)
text = text.strip()
print(f" - {text}")
corpus.append(text)
if(config__.emojis == True):
noteemojis = EMOJI.findall(text)
if(noteemojis):
emojis = emojis + noteemojis
if(config__.emojis == True):
print('[+] preserving emojis:')
emojis = list(set(emojis))
s_emo = []
for emo in emojis:
s_emo.append(emo.replace(":", ""))
print(s_emo)
with open("emojis.json", "w") as f:
json.dump(s_emo, f)
print('[+] building markov chain')
model = chain.Text("@@note@@".join(corpus), well_formed=False)
model_json = model.compile().to_json()
print('[+] exporting')
export = open('model.json', 'w')
export.write(model_json)
|
