1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
|
import re
import json
import sys
print('[+] loading nlp enhanced markov chain')
import chain
MENTION = re.compile(r'@[a-zA-Z0-9.-_]+(@[a-zA-Z0-9.]+-_)?')
MFM_BEGIN = re.compile(r'\$\[[a-z0-9.,=]+')
MFM_END = re.compile(r'\]+')
HTML = re.compile(r'</?[a-z]+>')
SPACE = re.compile(r'[ \n]+')
CONTRACTION = re.compile(r"(\w+)'(\w+)")
print('[+] loading note json')
export_f = sys.argv[1]
export = open(export_f)
export_json = json.load(export)
corpus = []
for note in export_json:
if note.get('visibility') not in ['public', 'unlisted']:
continue
if note.get('localOnly'):
continue
if note.get('cw'):
continue
text = note.get('text')
if not text:
continue
text = text.lower()
text = re.sub(MENTION, '', text)
text = re.sub(MFM_BEGIN, '', text)
text = re.sub(MFM_END, '', text)
text = re.sub(HTML, '', text)
text = re.sub(SPACE, ' ', text)
text = re.sub(CONTRACTION, r'\1\2', text)
text = text.strip()
print(f" - {text}")
corpus.append(text)
print('[+] building markov chain')
model = chain.Text("@@note@@".join(corpus), well_formed=False)
model_json = model.compile().to_json()
print('[+] exporting')
export = open(export_f.replace('.json', '.model.json'), 'w')
export.write(model_json)
|
