1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
|
import json
import html2text
import re
import sqlite3
from pathlib import Path
from mlx_lm.utils import load # type: ignore
from mlx_lm.generate import generate # type: ignore
model, tokenizer = load("mlx-community/SmolLM3-3B-8bit")
system_prompt = """/no_think
You are a JSON generator.
You are a JSON generator.
Extract ALL people, places, and topics from the provided input.
Please use structured JSON for the output:
```json
{
"people": [ "string", "string", etc. ],
"places": [ "string", "string", etc. ],
"topics": [ "string", "string", etc. ],
}
```
Return only JSON inside these markers:
<json>
{ ... }
</json>
Important: Your entire output must be a single valid JSON object.
Do not add comments, explanations, or text outside the JSON braces.
If you cannot comply, output {}.
"""
def init_database(db_path="posts_processed.db"):
"""Initialize SQLite database with posts table"""
conn = sqlite3.connect(db_path)
cursor = conn.cursor()
# Create table if it doesn't exist
cursor.execute('''
CREATE TABLE IF NOT EXISTS processed_posts (
post_id TEXT PRIMARY KEY,
json_data TEXT NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
''')
conn.commit()
return conn
def post_exists(conn, post_id):
"""Check if a post_id already exists in the database"""
cursor = conn.cursor()
cursor.execute('SELECT 1 FROM processed_posts WHERE post_id = ?', (post_id,))
return cursor.fetchone() is not None
def save_post(conn, post_id, json_data):
"""Save processed post to database"""
cursor = conn.cursor()
cursor.execute('''
INSERT INTO processed_posts (post_id, json_data)
VALUES (?, ?)
''', (post_id, json.dumps(json_data)))
conn.commit()
def extract_json_from_response(response):
"""Extract JSON from the model response between <json> markers"""
try:
# Look for content between <json> and </json> markers
import re
json_match = re.search(r'<json>\s*(.*?)\s*</json>', response, re.DOTALL)
if json_match:
json_str = json_match.group(1)
return json.loads(json_str)
else:
# If no markers found, try to parse the entire response as JSON
# First, try to find a JSON object in the response
json_match = re.search(r'\{.*\}', response, re.DOTALL)
if json_match:
return json.loads(json_match.group(0))
except json.JSONDecodeError as e:
print(f"Failed to parse JSON: {e}")
except Exception as e:
print(f"Error extracting JSON: {e}")
return {}
def load_all_posts(posts_dir="posts"):
"""Load all posts from all JSON files into a single list"""
all_posts = []
posts_path = Path(posts_dir)
for json_file in posts_path.glob("*.json"):
try:
with open(json_file, 'r', encoding='utf-8') as f:
posts = json.load(f)
all_posts.extend(posts)
except Exception as e:
print(f"Error reading {json_file}: {e}")
return all_posts
def main():
# Initialize database
conn = init_database()
# Load posts
posts = load_all_posts()
total_posts = len(posts)
processed_count = 0
skipped_count = 0
for i, post in enumerate(posts, 1):
post_id = post['post_id']
# Skip if post already exists in database
if post_exists(conn, post_id):
skipped_count += 1
print(f"\rProgress: {i}/{total_posts} | Processed: {processed_count} | Skipped: {skipped_count}", end="", flush=True)
continue
# Extract and clean text
html = post['post_html']
if html is None:
html = "<body></body>"
text2 = html2text.html2text(html)
cleaned = re.sub(r'https?://\S+|www\.\S+', '', text2)
prompt = f"""<input>
{cleaned}
</input>
"""
messages = [
{ "role": "system", "content": system_prompt },
{ "role": "user", "content": prompt }
]
prompt = tokenizer.apply_chat_template(messages, add_generation_prompt=True)
response = generate(
model,
tokenizer,
prompt=prompt,
verbose=False,
max_tokens=200
)
# Extract JSON from response and save to database
extracted_json = extract_json_from_response(response)
if extracted_json:
save_post(conn, post_id, extracted_json)
processed_count += 1
print(f"\rProgress: {i}/{total_posts} | Processed: {processed_count} | Skipped: {skipped_count}", end="", flush=True)
# Close database connection
conn.close()
print(f"\n✓ Complete: Processed {processed_count} new posts, skipped {skipped_count} existing posts")
if __name__ == "__main__":
main()
|
