LINUXDO_帖子随机数据库

rl.lower() for k in keys): links[name] = url found = True break if not found: links['其他'] = url # 从描述中移除裸链接 desc_text = url_pattern.sub('', desc_text) # 5. 全局正则提取所有#标签，并从描述中移除 tag_pattern = re.compile(r'#([\u4e00-\u9fa5A-Za-z0-9_]+)') found_tags = tag_pattern.findall(desc_text) if found_tags: tags.extend(found_tags) desc_text = tag_pattern.sub('', desc_text) # 去重 tags = list(set(tags)) # 移除所有网盘名关键词 netdisk_names = ['夸克', '迅雷', '百度', 'UC', '阿里', '天翼', '115', '123云盘'] netdisk_name_pattern = re.compile(r'(' + '|'.join(netdisk_names) + r')') desc_text = netdisk_name_pattern.sub('', desc_text) # 6. 最终description，去除无意义符号行 desc_lines_final = [line for line in desc_text.strip().split('\n') if line.strip() and not re.fullmatch(r'[.。·、,，-]+', line.strip())] description = '\n'.join(desc_lines_final) return { 'title': title, 'description': description, 'links': links, 'tags': tags, 'source': source, 'channel': channel, 'group_name': group, 'bot': bot } @client.on(events.NewMessage(chats=channel_usernames)) async def handler(event): message = event.raw_text timestamp = datetime.datetime.now() # 解析消息 parsed_data = parse_message(message) # 创建数据库会话 with Session(engine) as session: # 创建新消息记录 new_message = Message( timestamp=timestamp, **parsed_data ) session.add(new_message) session.commit() print(f"[{timestamp}] 新消息已保存到数据库") print(f"✅ 正在监听 Telegram 频道：{channel_usernames} ...") if __name__ == "__main__": if "--fix-tags" in sys.argv: # 检查并修复tags字段脏数据 from sqlalchemy import update from sqlalchemy.orm import Session with Session(engine) as session: msgs = session.query(Message).all() fixed = 0 for msg in msgs: # 如果tags不是list类型，尝试修正 if msg.tags is not None and not isinstance(msg.tags, list): try: import ast tags_fixed = ast.literal_eval(msg.tags) if isinstance(tags_fixed, list): session.execute(update(Message).where(Message.id==msg.id).values(tags=tags_fixed)) fixed += 1 except Exception as e: print(f"ID={msg.id} tags修复失败: {e}") session.commit() print(f"已修复tags字段脏数据条数: {fixed}") elif "--dedup-links" in sys.argv: # 定期去重：只保留每个网盘链接最新的消息 from sqlalchemy.orm import Session from sqlalchemy import delete with Session(engine) as session: all_msgs = session.query(Message).order_by(Message.timestamp.desc()).all() link_to_id = {} # {url: 最新消息id} id_to_delete = set() for msg in all_msgs: if not msg.links: continue for url in msg.links.values(): if url in link_to_id: id_to_delete.add(msg.id) else: link_to_id[url] = msg.id if id_to_delete: session.execute(delete(Message).where(Message.id.in_(id_to_delete))) session.commit() print(f"已删除重复网盘链接的旧消息条目: {len(id_to_delete)}") else: print("没有需要删除的重复网盘链接消息。") else: client.start() client.run_until_disconnected() web端web.py： import streamlit as st from sqlalchemy.orm import Session from models import Message, engine import pandas as pd from datetime import datetime, timedelta from collections import Counter from sqlalchemy import or_ # 初始化session_state用于标签筛选 if 'selected_tags' not in st.session_state: st.session_state['selected_tags'] = [] st.set_page_config( page_title="TG频道监控", page_icon="📱", layout="wide" ) # 设置页面标题 st.title("📱 TG频道监控") # 创建侧边栏过滤器 st.sidebar.header("筛选条件") # 时间范围选择 time_range = st.sidebar.selectbox( "时间范围", ["最近24小时", "最近7天", "最近30天", "全部"] ) # 标签选择（标签云，显示数量，降序） with Session(engine) as session: all_tags = session.query(Message.tags).all() tag_list = [tag for tags in all_tags for tag in (tags[0] if tags[0] else [])] tag_counter = Counter(tag_list) tag_items = sorted(tag_counter.items(), key=lambda x: x[1], reverse=True) tag_options = [f"{tag} ({count})" for tag, count in tag_items] tag_map = {f"{tag} ({count})": tag for tag, count in tag_items} # 默认选中session_state中的标签 selected_tag_labels = st.sidebar.multiselect( "标签", tag_options, default=[f"{tag} ({tag_counter[tag]})" for tag in st.session_state['selected_tags'] if tag in tag_counter] ) selected_tags = [tag_map[label] for label in selected_tag_labels] # 同步session_state st.session_state['selected_tags'] = selected_tags # 网盘类型筛选 netdisk_types = ['夸克网

1 view11:14