You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
50 lines
1.6 KiB
50 lines
1.6 KiB
# -*- coding: utf-8 -*-
|
|
import config
|
|
from .monitor_result_db import MonitorResult
|
|
import utils.date_format as date_format
|
|
from .monitor_task_db import MonitorTask
|
|
|
|
|
|
def gen_result(task: MonitorTask, title, url, publish_time):
|
|
"""
|
|
构建任务结果对象
|
|
:param task: 任务对象
|
|
:param title: 标题
|
|
:param url: 地址
|
|
:param publish_time: 发布时间
|
|
:return:
|
|
"""
|
|
if isinstance(publish_time, str):
|
|
timestamp = date_format.timestamp()
|
|
publish_time = int(date_format.parse(publish_time).timestamp())
|
|
module = MonitorResult(title=title, url=url, publish_time=publish_time,
|
|
is_del=1,
|
|
keyword=task.keyword, platform=task.platform,
|
|
gather_time=F"{task.gather_date} {task.setting_time}")
|
|
return module
|
|
|
|
|
|
async def save(results):
|
|
if config.RESULT_UNIQUE:
|
|
await save_unique(results)
|
|
else:
|
|
model = MonitorResult()
|
|
await model.bulk_create(results)
|
|
|
|
|
|
async def save_unique(results):
|
|
# 过滤列表中重复的结果
|
|
unique_results = {}
|
|
for result in results:
|
|
key = (result.platform, result.keyword, result.title)
|
|
if key not in unique_results:
|
|
unique_results[key] = result
|
|
unique_results = list(unique_results.values())
|
|
# 过滤数据库中重复的结果
|
|
save_results = []
|
|
model = MonitorResult()
|
|
for result in unique_results:
|
|
exist = await model.filter(platform=result.platform, keyword=result.keyword, title=result.title).exists()
|
|
if not exist:
|
|
save_results.append(result)
|
|
await model.bulk_create(save_results)
|