# -*- coding: utf-8 -*- import argparse import asyncio import os import sys import config import db import utils.date_format as date_format from base.enums import Platform from models import monitor_task_model from utils.scheduler import SchedulerManager from utils.utils import logger def task_group(tasks): groups = {} for name, enum in Platform.__members__.items(): groups[enum.value] = [] for task in tasks: if task.platform in groups: groups[task.platform].append(task) return list(groups.values()) async def do_get_task_job(): """ 获取任务信息 :return: """ await db.init() tasks = await monitor_task_model.get_today_task() if not tasks: logger.info(F"没有获取到任务信息") return # 分组 # groups = task_group(tasks) # random.shuffle(groups) schedular_manager = SchedulerManager() logger.info(F"============================== 获取到{len(tasks)}条任务信息 ==============================") schedular_manager.add_tasks(tasks, True) def restart(): os.execl(sys.executable, sys.executable, *sys.argv) def load_arg_parse(): """ 解析启动参数 :return: """ parse = argparse.ArgumentParser(description="抓取社媒新闻数据") parse.add_argument("-a", "--active", help="启动脚本时 立即进行一次任务拉取", default='false') args = parse.parse_args() logger.info(F"启动参数: {args}") return args def clear_system_proxy(): # 清除系统代理相关的环境变量 os.environ.pop('http_proxy', None) os.environ.pop('https_proxy', None) os.environ.pop('ftp_proxy', None) os.environ.pop('no_proxy', None) if __name__ == '__main__': try: clear_system_proxy() logger.info(F'启动成功 将在每天的{config.GET_TASK_TIME}拉取任务信息') get_task_time = date_format.gen_job_datetime(config.GET_TASK_TIME) manager = SchedulerManager() # 启动定时任务 manager.start() # 添加拉取任务信息的任务 manager.scheduler.add_job(do_get_task_job, 'cron', hour=get_task_time.hour, minute=get_task_time.minute) manager.scheduler.add_job(restart, 'cron', hour=get_task_time.hour, minute=0) # 参数检查 args = load_arg_parse() if args.active and args.active.lower() == 'true': logger.info(F"立即执行一次任务拉取...") asyncio.get_event_loop().run_until_complete(do_get_task_job()) # 开启事件循环 asyncio.get_event_loop().run_forever() except KeyboardInterrupt: sys.exit()