# -*- coding: utf-8 -*- """ @File : monitor_service.py @Date : 2024-01-28 @Author : Peng Shiyu """ import json from datetime import datetime, timedelta from functools import wraps import requests import six from peewee import fn, chunked from domain_admin.config import USER_AGENT from domain_admin.enums.monitor_status_enum import MonitorStatusEnum from domain_admin.enums.monitor_type_enum import MonitorTypeEnum from domain_admin.enums.time_unit_enum import TimeUnitEnum from domain_admin.log import logger from domain_admin.model import monitor_model from domain_admin.model.base_model import db from domain_admin.model.log_monitor_model import LogMonitorModel from domain_admin.model.monitor_model import MonitorModel from domain_admin.service import notify_service, file_service, async_task_service from domain_admin.service.scheduler_service import scheduler_main from domain_admin.utils import datetime_util, file_util def monitor_log_decorator(func): """ 监控任务的日志装饰器 """ @wraps(func) def wrapper(monitor_row, *args, **kwargs): # execute result = '' error = None start_time = datetime.now() try: result = func(monitor_row, *args, **kwargs) except Exception as e: error = e if error: result = six.text_type(error) # 记录日志 LogMonitorModel.create( monitor_id=monitor_row.id, monitor_type=monitor_row.monitor_type, create_time=start_time, update_time=datetime.now(), result=result or '', status=MonitorStatusEnum.ERROR if error else MonitorStatusEnum.SUCCESS, ) # 继续抛出异常 if error: raise error else: return result return wrapper def monitor_notify_decorator(func): """ 监控任务的日志装饰器 """ @wraps(func) def wrapper(monitor_row, *args, **kwargs): result = None error = None try: result = func(monitor_row, *args, **kwargs) except Exception as e: error = e # 继续抛出异常 if error: handle_monitor_exception(monitor_row, error) raise error else: handle_monitor_exception_restore(monitor_row) return result return wrapper def handle_monitor_exception_restore(monitor_row): # 如果上一次记录是失败的,则需要触发监测异常恢复通知 if monitor_row.status != MonitorStatusEnum.ERROR: return # 检查连续失败次数是否大于最大允许失败次数,增加容错 if monitor_row.allow_error_count > 0 and is_between_allow_error_count(monitor_row): return notify_service.notify_user_about_monitor_exception_restore(monitor_row) def handle_monitor_exception(monitor_row, error): # 检查连续失败次数是否大于最大允许失败次数,增加容错 if monitor_row.allow_error_count > 0 and is_between_allow_error_count(monitor_row): return # 发送异常通知 notify_service.notify_user_about_monitor_exception(monitor_row, error) def run_monitor_warp(monitor_row): error = None try: run_monitor(monitor_row) except Exception as e: error = e # 计算下次运行时间 # @since v1.6.56 精确到毫秒 next_run_time = datetime.now() + timedelta( milliseconds=TimeUnitEnum.to_millisecond(value=monitor_row.interval, unit=monitor_row.interval_unit)) # 同步任务 MonitorModel.update( next_run_time=next_run_time, status=MonitorStatusEnum.ERROR if error else MonitorStatusEnum.SUCCESS, version=MonitorModel.version + 1 ).where( MonitorModel.id == monitor_row.id ).execute() if monitor_row.is_active: return next_run_time @monitor_log_decorator @monitor_notify_decorator def run_monitor(monitor_row): """ :param monitor_row: :return: """ if monitor_row.monitor_type == MonitorTypeEnum.HTTP: # 毫秒 timeout = TimeUnitEnum.to_millisecond( value=monitor_row.http_timeout, unit=monitor_row.http_timeout_unit or TimeUnitEnum.Second # 系统原来默认是毫秒 ) run_http_monitor( method=monitor_row.content_dict['method'], url=monitor_row.content_dict['url'], timeout=timeout * 0.001 # 毫秒转为秒 ) def run_http_monitor(url, method='GET', timeout=3): res = requests.request( method=method, url=url, timeout=timeout, headers={ "User-Agent": USER_AGENT } ) if not res.ok: res.raise_for_status() return res.text @db.connection_context() def run_monitor_task(): monitor_rows = MonitorModel.select().where( MonitorModel.is_active == True, MonitorModel.next_run_time < datetime_util.get_datetime() ).order_by(MonitorModel.next_run_time.asc()) # last_next_run_time = None for monitor_row in monitor_rows: run_monitor_warp(monitor_row) # 返回下次唤醒时间 monitor_row = MonitorModel.select().where( MonitorModel.is_active == True, ).order_by(MonitorModel.next_run_time.asc()).first() if monitor_row: return monitor_row.next_run_time def load_monitor_log_count(lst): monitor_ids = [row['id'] for row in lst] # 日志条数 monitor_groups = LogMonitorModel.select( LogMonitorModel.monitor_id, fn.COUNT(LogMonitorModel.id).alias('count') ).where( LogMonitorModel.monitor_id.in_(monitor_ids) ).group_by( LogMonitorModel.monitor_id ) monitor_groups_map = { row.monitor_id: row.count for row in monitor_groups } for row in lst: row['log_count'] = monitor_groups_map.get(row['id']) def get_monitor_list_query(**params): query = MonitorModel.select() status = params.get("status") if isinstance(status, int): query = query.where(MonitorModel.status == status) keyword = params.get('keyword') if keyword: query = query.where(MonitorModel.title.contains(keyword)) user_id = params.get('user_id') if user_id: query = query.where(MonitorModel.user_id == user_id) return query def export_monitor_to_file(rows, ext): """ 导出域名到文件 :param rows: :return: """ filename = datetime.now().strftime("monitor_%Y%m%d%H%M%S") + '.' + ext temp_filename = file_service.resolve_temp_file(filename) lst = file_util.convert_to_export(rows, monitor_model.FIELD_MAPPING) file_util.write_data_to_file(temp_filename, lst) return filename def import_monitor_from_file(filename, user_id): rows = file_util.read_data_from_file(filename) lst = file_util.convert_to_import(rows, monitor_model.FIELD_MAPPING) lst = [ { 'title': item['title'], 'monitor_type': MonitorTypeEnum.HTTP, 'interval': int(item.get('interval') or '60'), 'interval_unit': TimeUnitEnum.get_value(item.get('interval_unit_label') or '分钟'), 'content': json.dumps({ 'url': item['http_url'], 'method': 'GET', 'timeout': int(item.get('http_timeout') or '3'), 'timeout_unit': TimeUnitEnum.get_value(item.get('http_timeout_unit_label') or '秒'), }), 'user_id': user_id, } for item in lst ] # fix: peewee.OperationalError: too many SQL variables # https://github.com/mouday/domain-admin/issues/63 for batch in chunked(lst, 500): MonitorModel.insert_many(batch).on_conflict_ignore().execute() def is_between_allow_error_count(monitor_row): """ 连续失败次数是否在允许范围内 :param monitor_row: :return: 注意:先触发的事件,才写入日志 """ # 检查连续失败次数是否大于最大允许失败次数,增加容错 rows = LogMonitorModel.select().where( LogMonitorModel.monitor_id == monitor_row.id, ).order_by( LogMonitorModel.id.desc() ).limit( monitor_row.allow_error_count + 1 ) error_count = len([row for row in rows if row.status == MonitorStatusEnum.ERROR]) return error_count <= monitor_row.allow_error_count @async_task_service.async_task_decorator("更新监控信息") def run_init_monitor_task_async(user_id): rows = MonitorModel.select().where( MonitorModel.user_id == user_id, MonitorModel.version == 0 ) for row in rows: scheduler_main.run_one_monitor_task(row)