From 549510c3a42622022617d7fe85629b804796af7a Mon Sep 17 00:00:00 2001 From: Johannes Findeisen Date: Sat, 1 Oct 2022 04:37:35 +0200 Subject: [PATCH] Lot of bug fixing, optimizations and refactoring. --- bin/linspector | 6 +- etc/linspector.conf | 10 +- etc/monitors/.gitkeep | 0 etc/monitors/network1/gateway.conf | 19 ++-- etc/monitors/network2/gateway.conf | 15 ++- etc/monitors/test/test1.conf | 8 +- etc/monitors/test/test10.conf | 3 + etc/monitors/test/test2.conf | 5 +- etc/monitors/test/test3.conf | 3 + etc/monitors/test/test4.conf | 3 + etc/monitors/test/test5.conf | 5 +- etc/monitors/test/test6.conf | 3 + etc/monitors/test/test7.conf | 5 +- etc/monitors/test/test8.conf | 5 +- etc/monitors/test/test9.conf | 3 + linspector/core/linspector.py | 12 ++- linspector/core/monitor.py | 108 ++++++++++++---------- linspector/core/monitors.py | 17 ++-- linspector/services/net/fritzboxuplink.py | 2 +- 19 files changed, 143 insertions(+), 89 deletions(-) delete mode 100644 etc/monitors/.gitkeep diff --git a/bin/linspector b/bin/linspector index a405026..8651772 100755 --- a/bin/linspector +++ b/bin/linspector @@ -32,7 +32,9 @@ from linspector.core.environment import Environment from linspector.core.linspector import Linspector from linspector.core.monitors import Monitors -__version__ = '0.19.12.dev1' +# i currently only set the 3rd number because the goal is that 0.19 will become the first stable +# version. +__version__ = '0.19.13.dev1' __author__ = 'Johannes Findeisen ' logger = logging.getLogger('linspector') @@ -40,7 +42,7 @@ logger = logging.getLogger('linspector') def parse_args(): parser = argparse.ArgumentParser( - description='linspector is a infrastructure monitoring daemon and toolchain.', + description='linspector is a infrastructure and system monitoring daemon and toolchain.', epilog='author: ' + __author__, prog='linspector') diff --git a/etc/linspector.conf b/etc/linspector.conf index a8eeaad..b2bb3c8 100644 --- a/etc/linspector.conf +++ b/etc/linspector.conf @@ -2,9 +2,9 @@ [linspector] ; report core errors to the following users error_receivers = admin@example.com -log_file = ~/code/linspector/log/linspector.log +log_file = ~/code/linspector/linspector/log/linspector.log ; available log levels are: "error", "warning", "info" and "debug". -log_level= debug +log_level= info log_count = 5 log_size = 10485760 pid_file = /var/run/user/1000/linspector.pid @@ -24,8 +24,14 @@ members = superadmin@example.com,developers@example.com ; scheduler configuration start_scheduler = true max_threads = 3500 +; i recommend to set this to your number of cpu cores available when running on a dedicated linspector host. but if the +; system is running other services you should lower this value when you have too high cpu load. max_processes = 24 timezone = CET +; this is for scheduling jobs to not run all at the same time. this should be set to the lowest interval you use. it +; can be set to a lower value if you only have a small amount of services you are monitoring. i recommend the lowest +; interval you use any any monitor. it is always starting with 0. default is 60. +delta_range = 10 ; hostgroup parents; if the hostgroup "group1" is down, don't alert for the hosts in group2. see TODO.txt for more ; information. diff --git a/etc/monitors/.gitkeep b/etc/monitors/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/etc/monitors/network1/gateway.conf b/etc/monitors/network1/gateway.conf index 8b4d2dc..17c9b13 100644 --- a/etc/monitors/network1/gateway.conf +++ b/etc/monitors/network1/gateway.conf @@ -1,10 +1,15 @@ +; required options: [monitor] ; currently i get errors when the service option is not spelled correct. error handling need to fix this. same for ; notifications and tasks. -service = net.FritzboxUplink +service = net.fritzboxuplink +hosts = 192.168.0.1,192.168.23.24,@group1,@testgroup1 interval = 60 + +; optional options when using notifications, plugins, services or tasks: +[args] ; notifications separated by ','. no whitespaces allowed. the case is not important. -notifications = sMs,EmaiL +notifications = sms,emaiL ; values from main configuration can be overridden for each defined monitor ; email_receivers separated by ','. no whitespaces allowed email_receivers = admin@example.com,fallback@example.com @@ -18,18 +23,10 @@ tasks = redis ; so we don't need to add a monitor for each host... but it would be a good idea to expose these to hostgroups ; internally to single hosts to become a monitor for each host internally. some more ideas are in ;) ; hosts with added hostgroup defined in main configuration file -hosts = 192.168.0.1,192.168.23.24,@group1,@testgroup1 ; monitors can add their self into a hostgroup -hostgroup = group1 +hostgroups = group1 user = USERNAME password = PASSWORD info = Cable Provider ; only alert when the error reaches the threshold threshold = 3 - -; need to think about it but since Linspector does not know variables defined by a service, notification or task they -; need to be handed over as **kwargs dict. so i think it would be a good idea to put them here and just give over all -; these vars as a dict. for now only interval and service are know and even required arguments to get a service running. -[args] -foo = bar -bar = foo \ No newline at end of file diff --git a/etc/monitors/network2/gateway.conf b/etc/monitors/network2/gateway.conf index 940a3dc..5663119 100644 --- a/etc/monitors/network2/gateway.conf +++ b/etc/monitors/network2/gateway.conf @@ -2,15 +2,14 @@ identifier = dsl service = Net.FritzboxUplink interval = 60 -notifications = Email -email_receivers = admin@example.com,fallback@example.com -sms_receivers = ++number1,+number2 -tasks = MariaDB,FileLogger hosts = 192.168.1.1 + +[args] +notifications = email +email_receivers = admin@example.com,fallback@example.com +sms_receivers = +number1,+number2 +tasks = mariadb,filelogger user = USERNAME password = PASSWORD info = Cable Provider - -[args] -foo = bary -bar = fooy \ No newline at end of file +hostgroups = group1 \ No newline at end of file diff --git a/etc/monitors/test/test1.conf b/etc/monitors/test/test1.conf index eadf86a..19e6006 100644 --- a/etc/monitors/test/test1.conf +++ b/etc/monitors/test/test1.conf @@ -1,7 +1,11 @@ +; requireed options [monitor] service = misc.dummy +hosts = 192.168.10.10 interval = 60 +; optional options [args] -foo = barx -bar = foox \ No newline at end of file +; the start is to schedule the execution of the monitor in at a date in the future. this is helpful to configure +; linspector before a host is up and running. +start_date = 2025-01-01 00:00:00 diff --git a/etc/monitors/test/test10.conf b/etc/monitors/test/test10.conf index d0691df..6ef2bdd 100644 --- a/etc/monitors/test/test10.conf +++ b/etc/monitors/test/test10.conf @@ -1,3 +1,6 @@ [monitor] service = misc.dummy interval = 60 +hosts = 192.168.10.10 + +[args] diff --git a/etc/monitors/test/test2.conf b/etc/monitors/test/test2.conf index d0691df..1acaec1 100644 --- a/etc/monitors/test/test2.conf +++ b/etc/monitors/test/test2.conf @@ -1,3 +1,6 @@ [monitor] service = misc.dummy -interval = 60 +interval = 10 +hosts = 192.168.10.10 + +[args] diff --git a/etc/monitors/test/test3.conf b/etc/monitors/test/test3.conf index 079c034..c6479ca 100644 --- a/etc/monitors/test/test3.conf +++ b/etc/monitors/test/test3.conf @@ -1,3 +1,6 @@ [monitor] service = misc.dummy interval = 120 +hosts = 192.168.10.10 + +[args] diff --git a/etc/monitors/test/test4.conf b/etc/monitors/test/test4.conf index d032ac7..aafae4a 100644 --- a/etc/monitors/test/test4.conf +++ b/etc/monitors/test/test4.conf @@ -1,3 +1,6 @@ [monitor] service = misc.dummy interval = 50 +hosts = 192.168.10.10 + +[args] diff --git a/etc/monitors/test/test5.conf b/etc/monitors/test/test5.conf index 18bd318..0e3ede3 100644 --- a/etc/monitors/test/test5.conf +++ b/etc/monitors/test/test5.conf @@ -1,3 +1,6 @@ [monitor] -service = misc.dummy +service = net.FritzboxUplink interval = 62 +hosts = 192.168.10.10 + +[args] diff --git a/etc/monitors/test/test6.conf b/etc/monitors/test/test6.conf index 01cd045..41ab715 100644 --- a/etc/monitors/test/test6.conf +++ b/etc/monitors/test/test6.conf @@ -1,3 +1,6 @@ [monitor] service = misc.dummy interval = 30 +hosts = 192.168.10.10 + +[args] diff --git a/etc/monitors/test/test7.conf b/etc/monitors/test/test7.conf index d0691df..410d99f 100644 --- a/etc/monitors/test/test7.conf +++ b/etc/monitors/test/test7.conf @@ -1,3 +1,6 @@ [monitor] service = misc.dummy -interval = 60 +interval = 20 +hosts = 192.168.10.10 + +[args] diff --git a/etc/monitors/test/test8.conf b/etc/monitors/test/test8.conf index d0691df..d48e6c7 100644 --- a/etc/monitors/test/test8.conf +++ b/etc/monitors/test/test8.conf @@ -1,3 +1,6 @@ [monitor] service = misc.dummy -interval = 60 +interval = 15 +hosts = 192.168.10.10 + +[args] diff --git a/etc/monitors/test/test9.conf b/etc/monitors/test/test9.conf index cfd41ce..7b758db 100644 --- a/etc/monitors/test/test9.conf +++ b/etc/monitors/test/test9.conf @@ -1,3 +1,6 @@ [monitor] service = misc.dummy interval = 240 +hosts = 192.168.10.10 + +[args] diff --git a/linspector/core/linspector.py b/linspector/core/linspector.py index 97bab99..ca00d0a 100644 --- a/linspector/core/linspector.py +++ b/linspector/core/linspector.py @@ -15,6 +15,7 @@ from linspector.core.helpers import log def job_function(monitor): + log('debug', __name__, monitor) monitor.handle_call() @@ -59,10 +60,15 @@ class Linspector: job_defaults=job_defaults) start_date = datetime.datetime.now() - #log('debug', __name__, monitors.get_monitors()) + log('debug', __name__, monitors.get_monitors()) monitors = self.__monitors.get_monitors() for monitor in monitors: - time_delta = round(random.uniform(1.00, 10.00), 2) + log('debug', __name__, monitor) + if configuration.get_option('linspector', 'delta_range'): + time_delta = round(random.uniform(0.00, float(configuration.get_option('linspector', 'delta_range'))), 2) + else: + time_delta = round(random.uniform(0.00, 60.00), 2) + new_start_date = start_date + datetime.timedelta(seconds=time_delta) monitor_job = monitors.get(monitor) interval = monitor_job.get_interval() @@ -74,7 +80,7 @@ class Linspector: monitor_job.set_job(scheduler_job) self.__jobs.append(monitor_job) log('info', __name__, 'scheduling job ' + monitor + ' with delta ' + str(time_delta) + - ' @' + str(new_start_date) + ' running service ') + ' @' + str(new_start_date) + ' running service ' + monitor_job.get_service()) if configuration.get_option('linspector', 'start_scheduler') == 'true': self.__scheduler['linspector'].start() diff --git a/linspector/core/monitor.py b/linspector/core/monitor.py index d7677b1..bb8a144 100644 --- a/linspector/core/monitor.py +++ b/linspector/core/monitor.py @@ -3,6 +3,8 @@ This file is part of Linspector (https://linspector.org/) Copyright (c) 2022 Johannes Findeisen . All Rights Reserved. See LICENSE (MIT license) """ +import configparser +import hashlib import importlib from datetime import datetime @@ -23,21 +25,25 @@ class Monitor: self.__interval = int(monitor_configuration.get('monitor', 'interval')) self.__monitor_configuration = monitor_configuration self.__notification_list = [] - self.__service = monitor_configuration.get('monitor', 'service') self.__notifications = notifications + self.__service = monitor_configuration.get('monitor', 'service') self.__services = services self.__task_list = [] # put tasks for the dedicated job here. self.__tasks = tasks self.service = self.__service - #self.host = monitor_configuration.get('monitor', 'host') - #self.members = members - #self.core = core - self.hostgroup = monitor_configuration.get('monitor', 'hostgroup') + self.host = monitor_configuration.get('monitor', 'hosts') + + try: + self.hostgroups = monitor_configuration.get('monitor', 'hostgroups') + except configparser.NoOptionError as err: + self.hostgroups = "None" + self.job_threshold = 0 self.enabled = True self.scheduler_job = None - self.job_id = self.hex_string() + # the job_id is a sha256 string. + self.job_id = self.generate_job_id() """ NONE job was not executed @@ -53,23 +59,24 @@ class Monitor: # self.service) self.monitor_information = MonitorInformation(self.job_id, self.service) - if configuration.get_option('linspector', 'notifications') or \ - monitor_configuration.get('monitor', 'notifications'): - - if configuration.get_option('linspector', 'notifications') and \ + try: + if configuration.get_option('linspector', 'notifications') or \ monitor_configuration.get('monitor', 'notifications'): - notification_list = \ - configuration.get_option('linspector', 'notifications') + ',' + \ - monitor_configuration.get('monitor', 'notifications') - elif configuration.get_option('linspector', 'notifications'): - notification_list = configuration.get_option('linspector', 'notifications') - elif monitor_configuration.get('monitor', 'notifications'): - notification_list = monitor_configuration.get('monitor', 'notifications') - else: - notification_list = None + if configuration.get_option('linspector', 'notifications') and \ + monitor_configuration.get('monitor', 'notifications'): - self.__notification_list = notification_list.split(',') + notification_list = \ + configuration.get_option('linspector', 'notifications') + ',' + \ + monitor_configuration.get('monitor', 'notifications') + elif configuration.get_option('linspector', 'notifications'): + notification_list = configuration.get_option('linspector', 'notifications') + elif monitor_configuration.get('monitor', 'notifications'): + notification_list = monitor_configuration.get('monitor', 'notifications') + else: + notification_list = None + + self.__notification_list = notification_list.split(',') for notification_option in notification_list.split(','): if notification_option not in notifications: @@ -77,6 +84,8 @@ class Monitor: notification_module = importlib.import_module(notification_package) notification = notification_module.create(configuration, environment) notifications[notification_option.lower()] = notification + except configparser.NoOptionError as err: + self.__notifications = notifications if self.__monitor_configuration.get('monitor', 'service'): if monitor_configuration.get('monitor', 'service') not in services: @@ -87,30 +96,32 @@ class Monitor: self.__service = monitor_configuration.get('monitor', 'service').lower() service = service_module.create(configuration, environment, **self.__args) self.__services[monitor_configuration.get('monitor', 'service').lower()] = service + try: + if configuration.get_option('linspector', 'tasks') or \ + monitor_configuration.get('args', 'tasks'): - if configuration.get_option('linspector', 'tasks') or \ - monitor_configuration.get('monitor', 'tasks'): + if configuration.get_option('linspector', 'tasks') and \ + monitor_configuration.get('args', 'tasks'): - if configuration.get_option('linspector', 'tasks') and \ - monitor_configuration.get('monitor', 'tasks'): + task_list = configuration.get_option('linspector', 'tasks') + ',' + \ + monitor_configuration.get('args', 'tasks') + elif configuration.get_option('linspector', 'tasks'): + task_list = configuration.get_option('linspector', 'tasks') + elif monitor_configuration.get('args', 'tasks'): + task_list = monitor_configuration.get('args', 'tasks') + else: + task_list = None - task_list = configuration.get_option('linspector', 'tasks') + ',' + \ - monitor_configuration.get('monitor', 'tasks') - elif configuration.get_option('linspector', 'tasks'): - task_list = configuration.get_option('linspector', 'tasks') - elif monitor_configuration.get('monitor', 'tasks'): - task_list = monitor_configuration.get('monitor', 'tasks') - else: - task_list = None + self.__task_list = task_list.split(',') - self.task_list = task_list.split(',') - - for task_option in task_list.split(','): - if task_option not in tasks: - task_package = 'linspector.tasks.' + task_option.lower() - task_module = importlib.import_module(task_package) - task = task_module.create(configuration, environment) - tasks[task_option.lower()] = task + for task_option in task_list.split(','): + if task_option not in tasks: + task_package = 'linspector.tasks.' + task_option.lower() + task_module = importlib.import_module(task_package) + task = task_module.create(configuration, environment) + tasks[task_option.lower()] = task + except configparser.NoOptionError: + self.__tasks = tasks def get_identifier(self): return self.__identifier @@ -124,11 +135,9 @@ class Monitor: def __str__(self): return str(self.__dict__) - def __hex__(self): - return hex(crc32(bytes(self.hostgroup + self.service, 'utf-8'))) - def hex_string(self): - ret = self.__hex__() + ret = hex(crc32(bytes(self.host + self.hostgroups + self.service, 'utf-8'))) + #ret = self.__hex__() if ret[0] == "-": ret = ret[3:] else: @@ -137,8 +146,9 @@ class Monitor: ret = "0" + ret return ret - def get_job_id(self): - return self.job_id + def generate_job_id(self): + return hashlib.sha256(bytes(self.__identifier + self.host + self.hostgroups + self.service, + 'utf-8')).hexdigest() def set_job(self, scheduler_job): self.scheduler_job = scheduler_job @@ -179,7 +189,7 @@ class Monitor: #TaskExecutor.instance().schedule_task(monitor_information, task) def handle_call(self): - log('info', __name__, "handle call to identifier: " + self.__identifier) + #log('info', __name__, "handle call to identifier: " + self.__identifier) self.__services[self.__service].execute() #logger.debug("handle call") #logger.debug(self.service) @@ -211,8 +221,8 @@ class Monitor: #def get_host(self): # return self.host - def get_hostgroup(self): - return self.hostgroup + def get_hostgroups(self): + return self.hostgroups class MonitorExecution: diff --git a/linspector/core/monitors.py b/linspector/core/monitors.py index 9ee5263..b387f7d 100644 --- a/linspector/core/monitors.py +++ b/linspector/core/monitors.py @@ -24,13 +24,16 @@ class Monitors: monitor_groups = os.listdir(self.__configuration.get_configuration_path() + '/monitors/') log('debug', __name__, 'monitor groups: ' + str(monitor_groups)) - monitor_configuration = configparser.ConfigParser() for monitor_group in monitor_groups: - monitors_file_list = glob.glob(self.__configuration.get_configuration_path() + '/monitors/' + monitor_group + '/*.conf') - #log('debug', __name__, 'monitor files: ' + str(monitors_file_list)) + + log('debug', __name__, 'monitor files: ' + str(monitors_file_list)) for monitor_file in monitors_file_list: + identifier = monitor_group + '_' + os.path.splitext(os.path.basename( + monitor_file))[0] + + monitor_configuration = configparser.ConfigParser() monitor_configuration.read(monitor_file, 'utf-8') kwargs = {} @@ -39,7 +42,8 @@ class Monitors: value = monitor_configuration.get('args', option) kwargs[option] = value - print(monitor_group + ' ' + str(kwargs)) + if kwargs: + log('debug', __name__, identifier + ' args ' + str(kwargs)) identifier = monitor_group + '_' + os.path.splitext(os.path.basename( monitor_file))[0] @@ -47,9 +51,8 @@ class Monitors: # create Monitor() object and copy monitor_configuration for each instance because # they else refer to the same object. self.__monitors[identifier] = Monitor(configuration, environment, identifier, - copy.deepcopy(monitor_configuration), - notifications, services, tasks, - copy.deepcopy(kwargs)) + monitor_configuration, notifications, + services, tasks, copy.deepcopy(kwargs)) del kwargs diff --git a/linspector/services/net/fritzboxuplink.py b/linspector/services/net/fritzboxuplink.py index 246052d..389b986 100644 --- a/linspector/services/net/fritzboxuplink.py +++ b/linspector/services/net/fritzboxuplink.py @@ -22,5 +22,5 @@ class FritzboxUplinkService(Service): self.__kwargs = kwargs def execute(self): - log('debug', __name__, 'dummy object @' + str(self)) + log('debug', __name__, 'fritzboxuplink object @' + str(self)) return