From 83827e807054b9a50e1204b0d50411578617c27a Mon Sep 17 00:00:00 2001 From: Johannes Findeisen Date: Thu, 31 Aug 2023 22:30:05 +0200 Subject: [PATCH] Reverted databases back to be tasks again. This is more native to the API of Linspector. (0.26.0) --- bin/linspector | 22 +---------- etc/linspector.conf | 5 +-- .../__init__.py => etc/monitors/.gitkeep | 0 etc/{databases => tasks}/mysql.conf | 1 + files/errors/sqlerror_001.txt | 39 +++++++++++++++++++ linspector/database.py | 1 + linspector/databases/none.py | 22 ----------- linspector/monitor.py | 31 +++++---------- linspector/monitors.py | 4 +- linspector/notification.py | 1 + linspector/task.py | 1 + linspector/{databases => tasks}/mysql.py | 36 ++++++++--------- linspector/tasks/none.py | 24 ++++++++++++ 13 files changed, 100 insertions(+), 87 deletions(-) rename linspector/databases/__init__.py => etc/monitors/.gitkeep (100%) rename etc/{databases => tasks}/mysql.conf (83%) create mode 100644 files/errors/sqlerror_001.txt delete mode 100644 linspector/databases/none.py rename linspector/{databases => tasks}/mysql.py (65%) create mode 100644 linspector/tasks/none.py diff --git a/bin/linspector b/bin/linspector index 16ed842..a7f216c 100755 --- a/bin/linspector +++ b/bin/linspector @@ -20,7 +20,7 @@ from linspector.environment import Environment from linspector.linspector import Linspector from linspector.monitors import Monitors -__version__ = '0.25.9' +__version__ = '0.26.0' __author__ = 'Johannes Findeisen ' @@ -49,7 +49,6 @@ def parse_args(): def linspector(): args = parse_args() - databases = {} monitors = {} notifications = {} plugins = {} @@ -104,23 +103,6 @@ def linspector(): environment = Environment(log) - try: - if configuration.get_option('linspector', 'databases'): - log.info('loading databases: ' + configuration.get_option('linspector', 'databases')) - database_list = configuration.get_option('linspector', 'databases') - database_list = database_list.split(',') - for database_name in database_list: - database_package = 'linspector.databases.' + database_name - if importlib.util.find_spec(database_package) is not None: - database_module = importlib.import_module(database_package) - database_object = database_module.create(configuration, environment, log) - databases[database_name] = database_object - else: - log.warning('initialization of database: {0} failed! seems it does not exist.'. - format(database_name)) - except Exception as err: - log.warning('database initialization error: {0}'.format(err)) - try: if configuration.get_option('linspector', 'tasks'): log.info('Loading tasks: ' + configuration.get_option('linspector', 'tasks')) @@ -138,7 +120,7 @@ def linspector(): log.warning('task initialization error: {0}'.format(err)) try: - monitors = Monitors(configuration, databases, environment, log, notifications, services, tasks) + monitors = Monitors(configuration, environment, log, notifications, services, tasks) except Exception as err: log.warning('monitor initialization error: {0}'.format(err)) diff --git a/etc/linspector.conf b/etc/linspector.conf index 052e71b..251510e 100644 --- a/etc/linspector.conf +++ b/etc/linspector.conf @@ -24,13 +24,12 @@ logfile_level = DEBUG ; size of logfiles (default: 1MB) logfile_size = 10MB -databases = mysql - notifications = plugins = -tasks = +; the tasks to be executed after each monitors service execution +tasks = mysql ; timezone can be set to a remote timezone to make monitors run at the remote time. this can be overridden in each ; monitor configuration. diff --git a/linspector/databases/__init__.py b/etc/monitors/.gitkeep similarity index 100% rename from linspector/databases/__init__.py rename to etc/monitors/.gitkeep diff --git a/etc/databases/mysql.conf b/etc/tasks/mysql.conf similarity index 83% rename from etc/databases/mysql.conf rename to etc/tasks/mysql.conf index 7dd1f61..96408a6 100644 --- a/etc/databases/mysql.conf +++ b/etc/tasks/mysql.conf @@ -3,3 +3,4 @@ host = HOST user = USER password = PASSWORD database = DATABASE +table = TABLE diff --git a/files/errors/sqlerror_001.txt b/files/errors/sqlerror_001.txt new file mode 100644 index 0000000..9105f76 --- /dev/null +++ b/files/errors/sqlerror_001.txt @@ -0,0 +1,39 @@ +[2023-08-29 14:55:06.350038 +0200] [0:46:32.576412] [INFO] [linspector]: identifier=remote1.port25 host=a.xw3.org service=net.port status=OK +[2023-08-29 14:55:13.571002 +0200] [0:46:39.797376] [INFO] [linspector]: identifier=remote1.port995 host=a.xw3.org service=net.port status=OK +[2023-08-29 14:55:14.394669 +0200] [0:46:40.621043] [INFO] [linspector]: identifier=remote1.port110 host=a.xw3.org service=net.port status=OK +[2023-08-29 14:55:20.288968 +0200] [0:46:46.515342] [INFO] [linspector]: identifier=remote1.port22 host=a.xw3.org service=net.port status=OK +Unable to get a connection: HTTPConnectionPool(host='192.168.23.1', port=49000): Max retries exceeded with url: /igddesc.xml (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')) +[2023-08-29 14:55:24.912093 +0200] [0:46:51.138467] [WARNING] [linspector]: database query failed: (1064, "You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'Unable to get a connection: HTTPConnectionPool(host='192.168.23.1', port=4900...' at line 1") +[2023-08-29 14:55:24.912280 +0200] [0:46:51.138654] [INFO] [linspector]: identifier=network1.gateway_cable host=192.168.23.1 service=vendor.avm.is_connected status=ERROR +[2023-08-29 14:55:28.801723 +0200] [0:46:55.028097] [INFO] [linspector]: identifier=remote1.port33 host=a.xw3.org service=net.port status=ERROR +[2023-08-29 14:55:35.183450 +0200] [0:47:01.409824] [INFO] [linspector]: identifier=remote1.port80 host=a.xw3.org service=net.port status=OK +[2023-08-29 14:55:38.292476 +0200] [0:47:04.518850] [INFO] [linspector]: identifier=remote1.port143 host=a.xw3.org service=net.port status=OK +[2023-08-29 14:55:41.825874 +0200] [0:47:08.052248] [INFO] [linspector]: identifier=network2.gateway_dsl host=192.168.178.1 service=vendor.avm.is_connected status=OK +[2023-08-29 14:55:44.313316 +0200] [0:47:10.539690] [INFO] [linspector]: identifier=remote1.port993 host=a.xw3.org service=net.port status=OK +[2023-08-29 14:55:46.462124 +0200] [0:47:12.688498] [INFO] [linspector]: identifier=remote1.port465 host=a.xw3.org service=net.port status=OK +[2023-08-29 14:55:49.422034 +0200] [0:47:15.648408] [INFO] [linspector]: identifier=remote1.port587 host=a.xw3.org service=net.port status=OK +[2023-08-29 14:55:55.033569 +0200] [0:47:21.259943] [INFO] [linspector]: identifier=remote1.port443 host=a.xw3.org service=net.port status=OK +[2023-08-29 14:55:57.316656 +0200] [0:47:23.543030] [INFO] [linspector]: identifier=remote1.port873 host=a.xw3.org service=net.port status=OK +[2023-08-29 14:56:06.352516 +0200] [0:47:32.578890] [INFO] [linspector]: identifier=remote1.port25 host=a.xw3.org service=net.port status=OK +[2023-08-29 14:56:13.568718 +0200] [0:47:39.795092] [INFO] [linspector]: identifier=remote1.port995 host=a.xw3.org service=net.port status=OK +[2023-08-29 14:56:14.387458 +0200] [0:47:40.613832] [INFO] [linspector]: identifier=remote1.port110 host=a.xw3.org service=net.port status=OK +[2023-08-29 14:56:20.287002 +0200] [0:47:46.513376] [INFO] [linspector]: identifier=remote1.port22 host=a.xw3.org service=net.port status=OK +Unable to get a connection: HTTPConnectionPool(host='192.168.23.1', port=49000): Max retries exceeded with url: /igddesc.xml (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 101] Network is unreachable')) +[2023-08-29 14:56:24.915253 +0200] [0:47:51.141627] [WARNING] [linspector]: database query failed: (1064, "You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'Unable to get a connection: HTTPConnectionPool(host='192.168.23.1', port=4900...' at line 1") +[2023-08-29 14:56:24.915418 +0200] [0:47:51.141792] [INFO] [linspector]: identifier=network1.gateway_cable host=192.168.23.1 service=vendor.avm.is_connected status=ERROR +[2023-08-29 14:56:28.804286 +0200] [0:47:55.030660] [INFO] [linspector]: identifier=remote1.port33 host=a.xw3.org service=net.port status=ERROR +[2023-08-29 14:56:35.180972 +0200] [0:48:01.407346] [INFO] [linspector]: identifier=remote1.port80 host=a.xw3.org service=net.port status=OK +[2023-08-29 14:56:38.293637 +0200] [0:48:04.520011] [INFO] [linspector]: identifier=remote1.port143 host=a.xw3.org service=net.port status=OK +[2023-08-29 14:56:41.817739 +0200] [0:48:08.044113] [INFO] [linspector]: identifier=network2.gateway_dsl host=192.168.178.1 service=vendor.avm.is_connected status=OK +[2023-08-29 14:56:44.311963 +0200] [0:48:10.538337] [INFO] [linspector]: identifier=remote1.port993 host=a.xw3.org service=net.port status=OK +[2023-08-29 14:56:46.462092 +0200] [0:48:12.688466] [INFO] [linspector]: identifier=remote1.port465 host=a.xw3.org service=net.port status=OK +[2023-08-29 14:56:49.423510 +0200] [0:48:15.649884] [INFO] [linspector]: identifier=remote1.port587 host=a.xw3.org service=net.port status=OK +[2023-08-29 14:56:55.034231 +0200] [0:48:21.260605] [INFO] [linspector]: identifier=remote1.port443 host=a.xw3.org service=net.port status=OK +[2023-08-29 14:56:57.320436 +0200] [0:48:23.546810] [INFO] [linspector]: identifier=remote1.port873 host=a.xw3.org service=net.port status=OK +[2023-08-29 14:57:06.350833 +0200] [0:48:32.577207] [INFO] [linspector]: identifier=remote1.port25 host=a.xw3.org service=net.port status=OK +[2023-08-29 14:57:13.569244 +0200] [0:48:39.795618] [INFO] [linspector]: identifier=remote1.port995 host=a.xw3.org service=net.port status=OK +[2023-08-29 14:57:14.386164 +0200] [0:48:40.612538] [INFO] [linspector]: identifier=remote1.port110 host=a.xw3.org service=net.port status=OK +[2023-08-29 14:57:20.286991 +0200] [0:48:46.513365] [INFO] [linspector]: identifier=remote1.port22 host=a.xw3.org service=net.port status=OK +[2023-08-29 14:57:24.963430 +0200] [0:48:51.189804] [WARNING] [linspector]: database query failed: (1064, 'You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near \'Allow access for applications".\\n\', \'host\': \'192.168.23.1\', \'log\': \'identifie...\' at line 1') +[2023-08-29 14:57:24.963601 +0200] [0:48:51.189975] [INFO] [linspector]: identifier=network1.gateway_cable host=192.168.23.1 service=vendor.avm.is_connected status=ERROR +[2023-08-29 14:57:28.801155 +0200] [0:48:55.027529] [INFO] [linspector]: identifier=remote1.port33 host=a.xw3.org service=net.port status=ERROR diff --git a/linspector/database.py b/linspector/database.py index b93b19f..0802ac5 100644 --- a/linspector/database.py +++ b/linspector/database.py @@ -4,6 +4,7 @@ Copyright (c) 2013-2023 Johannes Findeisen . All Rights Reserved. See LICENSE. """ + class Database: def __init__(self, configuration, environment, log): self._configuration = configuration diff --git a/linspector/databases/none.py b/linspector/databases/none.py deleted file mode 100644 index 4002ffa..0000000 --- a/linspector/databases/none.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -This file is part of Linspector (https://linspector.org/) -Copyright (c) 2013-2023 Johannes Findeisen . All Rights Reserved. -See LICENSE. -""" - -from linspector.database import Database - - -def create(configuration, environment, log): - return NoneDatabase(configuration, environment, log) - - -class NoneDatabase(Database): - """ - This is a dummy database module to make sure all monitors can work without errors even when no - database is configured. It should always be loaded or only if no other database is selected. - """ - - @staticmethod - def insert(host, identifier, json, message, service, status, timestamp): - pass diff --git a/linspector/monitor.py b/linspector/monitor.py index 30f36ee..cdf54e0 100644 --- a/linspector/monitor.py +++ b/linspector/monitor.py @@ -10,11 +10,10 @@ import time class Monitor: - def __init__(self, configuration, databases, environment, identifier, log, - monitor_configuration, notifications, services, tasks, kwargs): + def __init__(self, configuration, environment, identifier, log, monitor_configuration, + notifications, services, tasks, kwargs): self._args = kwargs self._configuration = configuration - self._databases = databases self._enabled = True self._environment = environment self._host = monitor_configuration.get('monitor', 'host') @@ -122,26 +121,16 @@ class Monitor: self._result = self._services[self._service].execute(self._identifier, self, self._service, **self._args) self._log.debug(self._result) - self._log.debug(self._databases) - for database in self._databases: - self._databases[database].insert(self._result['host'], - self._identifier, - self._result, - self._result['log'], - self._result['service'], - self._result['status'], - int(time.time())) - self._log.info(self._result['log']) - self._log.debug(self._tasks) for task in self._tasks: - self._tasks[task].execute() - self._log.debug("task: " + task) - self._log.debug("identifier: " + self._identifier) - self._log.debug("service: " + self._service) - self._log.debug("status: " + self._result['status']) - self._log.debug("log: " + self._result['log']) - self._log.debug("json: " + str(self._result)) + self._tasks[task].execute(self._result['host'], + self._identifier, + self._result, + self._result['log'], + self._result['service'], + self._result['status'], + int(time.time())) + self._log.info(self._result['log']) except Exception as err: self._log.error(err) diff --git a/linspector/monitors.py b/linspector/monitors.py index 2efbfd4..b9558aa 100644 --- a/linspector/monitors.py +++ b/linspector/monitors.py @@ -18,9 +18,8 @@ from linspector.monitor import Monitor # if a new monitor is added at runtime it needs to be checked manually by running a "reload" command # to lish which walks thrue all scheduled jobs and when an unknown monitor is found, schedule it. class Monitors: - def __init__(self, configuration, databases, environment, log, notifications, services, tasks): + def __init__(self, configuration, environment, log, notifications, services, tasks): self._configuration = configuration - self._databases = databases self._environment = environment self._log = log self._notifications = notifications @@ -59,7 +58,6 @@ class Monitors: # create Monitor() object and copy monitor_configuration for each instance because # they else refer to the same object? copy.deepcopy(monitor_configuration)??? self._monitors[identifier] = Monitor(self._configuration, - self._databases, self._environment, identifier, self._log, diff --git a/linspector/notification.py b/linspector/notification.py index 90d5505..416d770 100644 --- a/linspector/notification.py +++ b/linspector/notification.py @@ -4,6 +4,7 @@ Copyright (c) 2013-2023 Johannes Findeisen . All Rights Reserved. See LICENSE. """ + class Notification: def __init__(self, configuration, environment, log): super().__init__() diff --git a/linspector/task.py b/linspector/task.py index 522eca8..cda242b 100644 --- a/linspector/task.py +++ b/linspector/task.py @@ -4,6 +4,7 @@ Copyright (c) 2013-2023 Johannes Findeisen . All Rights Reserved. See LICENSE. """ + class Task: def __init__(self, configuration, environment, log, **kwargs): self._args = {} diff --git a/linspector/databases/mysql.py b/linspector/tasks/mysql.py similarity index 65% rename from linspector/databases/mysql.py rename to linspector/tasks/mysql.py index 65d6526..6f437fd 100644 --- a/linspector/databases/mysql.py +++ b/linspector/tasks/mysql.py @@ -6,14 +6,14 @@ See LICENSE. import pymysql.cursors -from linspector.database import Database +from linspector.task import Task def create(configuration, environment, log): - return MySQLDatabase(configuration, environment, log) + return MySQLTask(configuration, environment, log) -class MySQLDatabase(Database): +class MySQLTask(Task): def __init__(self, configuration, environment, log): super().__init__(configuration, environment, log) @@ -22,33 +22,33 @@ class MySQLDatabase(Database): try: self._database = None - if self._configuration.get_option('databases', 'mysql_database'): + if self._configuration.get_option('tasks', 'mysql_database'): self._database = self._configuration.get_option('databases', 'mysql_database') self._host = None - if self._configuration.get_option('databases', 'mysql_host'): - self._host = self._configuration.get_option('databases', 'mysql_host') + if self._configuration.get_option('tasks', 'mysql_host'): + self._host = self._configuration.get_option('tasks', 'mysql_host') self._password = None - if self._configuration.get_option('databases', 'mysql_password'): - self._password = self._configuration.get_option('databases', 'mysql_password') + if self._configuration.get_option('tasks', 'mysql_password'): + self._password = self._configuration.get_option('tasks', 'mysql_password') self._user = None - if self._configuration.get_option('databases', 'mysql_user'): - self._user = self._configuration.get_option('databases', 'mysql_user') + if self._configuration.get_option('tasks', 'mysql_user'): + self._user = self._configuration.get_option('tasks', 'mysql_user') self._table = None - if self._configuration.get_option('databases', 'mysql_table'): - self._table = self._configuration.get_option('databases', 'mysql_table') + if self._configuration.get_option('tasks', 'mysql_table'): + self._table = self._configuration.get_option('tasks', 'mysql_table') self._user = None - if self._configuration.get_option('databases', 'mysql_user'): - self._user = self._configuration.get_option('databases', 'mysql_user') + if self._configuration.get_option('tasks', 'mysql_user'): + self._user = self._configuration.get_option('tasks', 'mysql_user') except Exception as err: - log.warning('database configuration error: {0}'.format(err)) + log.warning('task configuration error: {0}'.format(err)) - def insert(self, host, identifier, json, log, service, status, timestamp): + def execute(self, host, identifier, json, log, service, status, timestamp): try: self._connection = pymysql.connect(cursorclass=pymysql.cursors.DictCursor, database=self._database, @@ -57,7 +57,7 @@ class MySQLDatabase(Database): user=self._user) self._log.debug(self._connection) except Exception as err: - self._log.warning('database connection failed: {0}'.format(err)) + self._log.warning('task mysql connection failed: {0}'.format(err)) try: with (self._connection): @@ -82,4 +82,4 @@ class MySQLDatabase(Database): cursor.execute(sql) self._connection.commit() except Exception as err: - self._log.warning('database query failed: {0}'.format(err)) + self._log.warning('task mysql query failed: {0}'.format(err)) diff --git a/linspector/tasks/none.py b/linspector/tasks/none.py new file mode 100644 index 0000000..66c6508 --- /dev/null +++ b/linspector/tasks/none.py @@ -0,0 +1,24 @@ +""" +This file is part of Linspector (https://linspector.org/) +Copyright (c) 2013-2023 Johannes Findeisen . All Rights Reserved. +See LICENSE. +""" + +# !!! DEPRECATED AND CAN BE REMOVED BECAUSE THERE IS NO NEED ANYMORE !!! + +from linspector.task import Task + + +def create(configuration, environment, log): + return NoneTask(configuration, environment, log) + + +class NoneTask(Task): + """ + This is a dummy task module to make sure all monitors can work without errors even when no + task is configured. It should always be loaded or only if no other task is selected. + """ + + @staticmethod + def execute(host, identifier, json, message, service, status, timestamp): + pass