diff --git a/README.md b/README.md index eb51609..df06efc 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,10 @@ **Because people are asking... Linspector is actually more some kind of a research project for evaluating stuff, but a lot of the code is used in some private projects. I am working on the API definition mostly and there is a lot of code that must be refactored/rewrited ASAP. (2023.11.29)** -**Linspector is currently not process safe because of the GIL! I am refactoring a lot of stuff -at the moment. When running the scheduler in thread-mode everything is fine, but I really want to make it possible to run Linspector using the process-mode of APScheduler...** - -**THE CORE NEEDS A REDESIGN AND PARTLY REWRITE TO BE FULLY PROCESS SAFE! I AM WORKING ON THIS -NOW. THE CONFIGURATION INTERFACE WILL NOT BE CHANGED! (2023.10.22)** +**Linspector is now thread-safe in scheduler thread-mode. Shared mutable state (Environment, +Monitor error counts) is protected by locks and the scheduler starts after daemonization to +avoid fork-with-threads issues. Process-mode (multi-process) is still not supported — that +requires a larger architectural redesign with serializable state and IPC.** ## About diff --git a/bin/linspector b/bin/linspector index 42d520a..3c2897b 100755 --- a/bin/linspector +++ b/bin/linspector @@ -194,6 +194,7 @@ def linspector(): log.critical('daemon error: {0}'.format(err)) sys.exit(1) else: + linspector.start() try: signal.pause() except KeyboardInterrupt: diff --git a/linspector/environment.py b/linspector/environment.py index 682463a..344768d 100644 --- a/linspector/environment.py +++ b/linspector/environment.py @@ -4,6 +4,9 @@ Copyright (c) 2013-2023 Johannes Findeisen . All Rights Reserved. See LICENSE. """ +import threading + + class Environment: """ Object for storing environment variables at runtime. These variables must not affect the @@ -12,27 +15,30 @@ class Environment: def __init__(self, log): self._env = {} + self._lock = threading.Lock() self._log = log def get_env_var(self, key): - if key in self._env: - return self._env[key] - else: - self._log.warning('environment var "' + key + '" not found! could be that it is ' - 'set later at runtime. if you ' - 'encounter any errors executing ' - 'linspector, something is wrong ' - 'in the logic of the code. please ' - 'consider reporting this as a ' - 'bug! btw. a WARNING is not an ' - 'ERROR! linspector should work ' - 'even with missing environment ' - 'variables.') - return None + with self._lock: + if key in self._env: + return self._env[key] + else: + self._log.warning('environment var "' + key + '" not found! could be that it is ' + 'set later at runtime. if you ' + 'encounter any errors executing ' + 'linspector, something is wrong ' + 'in the logic of the code. please ' + 'consider reporting this as a ' + 'bug! btw. a WARNING is not an ' + 'ERROR! linspector should work ' + 'even with missing environment ' + 'variables.') + return None def set_env_var(self, key, value): - if self._env[key]: - self._log('warning', __name__, 'environment var "' + key + - ' existed and was overwritten!') + with self._lock: + if self._env.get(key): + self._log('warning', __name__, 'environment var "' + key + + ' existed and was overwritten!') - self._env[key] = value + self._env[key] = value diff --git a/linspector/linspector.py b/linspector/linspector.py index e5c56e3..cda23d1 100644 --- a/linspector/linspector.py +++ b/linspector/linspector.py @@ -35,6 +35,7 @@ class Linspector: self._plugin_list = [] self._plugins = plugins self._scheduler = scheduler + self._started = False # load plugins if configuration.get_option('linspector', 'plugins'): @@ -145,5 +146,8 @@ class Linspector: print('Number of scheduled monitors: ' + str(monitor_count)) - if configuration.get_option('linspector', 'start_scheduler') == 'true': + def start(self): + if not self._started and \ + self._configuration.get_option('linspector', 'start_scheduler') == 'true': self._scheduler['linspector'].start() + self._started = True diff --git a/linspector/linspectord.py b/linspector/linspectord.py index 7af4286..2c227ba 100644 --- a/linspector/linspectord.py +++ b/linspector/linspectord.py @@ -88,6 +88,8 @@ class Linspectord: # start the daemon. self.daemonize() + # start scheduler after fork to avoid threading issues + self._linspector.start() self.run() def stop(self): diff --git a/linspector/monitor.py b/linspector/monitor.py index 92db5e2..314eebf 100644 --- a/linspector/monitor.py +++ b/linspector/monitor.py @@ -6,6 +6,7 @@ See LICENSE. import configparser import importlib +import threading import time @@ -17,6 +18,7 @@ class Monitor: self._enabled = True self._environment = environment self._error_count = 0 + self._lock = threading.Lock() self._host = monitor_configuration.get('monitor', 'host') try: @@ -123,23 +125,26 @@ class Monitor: self._service, **self._args) notification_error_mode = 'decrease' - if self._result['status'] == 'ERROR': - self._error_count += 1 - elif self._result['status'] == 'OK': - if self._configuration.get_option('linspector', 'notification_error_mode'): - configuration_notification_error_mode = ( - self._configuration.get_option('linspector', 'notification_error_mode')) - if (configuration_notification_error_mode == 'decrease' or - configuration_notification_error_mode == 'reset'): - notification_error_mode = ( + with self._lock: + if self._result['status'] == 'ERROR': + self._error_count += 1 + elif self._result['status'] == 'OK': + if self._configuration.get_option('linspector', + 'notification_error_mode'): + configuration_notification_error_mode = ( self._configuration.get_option('linspector', 'notification_error_mode')) + if (configuration_notification_error_mode == 'decrease' or + configuration_notification_error_mode == 'reset'): + notification_error_mode = ( + self._configuration.get_option('linspector', + 'notification_error_mode')) - if notification_error_mode == 'decrease': - if self._error_count > 0: - self._error_count -= 1 - elif notification_error_mode == 'reset': - self._error_count = 0 + if notification_error_mode == 'decrease': + if self._error_count > 0: + self._error_count -= 1 + elif notification_error_mode == 'reset': + self._error_count = 0 self._log.debug('notification error mode: ' + notification_error_mode)