fix threading (!?) plz test
This commit is contained in:
parent
bb53338fc1
commit
475426cbe3
6 changed files with 56 additions and 39 deletions
|
|
@ -6,11 +6,10 @@
|
||||||
|
|
||||||
**Because people are asking... Linspector is actually more some kind of a research project for evaluating stuff, but a lot of the code is used in some private projects. I am working on the API definition mostly and there is a lot of code that must be refactored/rewrited ASAP. (2023.11.29)**
|
**Because people are asking... Linspector is actually more some kind of a research project for evaluating stuff, but a lot of the code is used in some private projects. I am working on the API definition mostly and there is a lot of code that must be refactored/rewrited ASAP. (2023.11.29)**
|
||||||
|
|
||||||
**Linspector is currently not process safe because of the GIL! I am refactoring a lot of stuff
|
**Linspector is now thread-safe in scheduler thread-mode. Shared mutable state (Environment,
|
||||||
at the moment. When running the scheduler in thread-mode everything is fine, but I really want to make it possible to run Linspector using the process-mode of APScheduler...**
|
Monitor error counts) is protected by locks and the scheduler starts after daemonization to
|
||||||
|
avoid fork-with-threads issues. Process-mode (multi-process) is still not supported — that
|
||||||
**THE CORE NEEDS A REDESIGN AND PARTLY REWRITE TO BE FULLY PROCESS SAFE! I AM WORKING ON THIS
|
requires a larger architectural redesign with serializable state and IPC.**
|
||||||
NOW. THE CONFIGURATION INTERFACE WILL NOT BE CHANGED! (2023.10.22)**
|
|
||||||
|
|
||||||
## About
|
## About
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -194,6 +194,7 @@ def linspector():
|
||||||
log.critical('daemon error: {0}'.format(err))
|
log.critical('daemon error: {0}'.format(err))
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
else:
|
else:
|
||||||
|
linspector.start()
|
||||||
try:
|
try:
|
||||||
signal.pause()
|
signal.pause()
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,9 @@ Copyright (c) 2013-2023 Johannes Findeisen <you@hanez.org>. All Rights Reserved.
|
||||||
See LICENSE.
|
See LICENSE.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import threading
|
||||||
|
|
||||||
|
|
||||||
class Environment:
|
class Environment:
|
||||||
"""
|
"""
|
||||||
Object for storing environment variables at runtime. These variables must not affect the
|
Object for storing environment variables at runtime. These variables must not affect the
|
||||||
|
|
@ -12,27 +15,30 @@ class Environment:
|
||||||
|
|
||||||
def __init__(self, log):
|
def __init__(self, log):
|
||||||
self._env = {}
|
self._env = {}
|
||||||
|
self._lock = threading.Lock()
|
||||||
self._log = log
|
self._log = log
|
||||||
|
|
||||||
def get_env_var(self, key):
|
def get_env_var(self, key):
|
||||||
if key in self._env:
|
with self._lock:
|
||||||
return self._env[key]
|
if key in self._env:
|
||||||
else:
|
return self._env[key]
|
||||||
self._log.warning('environment var "' + key + '" not found! could be that it is '
|
else:
|
||||||
'set later at runtime. if you '
|
self._log.warning('environment var "' + key + '" not found! could be that it is '
|
||||||
'encounter any errors executing '
|
'set later at runtime. if you '
|
||||||
'linspector, something is wrong '
|
'encounter any errors executing '
|
||||||
'in the logic of the code. please '
|
'linspector, something is wrong '
|
||||||
'consider reporting this as a '
|
'in the logic of the code. please '
|
||||||
'bug! btw. a WARNING is not an '
|
'consider reporting this as a '
|
||||||
'ERROR! linspector should work '
|
'bug! btw. a WARNING is not an '
|
||||||
'even with missing environment '
|
'ERROR! linspector should work '
|
||||||
'variables.')
|
'even with missing environment '
|
||||||
return None
|
'variables.')
|
||||||
|
return None
|
||||||
|
|
||||||
def set_env_var(self, key, value):
|
def set_env_var(self, key, value):
|
||||||
if self._env[key]:
|
with self._lock:
|
||||||
self._log('warning', __name__, 'environment var "' + key +
|
if self._env.get(key):
|
||||||
' existed and was overwritten!')
|
self._log('warning', __name__, 'environment var "' + key +
|
||||||
|
' existed and was overwritten!')
|
||||||
|
|
||||||
self._env[key] = value
|
self._env[key] = value
|
||||||
|
|
|
||||||
|
|
@ -35,6 +35,7 @@ class Linspector:
|
||||||
self._plugin_list = []
|
self._plugin_list = []
|
||||||
self._plugins = plugins
|
self._plugins = plugins
|
||||||
self._scheduler = scheduler
|
self._scheduler = scheduler
|
||||||
|
self._started = False
|
||||||
|
|
||||||
# load plugins
|
# load plugins
|
||||||
if configuration.get_option('linspector', 'plugins'):
|
if configuration.get_option('linspector', 'plugins'):
|
||||||
|
|
@ -145,5 +146,8 @@ class Linspector:
|
||||||
|
|
||||||
print('Number of scheduled monitors: ' + str(monitor_count))
|
print('Number of scheduled monitors: ' + str(monitor_count))
|
||||||
|
|
||||||
if configuration.get_option('linspector', 'start_scheduler') == 'true':
|
def start(self):
|
||||||
|
if not self._started and \
|
||||||
|
self._configuration.get_option('linspector', 'start_scheduler') == 'true':
|
||||||
self._scheduler['linspector'].start()
|
self._scheduler['linspector'].start()
|
||||||
|
self._started = True
|
||||||
|
|
|
||||||
|
|
@ -88,6 +88,8 @@ class Linspectord:
|
||||||
|
|
||||||
# start the daemon.
|
# start the daemon.
|
||||||
self.daemonize()
|
self.daemonize()
|
||||||
|
# start scheduler after fork to avoid threading issues
|
||||||
|
self._linspector.start()
|
||||||
self.run()
|
self.run()
|
||||||
|
|
||||||
def stop(self):
|
def stop(self):
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,7 @@ See LICENSE.
|
||||||
|
|
||||||
import configparser
|
import configparser
|
||||||
import importlib
|
import importlib
|
||||||
|
import threading
|
||||||
import time
|
import time
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -17,6 +18,7 @@ class Monitor:
|
||||||
self._enabled = True
|
self._enabled = True
|
||||||
self._environment = environment
|
self._environment = environment
|
||||||
self._error_count = 0
|
self._error_count = 0
|
||||||
|
self._lock = threading.Lock()
|
||||||
self._host = monitor_configuration.get('monitor', 'host')
|
self._host = monitor_configuration.get('monitor', 'host')
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|
@ -123,23 +125,26 @@ class Monitor:
|
||||||
self._service, **self._args)
|
self._service, **self._args)
|
||||||
|
|
||||||
notification_error_mode = 'decrease'
|
notification_error_mode = 'decrease'
|
||||||
if self._result['status'] == 'ERROR':
|
with self._lock:
|
||||||
self._error_count += 1
|
if self._result['status'] == 'ERROR':
|
||||||
elif self._result['status'] == 'OK':
|
self._error_count += 1
|
||||||
if self._configuration.get_option('linspector', 'notification_error_mode'):
|
elif self._result['status'] == 'OK':
|
||||||
configuration_notification_error_mode = (
|
if self._configuration.get_option('linspector',
|
||||||
self._configuration.get_option('linspector', 'notification_error_mode'))
|
'notification_error_mode'):
|
||||||
if (configuration_notification_error_mode == 'decrease' or
|
configuration_notification_error_mode = (
|
||||||
configuration_notification_error_mode == 'reset'):
|
|
||||||
notification_error_mode = (
|
|
||||||
self._configuration.get_option('linspector',
|
self._configuration.get_option('linspector',
|
||||||
'notification_error_mode'))
|
'notification_error_mode'))
|
||||||
|
if (configuration_notification_error_mode == 'decrease' or
|
||||||
|
configuration_notification_error_mode == 'reset'):
|
||||||
|
notification_error_mode = (
|
||||||
|
self._configuration.get_option('linspector',
|
||||||
|
'notification_error_mode'))
|
||||||
|
|
||||||
if notification_error_mode == 'decrease':
|
if notification_error_mode == 'decrease':
|
||||||
if self._error_count > 0:
|
if self._error_count > 0:
|
||||||
self._error_count -= 1
|
self._error_count -= 1
|
||||||
elif notification_error_mode == 'reset':
|
elif notification_error_mode == 'reset':
|
||||||
self._error_count = 0
|
self._error_count = 0
|
||||||
|
|
||||||
self._log.debug('notification error mode: ' + notification_error_mode)
|
self._log.debug('notification error mode: ' + notification_error_mode)
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue