Made the logger an object to be multiprocessing compatible.

The initialization of the Configuration() object can not log anymore since the Log() object initialization requires the Configuration(). Maybe I will find a better solution in the future. For now, it fixes bugs and makes my life easier.
This commit is contained in:
Johannes Findeisen 2022-10-11 02:57:13 +02:00
commit e5cc2a4596
19 changed files with 201 additions and 211 deletions

View file

@ -11,13 +11,12 @@ import os
# TODO: check for all required configuration options and set defaults if needed. do this only for
# options in the "linspector" section of linspector.ini.
class Configuration:
def __init__(self, configuration_path, environment, log):
def __init__(self, configuration_path):
self.__configuration = configparser.ConfigParser()
self.__configuration_path = configuration_path
self.__environment = environment
self.__log = log
log('info', 'reading configuration file: ' + configuration_path + '/linspector.conf')
#print('[linspector] reading configuration file: ' + configuration_path +
# '/linspector.conf')
if os.path.isfile(configuration_path + '/linspector.conf'):
try:
self.__configuration.read(configuration_path + '/linspector.conf', 'utf-8')
@ -36,7 +35,7 @@ class Configuration:
section_list = glob.glob(configuration_path + '/' + target_section + '/*.conf')
for section_file in section_list:
log('debug', 'reading section file: ' + section_file)
#print('reading section file: ' + section_file)
configuration = configparser.ConfigParser()
configuration.read(section_file, 'utf-8')
for source_section in configuration.sections():
@ -47,6 +46,8 @@ class Configuration:
configuration.get(source_section,
source_section_option))
#print('configuration dump: ' + self.dump_to_ini())
def dump_to_ini(self):
dump = ''
i = 0

View file

@ -3,7 +3,6 @@ This file is part of Linspector (https://linspector.org/)
Copyright (c) 2022 Johannes Findeisen <you@hanez.org>. All Rights Reserved.
See LICENSE (MIT license)
"""
from linspector.core.helpers import log
class Environment:
@ -12,23 +11,24 @@ class Environment:
stability or runtime of Linspector.
"""
def __init__(self):
def __init__(self, log):
self.__env = {}
self.__log = log
def get_env_var(self, key):
if key in self.__env:
return self.__env[key]
else:
log('warning', __name__, 'environment var "' + key + '" not found! could be that it is '
'set later at runtime. if you '
'encounter any errors executing '
'linspector, something is wrong '
'in the logic of the code. please '
'consider reporting this as a '
'bug! btw. WARNING is not an '
'ERROR! Linspector should work '
'even with missing environment '
'variables.')
self.__log.warning('environment var "' + key + '" not found! could be that it is '
'set later at runtime. if you '
'encounter any errors executing '
'linspector, something is wrong '
'in the logic of the code. please '
'consider reporting this as a '
'bug! btw. WARNING is not an '
'ERROR! Linspector should work '
'even with missing environment '
'variables.')
return None
def set_env_var(self, key, value):

View file

@ -1,52 +0,0 @@
"""
This file is part of Linspector (https://linspector.org/)
Copyright (c) 2022 Johannes Findeisen <you@hanez.org>. All Rights Reserved.
See LICENSE (MIT license)
"""
from logging import getLogger
logger = getLogger('linspector')
def log(level, msg):
# only use inspect when log level NOTSET or DEBUG is enabled.
if logger.isEnabledFor(0) or logger.isEnabledFor(10):
import inspect
import multiprocessing
import threading
current_process = multiprocessing.current_process()
from_stack = inspect.stack()[1]
function_name = from_stack.function
line_number = str(from_stack.lineno)
module_name = inspect.getmodule(from_stack[0]).__name__
if level == 'critical':
logger.critical('[' + str(current_process.name) + ']:[' + str(threading.get_ident()) +
']:[' + str(threading.get_native_id()) + '] [' + module_name + ']:[' +
function_name + ']:[' + line_number + '] ' + str(msg))
if level == 'error':
logger.error('[' + str(current_process.name) + ']:[' + str(threading.get_ident()) +
']:[' + str(threading.get_native_id()) + '] [' + module_name + ']:[' +
function_name + ']:[' + line_number + '] ' + str(msg))
elif level == 'warning':
logger.warning('[' + str(current_process.name) + ']:[' + str(threading.get_ident()) +
']:[' + str(threading.get_native_id()) + '] [' + module_name + ']:[' +
function_name + ']:[' + line_number + '] ' + str(msg))
elif level == 'info':
logger.info('[' + str(current_process.name) + ']:[' + str(threading.get_ident()) +
']:[' + str(threading.get_native_id()) + '] [' + module_name + ']:[' +
function_name + ']:[' + line_number + '] ' + str(msg))
elif level == 'debug':
logger.debug('[' + str(current_process.name) + ']:[' + str(threading.get_ident()) +
']:[' + str(threading.get_native_id()) + '] [' + module_name + ']:[' +
function_name + ']:[' + line_number + '] ' + str(msg))
else:
if level == 'critical':
logger.critical(str(msg))
if level == 'error':
logger.error(str(msg))
elif level == 'warning':
logger.warning(str(msg))
elif level == 'info':
logger.info(str(msg))
elif level == 'debug':
logger.debug(str(msg))

View file

@ -14,13 +14,13 @@ from apscheduler.executors.pool import ThreadPoolExecutor, ProcessPoolExecutor
def job_function(log, monitor):
try:
log('debug', 'executing job_function for monitor identifier: ' + monitor.get_identifier() +
' with monitor object: ' + str(monitor))
log.debug('executing job_function for monitor identifier: ' + monitor.get_identifier() +
' with monitor object: ' + str(monitor))
monitor.handle_call()
except Exception as err:
log('warning', 'execution failed for job_function for monitor identifier: ' +
monitor.get_identifier() + ' with monitor object: ' + str(monitor) + ' error: ' +
str(err))
log.warning('execution failed for job_function for monitor identifier: ' +
monitor.get_identifier() + ' with monitor object: ' + str(monitor) +
' error: ' + str(err))
class Linspector:
@ -35,13 +35,13 @@ class Linspector:
self.__scheduler = scheduler
# load plugins
log('info', 'loading plugins...')
log.info('loading plugins...')
if configuration.get_option('linspector', 'plugins'):
plugin_list = configuration.get_option('linspector', 'plugins')
self.__plugin_list = plugin_list.split(',')
for plugin_option in self.__plugin_list:
if plugin_option not in plugins:
log('info', 'loading plugin: ' + plugin_option)
log.info('loading plugin: ' + plugin_option)
plugin_package = 'linspector.plugins.' + plugin_option.lower()
plugin_module = importlib.import_module(plugin_package)
plugin = plugin_module.create(configuration, environment, log, self)
@ -75,10 +75,10 @@ class Linspector:
job_defaults=job_defaults)
start_date = datetime.datetime.now()
log('debug', monitors.get_monitors())
log.debug(monitors.get_monitors())
monitors = self.__monitors.get_monitors()
for monitor in monitors:
log('debug', monitor)
log.debug(monitor)
if configuration.get_option('linspector', 'delta_range'):
time_delta = round(random.uniform(0.00, float(
configuration.get_option('linspector', 'delta_range'))), 3)
@ -98,6 +98,9 @@ class Linspector:
if configuration.get_option('linspector', 'timezone'):
timezone = configuration.get_option('linspector', 'timezone')
if monitors.get(monitor).get_monitor_configuration_option('args', 'timezone'):
timezone = monitors.get(monitor).get_monitor_configuration_option('args',
'timezone')
else:
timezone = 'UTC'
@ -110,8 +113,8 @@ class Linspector:
monitor_job.set_job(scheduler_job)
self.__jobs.append(monitor_job)
log('info', 'scheduling job ' + monitor + ' with delta ' + str(time_delta) +
' @' + str(new_start_date) + ' running service ' + monitor_job.get_service())
log.info('scheduling job ' + monitor + ' with delta ' + str(time_delta) +
' @' + str(new_start_date) + ' running service ' + monitor_job.get_service())
if configuration.get_option('linspector', 'start_scheduler') == 'true':
self.__scheduler['linspector'].start()

View file

@ -20,7 +20,7 @@ class Linspectord:
try:
self.__pid_file = configuration.get_option('linspector', 'pid_file')
except Exception as err:
log('critical', 'daemonize error (no pid_file set): {0}'.str(format(err)))
log.critical('daemonize error (no pid_file set): {0}'.str(format(err)))
def daemonize(self):
# daemonize the class using the UNIX double fork mechanism.
@ -32,7 +32,7 @@ class Linspectord:
# exit first parent.
sys.exit(0)
except OSError as err:
self.__log('critical', 'fork #1 failed: {0}'.str(format(err)))
self.__log.critical('fork #1 failed: {0}'.str(format(err)))
sys.exit(1)
# decouple from parent environment.
@ -47,7 +47,7 @@ class Linspectord:
# Exit from second parent.
sys.exit(0)
except OSError as err:
self.__log('critical', 'fork #2 failed: {0}'.str(format(err)))
self.__log.critical('fork #2 failed: {0}'.str(format(err)))
sys.exit(1)
# redirect standard file descriptors.
@ -73,7 +73,7 @@ class Linspectord:
def start(self):
# start the daemon. check for a pidfile to see if the daemon already runs before.
self.__log('info', 'starting daemon using pid_file: ' + str(self.__pid_file))
self.__log.info('starting daemon using pid_file: ' + str(self.__pid_file))
try:
with open(self.__pid_file, 'r') as pf:
pid = int(pf.read().strip())
@ -82,7 +82,7 @@ class Linspectord:
if pid:
message = 'pid_file {0} already exist. daemon already running?'
self.__log('critical', str(message.format(self.__pid_file)))
self.__log.critical(str(message.format(self.__pid_file)))
sys.exit(1)
# start the daemon.
@ -91,7 +91,7 @@ class Linspectord:
def stop(self):
# stop the daemon.
self.__log('info', 'stopping daemon using pid_file: ' + str(self.__pid_file))
self.__log.info('stopping daemon using pid_file: ' + str(self.__pid_file))
# get the pid from the pid file.
try:
with open(self.__pid_file, 'r') as pf:
@ -101,7 +101,7 @@ class Linspectord:
if not pid:
message = 'pid_file {0} does not exist. daemon not running?'
self.__log('error', str(message.format(self.__pid_file)))
self.__log.error(str(message.format(self.__pid_file)))
return # not an error in a restart
# try killing the daemon process.
@ -115,12 +115,12 @@ class Linspectord:
if os.path.exists(self.__pid_file):
os.remove(self.__pid_file)
else:
self.__log('critical', str(err.args))
self.__log.critical(str(err.args))
sys.exit(1)
def restart(self):
# restart the daemon.
self.__log('info', 'restarting daemon using pid_file: ' + str(self.__pid_file))
self.__log.info('restarting daemon using pid_file: ' + str(self.__pid_file))
self.stop()
self.start()

View file

@ -3,17 +3,119 @@ This file is part of Linspector (https://linspector.org/)
Copyright (c) 2022 Johannes Findeisen <you@hanez.org>. All Rights Reserved.
See LICENSE (MIT license)
"""
from linspector.core.helpers import log
import logging
import os
import sys
from logging import getLogger
from logging import handlers
logger = getLogger('linspector')
# The logger can be used by any monitor to write arbitrary data to any arbitrary place.
# Need to think about this more but some monitors are or can collect more data than needed for
# running monipyd. This enables longtime storage of collected data like in uplink.
# Maybe this can be archived by storing an arbitrary JSON string in a none defined field in the
# database. then maybe redis can be used for everything. Storing data should be optional for
# running monipyd.
class Logger:
def __init__(self, configuration, environment):
class Log:
def __init__(self, configuration, stdout, verbose):
self.__configuration = configuration
self.__environment = environment
self.__stdout = stdout
self.__verbose = verbose
if stdout:
if verbose:
self.set_level(logging.DEBUG)
else:
# setting pre initialization default log level to INFO. this changes after
# initialization of the configuration. maybe there are better solutions...?
self.set_level(logging.INFO)
stdout_formatter = logging.Formatter('[%(asctime)s] [%(levelname)s] [%(name)s] %(message)s')
stdout_handler = logging.StreamHandler(sys.stdout)
stdout_handler.setFormatter(stdout_formatter)
self.add_handler(stdout_handler)
if configuration.get_option('linspector', 'log_file'):
log_file = os.path.expanduser(configuration.get_option('linspector', 'log_file'))
if not os.path.exists(os.path.dirname(log_file)):
os.makedirs(os.path.dirname(log_file))
log_file_formatter = \
logging.Formatter('[%(asctime)s]:[%(levelname)s]:[%(name)s]:%(message)s')
if configuration.get_option('linspector', 'log_file_size'):
log_file_size_mb = int(configuration.get_option('linspector', 'log_file_size'))
log_file_size_bytes = int(log_file_size_mb * 1000000)
elif configuration.get_option('linspector', 'log_file_size_bytes'):
log_file_size_bytes = int(configuration.get_option('linspector',
'log_file_size_bytes'))
else:
# default log file size is 10000000 bytes (10MiB)
log_file_size_bytes = int(10000000)
if configuration.get_option('linspector', 'log_file_count'):
log_file_count = int(configuration.get_option('linspector', 'log_file_count'))
else:
# default log file count is 1.
log_file_count = 1
log_file_handler = logging.handlers.RotatingFileHandler(log_file,
maxBytes=log_file_size_bytes,
backupCount=log_file_count)
log_file_handler.setFormatter(log_file_formatter)
self.add_handler(log_file_handler)
log_level = 'None'
# critical errors will always show up even when no log_level is set. this is most silent.
self.set_level(logging.CRITICAL)
if configuration.get_option('linspector', 'log_level'):
log_level = str(configuration.get_option('linspector', 'log_level'))
if log_level == "error":
self.set_level(logging.ERROR)
elif log_level == "warning":
self.set_level(logging.WARNING)
elif log_level == "info":
self.set_level(logging.INFO)
elif log_level == "debug":
self.set_level(logging.DEBUG)
#elif configuration.get_option('linspector', 'log_level') != 'error' != 'warning' \
# != 'info' != 'debug':
# logger.warning('[linspector] log level: "' + log_level + '" not found!')
@staticmethod
def add_handler(handler):
logger.addHandler(handler)
@staticmethod
def critical(msg):
logger.critical(str(msg))
@staticmethod
def debug(msg):
# only use inspect when log level NOTSET or DEBUG is enabled.
if logger.isEnabledFor(0) or logger.isEnabledFor(10):
import inspect
import multiprocessing
import threading
current_process = multiprocessing.current_process()
from_stack = inspect.stack()[1]
function_name = from_stack.function
line_number = str(from_stack.lineno)
module_name = inspect.getmodule(from_stack[0]).__name__
logger.debug('[' + str(current_process.name) + ']:[' + str(threading.get_ident()) +
']:[' + str(threading.get_native_id()) + '] [' + module_name + ']:[' +
function_name + ']:[' + line_number + '] ' + str(msg))
@staticmethod
def error(msg):
logger.error(str(msg))
@staticmethod
def info(msg):
logger.info(str(msg))
@staticmethod
def warning(msg):
logger.warning(str(msg))
@staticmethod
def set_level(level):
logger.setLevel(level)

View file

@ -3,13 +3,13 @@ This file is part of Linspector (https://linspector.org/)
Copyright (c) 2022 Johannes Findeisen <you@hanez.org>. All Rights Reserved.
See LICENSE (MIT license)
"""
from linspector.core.helpers import log
# This class can maybe be used for a general data model for Linspector data processing. currently
# the is no use for it and no place known where it could be used with sense.
class Model:
def __init__(self, configuration, environment):
def __init__(self, configuration, environment, log):
self.__configuration = configuration
self._environment = environment
self.__environment = environment
self.__log = log

View file

@ -23,17 +23,17 @@ class Monitor:
try:
self.__interval = int(monitor_configuration.get('args', 'interval'))
except Exception as err:
log('warning', 'no interval set in identifier ' + identifier + ', trying to get a '
'monitor configuration '
'setting. error: ' +
str(err))
log.warning('no interval set in identifier ' + identifier + ', trying to get a monitor '
'configuration '
'setting. error: ' +
str(err))
try:
self.__interval = int(configuration.get_option('linspector', 'default_interval'))
log('warning', 'set default_interval as per core configuration with '
'identifier: ' + identifier + ' to: ' + str(self.__interval))
log.warning('set default_interval as per core configuration with '
'identifier: ' + identifier + ' to: ' + str(self.__interval))
except Exception as err:
log('warning', 'no default_interval found in core configuration for identifier ' +
identifier + ', set to default interval 300 seconds. error: ' + str(err))
log.warning('no default_interval found in core configuration for identifier ' +
identifier + ', set to default interval 300 seconds. error: ' + str(err))
# default interval is 300 seconds (5 minutes) if not set in the monitor
# configuration args or a default_interval in the core configuration.
self.__interval = 300
@ -47,7 +47,7 @@ class Monitor:
except Exception as err:
# if no service is set in the monitor configuration, the service is set to misc.dummy
# instead. just to make Linspector run but with no real result.
log('debug', 'no service set for identifier: ' + identifier + ' setting to '
log.debug('no service set for identifier: ' + identifier + ' setting to '
'misc.dummy as '
'default to ensure '
'Linspector will run. '
@ -221,13 +221,13 @@ class Monitor:
def handle_tasks(self, monitor_information):
for task in self.__tasks:
if self.status.lower() in task.get_task_type().lower():
self.__log('debug', 'executing task of type: ' + self.status)
self.__log.debug('executing task of type: ' + self.status)
# tasks can but should not be executed here. putting them in a queue is the better
# solution to execute them in a serial process.
#TaskExecutor.instance().schedule_task(monitor_information, task)
def handle_call(self):
self.__log('info', 'handle call to monitor with identifier: ' + self.__identifier)
self.__log.info('handle call to monitor with identifier: ' + self.__identifier)
#logger.debug("handle call")
#logger.debug(self.service)
if self.enabled:
@ -237,7 +237,7 @@ class Monitor:
#self.__services[self.__service].execute(self.last_execution)
self.__services[self.__service].execute(**self.__args)
except Exception as err:
self.__log('error', err)
self.__log.error(err)
#self.last_execution.set_execution_end()
@ -253,7 +253,7 @@ class Monitor:
#self.handle_tasks(self.monitor_information)
else:
self.__log('info', "job " + self.get_monitor_id() + " disabled")
self.__log.info('job ' + self.get_monitor_id() + ' disabled')
def get_host(self):
return self.host

View file

@ -22,12 +22,12 @@ class Monitors:
self.__monitors = {}
monitor_groups = os.listdir(self.__configuration.get_configuration_path() + '/monitors/')
log('debug', 'monitor groups: ' + str(monitor_groups))
log.debug('monitor groups: ' + str(monitor_groups))
for monitor_group in monitor_groups:
monitors_file_list = glob.glob(self.__configuration.get_configuration_path() +
'/monitors/' + monitor_group + '/*.conf')
log('debug', 'monitor files: ' + str(monitors_file_list))
log.debug('monitor files: ' + str(monitors_file_list))
for monitor_file in monitors_file_list:
identifier = monitor_group + '_' + os.path.splitext(os.path.basename(
monitor_file))[0]
@ -38,13 +38,13 @@ class Monitors:
kwargs = {}
for option in monitor_configuration.options('args'):
value = monitor_configuration.get('args', option)
log('debug', 'added option in ' + identifier + ' to kwargs: ' + option + ' = '
+ value)
log.debug('added option in ' + identifier + ' to kwargs: ' + option + ' = ' +
value)
kwargs[option] = value
if kwargs:
log('debug', identifier + ' args ' + str(kwargs))
log.debug(identifier + ' args ' + str(kwargs))
identifier = monitor_group + '_' + os.path.splitext(os.path.basename(
monitor_file))[0]

View file

@ -8,6 +8,6 @@ See LICENSE (MIT license)
class Plugin:
def __init__(self, configuration, environment, linspector, log):
self.__configuration = configuration
self._environment = environment
self.__environment = environment
self.__linspector = linspector
self.__log = log

View file

@ -8,5 +8,5 @@ See LICENSE (MIT license)
class Service:
def __init__(self, configuration, environment, log):
self.__configuration = configuration
self._environment = environment
self.__environment = environment
self.__log = log

View file

@ -84,12 +84,12 @@ class TaskExecutor:
try:
msg, task = self.queue.get()
if task:
self.__log('debug', "starting task execution...")
self.__log.debug('starting task execution...')
#task.execute(msg)
self.queue.task_done()
except Exception as err:
self.__log('error', "error " + str(err))
self.__log.error('error ' + str(err))
def is_instant_end(self):
return self._instantEnd

View file

@ -18,6 +18,6 @@ class DummyService(Service):
self.__log = log
def execute(self, **kwargs):
self.__log('debug', 'DummyService object ' + str(self) + ' using kwargs: ' + str(kwargs))
self.__log.debug('DummyService object ' + str(self) + ' using kwargs: ' + str(kwargs))
#log('debug', 'dummy object @' + str(self) + str(self.__kwargs['foo']))
return

View file

@ -20,5 +20,5 @@ class FritzboxPhoneStatusService(Service):
self.__log = log
def execute(self, **kwargs):
self.__log('debug', 'FritzboxPhoneStatusService object ' + str(self))
self.__log.debug('FritzboxPhoneStatusService object ' + str(self))
return

View file

@ -20,5 +20,6 @@ class FritzboxUplinkService(Service):
self.__log = log
def execute(self, **kwargs):
self.__log('debug', 'FritzboxUplinkService object ' + str(self) + ' using kwargs: ' + str(kwargs))
self.__log.debug('FritzboxUplinkService object ' + str(self) + ' using kwargs: ' +
str(kwargs))
return

View file

@ -64,10 +64,10 @@ class SpeedtestService(Service):
self.__speedtest_time_elapsed = time.perf_counter() - start
self.__environment.set_env_var('_speedtest_time_elapsed',
str(self.__speedtest_time_elapsed))
self.__log('info', 'speedtest average: ' + str(self.__speedtest_average_speed) +
', max: ' + str(self.__speedtest_maximum_speed) +
', time: ' + str(self.__speedtest_time_elapsed))
self.__log.info('speedtest average: ' + str(self.__speedtest_average_speed) +
', max: ' + str(self.__speedtest_maximum_speed) +
', time: ' + str(self.__speedtest_time_elapsed))
else:
self.__log('warning', 'could not calculate download speed!')
self.__log.warning('could not calculate download speed!')
time.sleep(self.__configuration.get_speedtest_interval())

View file

@ -12,7 +12,6 @@ def create(configuration, environment, log):
# TODO: check for all required configuration options and set defaults if needed.
class SQLiteTask(Task):
def __init__(self, configuration, environment, log):
super().__init__(configuration, environment, log)
self.__configuration = configuration