cleaned up all this logging stuff. logging to files does not work anymore but that will be fixed. use -q when starting linspector for now

This commit is contained in:
Johannes Findeisen 2013-10-27 02:28:20 +01:00
commit cbb106ee8a
38 changed files with 188 additions and 164 deletions

View file

@ -38,6 +38,8 @@ from linspector.core.scheduler import Scheduler
from linspector.backends.jsonrpc import JsonrpcBackend
from linspector.frontends.lish import LishFrontend
logger = logging.getLogger(__name__)
def parse_args():
parser = argparse.ArgumentParser(
@ -49,7 +51,7 @@ def parse_args():
#TODO: add --nocolor to disable colored output in lish
parser.add_argument("--version", action="version", version="%(prog)s " + str(__version__))
#TODO: make the config file a required field without -c or --config at the end eg: linspector config.json
parser.add_argument("-c", "--config", default=__default_config__,
parser.add_argument("-c", "--config", default=__default_config__, metavar="FILE",
help="select configfile to use")
parser.add_argument("-l", "--logfile", default="./log/linspector.log", metavar="FILE",
help="set logfile to use")
@ -67,50 +69,29 @@ def parse_args():
return parser.parse_args()
def setup_logging(logfile="./log/linspector.log", logLevel=logging.ERROR, logfileLevel=logging.DEBUG):
#TODO: catch all log messages (apscheduler etc.); make logging more generic to support libraries logging
logfile = path.expanduser(logfile)
if not path.exists(path.dirname(logfile)):
os.makedirs(path.dirname(logfile))
logging.basicConfig(level=logging.WARNING)
log = logging.getLogger(__name__)
#log.setLevel(logging.DEBUG)
consoleHandler = logging.StreamHandler()
consoleHandler.setLevel(logLevel)
fileHandler = logging.handlers.RotatingFileHandler(logfile, maxBytes=10485760, backupCount=4)
fileHandler.setLevel(logfileLevel)
consoleFormatter = logging.Formatter('[%(levelname)s]: %(message)s')
#TODO: if debug with file and function else without that
fileFormatter = logging.Formatter('%(asctime)s %(pathname)s %(module)s %(funcName)s %(lineno)d [%(levelname)s]: %(message)s')
consoleHandler.setFormatter(consoleFormatter)
fileHandler.setFormatter(fileFormatter)
log.addHandler(consoleHandler)
log.addHandler(fileHandler)
return log
def main():
args = parse_args()
log = setup_logging(args.logfile, args.loglevel)
logfile = path.expanduser(args.logfile)
if not path.exists(path.dirname(logfile)):
os.makedirs(path.dirname(logfile))
log.info("parsed arguments")
#logger.propagate = False
logging.basicConfig(level=args.loglevel)
#file_handler = logging.handlers.RotatingFileHandler(args.logfile, maxBytes=10485760, backupCount=4)
#file_formatter = logging.Formatter('%(asctime)s %(pathname)s %(module)s %(funcName)s %(lineno)d [%(levelname)s]: %(message)s')
#file_handler.setFormatter(file_formatter)
#file_handler.setLevel(level=args.loglevel)
#logger.addHandler(file_handler)
configParser = FullConfigParser(log)
configParser = FullConfigParser()
linConf, core = configParser.parse_config(args.config)
scheduler = Scheduler({"apscheduler.threadpool.max_threads": 1000})
print scheduler._threadpool
scheduler.start()
q = Queue.Queue()
linspector = Linspector(linConf, core, scheduler, log, q)
linspector = Linspector(linConf, core, scheduler, q)
linspector.daemon = True
linspector.start()
@ -131,7 +112,7 @@ def main():
LishFrontend(interface)
log.debug("shutting down scheduler")
logger.debug("shutting down scheduler")
shutdown_wait = True
if "shutdown_wait" in core:

View file

@ -25,6 +25,10 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
import threading
from logging import getLogger
logger = getLogger(__name__)
class Backend(threading.Thread):
def __init__(self, interface, **kwargs):

View file

@ -22,13 +22,15 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
import json
import time
from bjsonrpc.handlers import BaseHandler
from bjsonrpc import createserver
from logging import getLogger
from linspector.backends.backend import Backend
logger = getLogger(__name__)
class JsonrpcBackend(Backend):
def __init__(self, interface, config, **kwargs):

View file

@ -19,6 +19,10 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
from logging import getLogger
logger = getLogger(__name__)
class LinspectorConfig(object):
def __init__(self):

View file

@ -17,6 +17,10 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
from logging import getLogger
logger = getLogger(__name__)
class HostGroupException(Exception):
def __init__(self, msg):
@ -124,7 +128,7 @@ class HostGroupService:
return "HostgroupService { " + str([str(s) for s in self.services]) + ", " + str([p.name for p in self.periods]) + "}"
def parseHostGroupList(hostgroups, hosts, members, periods, services, log):
def parseHostGroupList(hostgroups, hosts, members, periods, services):
parsedHostGroups = []
for hgname, hgValues in hostgroups.items():
hostGroup = HostGroup(hgname)
@ -141,7 +145,7 @@ def parseHostGroupList(hostgroups, hosts, members, periods, services, log):
if service is not None:
services.append(service)
else:
log.w("could not find HostService(" + str(serviceName) + ") for host " + host.name)
logger.warning("could not find HostService(" + str(serviceName) + ") for host " + host.name)
hostGroupPeriods = [p for p in periods if p.name in servicePeriods]
hostGroup.services.append(HostGroupService(services, hostGroupPeriods))
parsedHostGroups.append(hostGroup)

View file

@ -17,6 +17,10 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
from logging import getLogger
logger = getLogger(__name__)
class LayoutException(Exception):
def __init__(self, msg):

View file

@ -19,6 +19,10 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
import re
from logging import getLogger
logger = getLogger(__name__)
class Member:
def __init__(self, id, name="", comment="", tasks=None):
@ -58,7 +62,7 @@ class MemberFilter:
return "Filter:" + str(self.filter) + " Value:" + self.value
def parseMemberList(members, filters, log):
def parseMemberList(members, filters):
parsedMembers = [Member(nameid, **values) for nameid, values in members.items()]
for member in parsedMembers:
mFilter = []
@ -72,6 +76,6 @@ def parseMemberList(members, filters, log):
memberFilter.command = re.sub('@member', replacement, filt.command)
mFilter.append(memberFilter)
if not found:
log.w("filter: " + filtername + " is not defined in member " + member.name)
logger.warning("filter: " + filtername + " is not defined in member " + member.name)
member.filters = mFilter
return parsedMembers

View file

@ -17,9 +17,11 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
import imp
from logging import getLogger
from os.path import isfile
from os.path import join
import imp
from layouts import Layout
from hostgroups import HostGroup
@ -43,6 +45,8 @@ KEY_MEMBERS = "members"
KEY_PERIODS = "periods"
KEY_CORE = "core"
logger = getLogger(__name__)
class ConfigurationException(Exception):
def __init__(self, msg, log):
@ -54,13 +58,10 @@ class ConfigurationException(Exception):
class ConfigParser:
def __init__(self, log):
def __init__(self):
"""
initializes a new ConfigParser Object
:param log: pre configured logger Object to post messages while parsing"
"""
self.log = log
self.hostgroups = {}
self.members = {}
self.periods = {}
@ -83,14 +84,14 @@ class ConfigParser:
"""
if not isfile(configFilename):
msg = "config file not found at " + str(configFilename)
raise ConfigurationException(msg, self.log)
raise ConfigurationException(msg, logger)
self.configFilename = configFilename
with open(configFilename) as cfgFile:
config = cfgFile.read()
self.log.info("reading Config: " + configFilename)
logger.info("reading Config: " + configFilename)
if self.configFilename.endswith(".yaml") or configFilename.endswith("yml"):
import yaml
@ -118,8 +119,8 @@ class ConfigParser:
item = creator(key, val)
items.append(item)
except Exception:
self.log.warning("ignoring " + msgName + ": " + key + "! reason:")
self.log.warning(str(Exception))
logger.warning("ignoring " + msgName + ": " + key + "! reason:")
logger.warning(str(Exception))
return items
def _load_module(self, clazz, modPart):
@ -163,15 +164,15 @@ class ConfigParser:
if class_check(item):
repl.append(item)
else:
self.log.warning("Ignoring class " + clazzItem["class"] + "! It does not pass the class check!")
logger.warning("Ignoring class " + clazzItem["class"] + "! It does not pass the class check!")
except ImportError, err:
self.log.warning("Could not import " + clazz + ": " + str(clazzItem) + "! reason")
self.log.warning(str(err))
logger.warning("Could not import " + clazz + ": " + str(clazzItem) + "! reason")
logger.warning(str(err))
except KeyError, k:
self.log.warning("Key " + str(k) + " not in classItem " + str(clazzItem))
logger.warning("Key " + str(k) + " not in classItem " + str(clazzItem))
except Exception, e:
self.log.warning("Error while replacing class ( " + clazz + " ): " + str(e))
logger.warning("Error while replacing class ( " + clazz + " ): " + str(e))
del items[:]
items.extend(repl)
@ -241,8 +242,6 @@ class FullConfigParser(ConfigParser):
creator = lambda name, values: HostGroup(name, **values)
self.hostgroups = self._create_raw_Object(self.dict[KEY_HOSTGROUPS], "Hostgroup", creator)
creator = parsePeriodList
periods = self._create_raw_Object(self.dict[KEY_PERIODS], "Period", creator)

View file

@ -18,6 +18,10 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
from logging import getLogger
logger = getLogger(__name__)
class Period(object):
def __init__(self, name):

View file

@ -18,16 +18,20 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
import subprocess as sp
from subprocess import Popen
from subprocess import CalledProcessError
from datetime import datetime as dt
from logging import getLogger
logger = getLogger(__name__)
#TODO: Move this to shell.py service file. this definitely is shell execution. (hanez)
class Command:
def __init__(self, command, log):
def __init__(self, command):
self.command = command
self.log = log
self.output = None
self.error = None
self.retcode = 0
@ -46,12 +50,12 @@ class Command:
try:
self.commandStart = dt.now()
self.log.info("calling command " + str(self.command) + " at " + str(self.commandStart))
logger.info("calling command " + str(self.command) + " at " + str(self.commandStart))
#self.output=sp.check_output(self.command.split())
process = Popen(self.command, stdout=sp.PIPE, stderr=sp.PIPE, shell=True)
self.output, self.error = process.communicate()
self.log.debug(str(self.output))
self.log.debug(str(self.error))
logger.debug(str(self.output))
logger.debug(str(self.error))
self.retcode = process.poll()
except CalledProcessError:
self.error = CalledProcessError.output

View file

@ -22,14 +22,11 @@ import sys
import os
import time
import atexit
from signal import SIGTERM
"""
TODO: Think about, that daemonizing this software is not our goal but when we want to that, this needs a rewrite and
should maybe move over to the daemon frontend. daemon.py will remain the the base for this and should stay in linspector/core .
Since a daemon it normally not a frontend we should think about how to handle this. When daemonizing a real frontend
like "Lish" will make no sense but a frontend like "https" or "xmpp" could be useful anyway...
"""
from signal import SIGTERM
from logging import getLogger
logger = getLogger(__name__)
class LinspectorDaemon(Daemon):

View file

@ -21,6 +21,9 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
from collections import OrderedDict
from logging import getLogger
logger = getLogger(__name__)
class LinspectorInterface(object):

View file

@ -22,6 +22,9 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
from datetime import datetime
from binascii import crc32
from logging import getLogger
logger = getLogger(__name__)
def generateId():
@ -60,9 +63,6 @@ class Job:
ret = "0" + ret
return ret
def set_logger(self, log):
self.log = log
def set_job(self, job):
self.job = job
@ -85,7 +85,7 @@ class Job:
self.jobThreshold += 1
if self.jobThreshold >= serviceThreshold:
self.log.debug("Threshold reached!")
logger.debug("Threshold reached!")
self.handle_alarm(jobInfo, self.jobThreshold - serviceThreshold)
def handle_alarm(self, jobInfo, thresholdOffset):
@ -95,8 +95,8 @@ class Job:
task.execute(jobInfo.get_message(), self.core)
def handle_call(self):
self.log.debug("handle call")
self.log.debug(self.service)
logger.debug("handle call")
logger.debug(self.service)
if self._enabled:
try:
jobInfo = JobInfo(self.hex_string(), self.host, self.service)
@ -105,14 +105,14 @@ class Job:
self.handle_threshold(jobInfo, self.service.get_threshold(), jobInfo.was_execution_successful())
self.log.debug("Code: " + str(jobInfo.get_errorcode()) + ", Message: " + str(jobInfo.get_message()))
logger.debug("Code: " + str(jobInfo.get_errorcode()) + ", Message: " + str(jobInfo.get_message()))
self.jobInfos.append(jobInfo)
except Exception, e:
self.log.debug(e)
logger.debug(e)
else:
self.log.debug("Job " + self.hex_string() + " disabled")
logger.debug("Job " + self.hex_string() + " disabled")
class JobInfo(object):

View file

@ -17,15 +17,11 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
from ..core import logger
from logging import getLogger
from ..core.daemon import Daemon
"""
TODO: Think about, that daemonizing this software is not our goal but when we want to that, this needs a rewrite and
should maybe move over to the daemon frontend. daemon.py will remain the the base for this and should stay in linspector/core .
Since a daemon it normally not a frontend we should think about how to handle this. When daemonizing a real frontend
like "Lish" will make no sense but a frontend like "https" or "xmpp" could be useful anyway...
"""
logger = getLogger(__name__)
class LinspectorDaemon(Daemon):

View file

@ -1,77 +0,0 @@
"""
Copyright (c) 2011-2013 "Johannes Findeisen and Rafael Timmerberg"
This file is part of Linspector (http://linspector.org).
Linspector is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
import logging
import logging.handlers
import os
import os.path as path
class Logger():
"""
Logger class that prints its messages and keeps them also inside a logfile
"""
def __init__(self, logfile="./linspector.log", logLevel=logging.DEBUG, logfileLevel=logging.DEBUG):
"""
initializes a new Logger object.
:param logLevel: the LoggingLevel from the console output (DEBUG default)
:param logfile: the file where to log. Logs are rotated by default.
:param logfileLevel: the LoggingLevel for the file Logger. (DEBUG default)
"""
logfile = path.expanduser(logfile)
if not path.exists(path.dirname(logfile)):
os.makedirs(path.dirname(logfile))
self.log = logging.getLogger("LinspectorLogger")
self.log.setLevel(logging.DEBUG)
consoleHandler = logging.StreamHandler()
consoleHandler.setLevel(logLevel)
fileHandler = logging.handlers.RotatingFileHandler(logfile, maxBytes=1024000, backupCount=4)
fileHandler.setLevel(logfileLevel)
consoleFormatter = logging.Formatter('[%(levelname)s]: %(message)s')
fileFormatter = logging.Formatter('%(asctime)s [%(levelname)s]: %(message)s')
consoleHandler.setFormatter(consoleFormatter)
fileHandler.setFormatter(fileFormatter)
self.log.addHandler(consoleHandler)
self.log.addHandler(fileHandler)
def d(self, message):
self.log.debug(message)
def i(self, message):
self.log.info(message)
def w(self, message):
self.log.warn(message)
def e(self, message):
self.log.error(message)
def c(self, message):
self.log.critical(message)
def close(self):
logging.shutdown()

View file

@ -17,8 +17,12 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
from logging import getLogger
from apscheduler.scheduler import Scheduler
logger = getLogger(__name__)
class Scheduler(Scheduler):
def test(self):

View file

@ -22,6 +22,10 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
from logging import getLogger
logger = getLogger(__name__)
class Frontend(object):
def __init__(self, linspectorInterface):

View file

@ -24,7 +24,9 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
import socket
from cmd import Cmd
from logging import getLogger
from shlex import split as shsplit
from linspector.frontends.frontend import Frontend
@ -38,6 +40,8 @@ GREEN = "\033[92m"
RED = "\033[91m"
END = "\033[0m"
logger = getLogger(__name__)
class LishFrontend(Frontend):
def __init__(self, linpsectorInterface):

View file

@ -23,19 +23,22 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
import datetime
import threading
from logging import getLogger
from core.job import Job
logger = getLogger(__name__)
def handle_job(jobInfo):
jobInfo.handle_call()
class Linspector(threading.Thread):
def __init__(self, linConf, core, scheduler, log, q):
def __init__(self, linConf, core, scheduler, q):
self.linConf = linConf
self.core = core
self.scheduler = scheduler
self.log = log
self.q = q
threading.Thread.__init__(self)
@ -62,6 +65,5 @@ class Linspector(threading.Thread):
schedulerJob = period.createJob(self.scheduler, job, handle_job, start_date=new_start_date)
if schedulerJob is not None:
job.set_job(schedulerJob)
job.set_logger(self.log)
jobs.append(job)
self.q.put(jobs)

View file

@ -17,6 +17,10 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
from logging import getLogger
logger = getLogger(__name__)
class Parser:
def __init__(self, **kwargs):

View file

@ -17,8 +17,12 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
from logging import getLogger
from linspector.parsers.parser import Parser
logger = getLogger(__name__)
class ShellParser(Parser):
def __init__(self, **kwargs):

View file

@ -19,8 +19,11 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
from logging import getLogger
from linspector.processors.processor import Processor
logger = getLogger(__name__)
class MariadbProcessor(Processor):
def __init__(self, **kwargs):

View file

@ -19,8 +19,11 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
from logging import getLogger
from linspector.processors.processor import Processor
logger = getLogger(__name__)
class MongodbProcessor(Processor):
def __init__(self, **kwargs):

View file

@ -19,6 +19,10 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
from logging import getLogger
logger = getLogger(__name__)
class Processor:
def __init__(self, **kwargs):

View file

@ -19,8 +19,11 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
from logging import getLogger
from linspector.processors.processor import Processor
logger = getLogger(__name__)
class SyslogProcessor(Processor):
def __init__(self, **kwargs):

View file

@ -26,8 +26,13 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
import urllib
from logging import getLogger
from linspector.services.service import Service
logger = getLogger(__name__)
class HttpService(Service):
def __init__(self, **kwargs):

View file

@ -20,9 +20,14 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
# http://code.activestate.com/recipes/409689-icmplib-library-for-creating-and-reading-icmp-pack/
from linspector.services.service import Service
import struct
from logging import getLogger
from linspector.services.service import Service
logger = getLogger(__name__)
class Packet(object):
"""Creates ICMPv4 and v6 packets.

View file

@ -17,6 +17,8 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
from logging import getLogger
KEY_PARSER = "parser"
KEY_COMMENT = "comment"
KEY_THRESHOLD = "threshold"
@ -24,6 +26,8 @@ KEY_FAILS = "fails"
KEY_PERIODS = "periods"
KEY_ARGS = "args"
logger = getLogger(__name__)
class Service(object):
def __init__(self, **kwargs):

View file

@ -19,8 +19,12 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
from logging import getLogger
from linspector.services.service import Service
logger = getLogger(__name__)
class ShellService(Service):
def __init__(self, **kwargs):

View file

@ -19,9 +19,13 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
from logging import getLogger
#from pysnmp.entity.rfc3413.oneliner import cmdgen
from linspector.services.service import Service
logger = getLogger(__name__)
class SnmpgetService(Service):
def __init__(self, **kwargs):

View file

@ -24,8 +24,13 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
import paramiko
import pprint
import os
from logging import getLogger
from linspector.services.service import Service
logger = getLogger(__name__)
class SshService(Service):
def __init__(self, **kwargs):

View file

@ -23,8 +23,13 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
import socket
from logging import getLogger
from linspector.services.service import Service
logger = getLogger(__name__)
class TcpconnectService(Service):
def __init__(self, **kwargs):

View file

@ -23,8 +23,13 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
import socket
from logging import getLogger
from linspector.services.service import Service
logger = getLogger(__name__)
class UdpconnectService(Service):
def __init__(self, **kwargs):

View file

@ -22,8 +22,13 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
import xmpp
from logging import getLogger
from linspector.tasks.task import Task
logger = getLogger(__name__)
class JabberTask(Task):
def __init__(self, **kwargs):

View file

@ -23,9 +23,14 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
import datetime
import smtplib
from email.mime.text import MIMEText
from logging import getLogger
from linspector.tasks.task import Task
logger = getLogger(__name__)
class MailTask(Task):
def __init__(self, **kwargs):

View file

@ -19,8 +19,12 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
from logging import getLogger
from linspector.tasks.task import Task
logger = getLogger(__name__)
class SmsTask(Task):
def __init__(self, **kwargs):

View file

@ -19,6 +19,10 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
from logging import getLogger
logger = getLogger(__name__)
class Task:
def set_task_type(self, taskType):

View file

@ -22,8 +22,13 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
import tweepy
from logging import getLogger
from linspector.tasks.task import Task
logger = getLogger(__name__)
class TweetTask(Task):
def __init__(self, **kwargs):