Logger with es handler and helper utils functions

logger.py

@@ -0,0 +1,89 @@
+#!/bin/usr/python3
+
+import logging
+import os
+import json
+import uuid
+from datetime import datetime, date
+import urllib.request
+
+from utils import getConfig, timezoneOffset
+
+config = getConfig()
+LOGGER_NAME = config['logger']['name']
+esHost = config['elastic']['host']
+esPort = config['elastic']['port']
+esApiKey = config['elastic']['apiKey']
+systemTimezone = timezoneOffset()
+
+class ESHandler(logging.Handler):
+  def __init__(self, *args, **kwargs):
+    self.host = kwargs.get('host')
+    self.port = kwargs.get('port')
+    self.apiKey = kwargs.get('apiKey')
+    self.date = date.today()
+    self.sessionID = uuid.uuid4()
+
+    logging.StreamHandler.__init__(self)
+
+  def emit(self, record):
+    self.format(record)
+    datetimeTemplate = '%Y-%m-%dT%H:%M:%S.%f{}'.format(systemTimezone)
+    timestamp = datetime.fromtimestamp(record.created).strftime(datetimeTemplate)
+    headers = { 'Content-Type': 'application/json' }
+
+    indexURL = 'http://{}:{}/{}-{}/_doc'.format(self.host, self.port, LOGGER_NAME, self.date.strftime('%Y.%m.%d'))
+
+    if self.apiKey:
+      headers['Authorization'] = 'ApiKey {}'.format(self.apiKey)
+
+    doc = {
+      'severity': record.levelname,
+      'message': record.message,
+      '@timestamp': timestamp,
+      'sessionID': str(self.sessionID)
+    }
+
+    if hasattr(record, 'es'):
+      for param in record.es.values():
+        if ': {}'.format(param) in record.message:
+          doc['message'] = record.message.replace(': {}'.format(str(param)), '')
+
+      doc = {**record.es, **doc}
+
+    payload = json.dumps(doc).encode('utf8')
+    try:
+      req = urllib.request.Request(indexURL, data=payload, headers=headers)
+      response = urllib.request.urlopen(req)
+      response = response.read().decode('utf8')
+      return response
+    except Exception as err:
+      print('Error from elastic logs:', str(err))
+
+class ElasticFieldParameterAdapter(logging.LoggerAdapter):
+  def __init__(self, logger, extra={}):
+    super().__init__(logger, extra)
+
+  def process(self, msg, kwargs):
+    if kwargs == {}:
+      return (msg, kwargs)
+    extra = kwargs.get("extra", {})
+    extra.update({"es": kwargs.pop("es", True)})
+    kwargs["extra"] = extra
+    return (msg, kwargs)
+
+logger = logging.getLogger(LOGGER_NAME)
+logger.setLevel(logging.DEBUG)
+
+ch = logging.StreamHandler()
+ch.setLevel(logging.WARNING)
+
+eh = ESHandler(host=esHost, port=esPort, apiKey=esApiKey)
+eh.setLevel(logging.DEBUG)
+
+formatter = logging.Formatter('%(asctime)s %(levelname)8s | %(message)s')
+logger.addHandler(ch)
+logger.addHandler(eh)
+logger = ElasticFieldParameterAdapter(logger)
+
+

utils.py

@@ -0,0 +1,42 @@
+#!/bin/usr/python3
+import os
+import subprocess
+import yaml
+
+def loadYaml(filePath):
+  with open(filePath, "r") as stream:
+    try:
+        return yaml.safe_load(stream)
+    except yaml.YAMLError as exception:
+        print('Error: {} is unparsable'.format(filePath))
+        print(exception)
+
+def getConfig():
+  pwd = os.path.dirname(os.path.abspath(__file__))
+  path = os.path.join(pwd, 'config.yaml')
+
+  if not os.path.isfile(path):
+    print('Please fill out and rename config file. Check README for more info.')
+    exit(0)
+
+  return loadYaml(path)
+
+def timezoneOffset():
+  localTimezoneCommand = 'date +%z'
+  process = subprocess.Popen(localTimezoneCommand.split(), stdout=subprocess.PIPE)
+  output, error = process.communicate()
+
+  fallbackTimezone = '+0100'
+  if error:
+    print('Error when trying to fetch timezone: {}. Returning fallbacktimezone: {}.'.format(error, fallbackTimezone))
+    return fallbackTimezone
+
+  try:
+    output = output.decode("utf-8")
+    if '\n' in output:
+      output = output.replace('\n', '')
+    return output or fallbackTimezone
+  except Error as error:
+    print('Error occured while decoding output from system timezone: {}'.format(error))
+    return fallbackTimezone
+