logwatch.py

from __future__ import print_function
from os.path import exists, join, basename, getmtime
from sys import exit
from _py2with3compatibility import getstatusoutput
from hashlib import sha256
from time import time

LOGWATCH_APACHE_IGNORE_AGENTS = [
    "www.google.com/bot.html",
    "ahrefs.com",
    "yandex.com",
    "www.exabot.com",
    "www.bing.com",
]


def run_cmd(cmd, exit_on_error=True):
    err, out = getstatusoutput(cmd)
    if err and exit_on_error:
        print(out)
        exit(1)
    return out


class logwatch(object):
    def __init__(self, service, log_dir="/var/log"):
        self.log_dir = join(log_dir, "logwatch_" + service)

    def process(self, logs, callback, **kwrds):
        if not logs:
            return True, 0
        info_file = join(self.log_dir, "info")
        if not exists("%s/logs" % self.log_dir):
            run_cmd("mkdir -p %s/logs" % self.log_dir)
        prev_lnum, prev_hash, count, data = 1, "", 0, []
        if exists(info_file):
            prev_hash, ln = run_cmd("head -1 %s" % info_file).strip().split(" ", 1)
            prev_lnum = int(ln)
            if prev_lnum < 1:
                prev_lnum = 1
        found = False
        for log in reversed(logs):
            service_log = join(self.log_dir, "logs", basename(log))
            if (len(data) > 0) and ((time() - getmtime(log)) < 600):
                return True, 0
            if found:
                if exists(service_log):
                    run_cmd("rm -f %s" % service_log)
                    continue
                else:
                    break
            run_cmd("rsync -a %s %s" % (log, service_log))
            cur_hash = sha256(run_cmd("head -1 %s" % service_log).encode()).hexdigest()
            data.insert(0, [log, service_log, 1, cur_hash, False])
            if cur_hash == prev_hash:
                found = True
                data[0][2] = prev_lnum
        data[-1][4] = True
        for item in data:
            lnum, service_log = item[2], item[1]
            get_lines_cmd = "tail -n +%s %s" % (str(lnum), service_log)
            if lnum <= 1:
                get_lines_cmd = "cat %s" % service_log
            print("Processing %s:%s" % (item[0], str(lnum)))
            lnum -= 1
            xlines = 0
            for line in run_cmd(get_lines_cmd).split("\n"):
                count += 1
                lnum += 1
                xlines += 1
                try:
                    ok = callback(line, count, **kwrds)
                except:
                    ok = False
                if not ok:
                    if (prev_lnum != lnum) or (prev_hash != item[3]):
                        run_cmd("echo '%s %s' >  %s" % (item[3], str(lnum), info_file))
                    return ok, count
                if (xlines % 1000) == 0:
                    prev_lnum = lnum
                    prev_hash = item[3]
                    run_cmd("echo '%s %s' >  %s" % (item[3], str(lnum), info_file))
            if (prev_lnum != lnum) or (prev_hash != item[3]):
                prev_lnum = lnum
                prev_hash = item[3]
                cmd = "echo '%s %s' >  %s" % (item[3], str(lnum), info_file)
                if not item[4]:
                    cmd = cmd + " && rm -f %s" % service_log
                run_cmd(cmd)
        return True, count