agrigis / pylogparser Goto Github PK

pyLogParser

[pylogparser] A Python project that provides common parser for log files. It is also connected with ElasticSearch in order to centralize the data and to provide a sophisticated RESTful API to request the data.

Demonstration

"""
System imports.
"""
from __future__ import print_function
import os
import tempfile

"""
Pylogparser imports.
"""
import pylogparser
from pylogparser import LogParser
from pylogparser import dump_log_es
from pylogparser import load_log_es
from pylogparser import tree

"""
First we define where to find the demonstration data.
"""
demodir = os.path.abspath(os.path.join(os.path.dirname(pylogparser.__file__),
                          "demo"))

"""
We create a log parser. All the parser data will be stored in the 'data'
instance parameter.
"""
parser = LogParser()

"""
We examplify here the parser object singleton property .
"""
print(parser)
for i in range(3):
    p = LogParser()
    print(p)

"""
We parse data from log files containing multiple processings of the same type.
"""
for basename in ("fsreconall_1.txt", "fsreconall_2.txt"):
    logfile = os.path.join(demodir, basename)
    parser.parse_logfile(
        logfile=logfile,
        job_pattern="job_\d+",
        timestamp_pattern="\d{4}-\d{2}-\d{2}T\d{2}:\d{2}",
        custom_patterns={
            "code_in_study": {
                "regex": "subjectid = \d{4}",
                "splitter": (" = ", 1)
            },
            "cmd": {
                "regex": "cmd = .*",
                "splitter": (" = ", 1)
            },
            "exitcode": {
                "regex": "exitcode = \d",
                "splitter": (" = ", 1)
            },
            "hostname": {
                "regex": "hostname = .*",
                "splitter": (" = ", 1)
            }
        },
        hierarchy={
            "job_id": {
                "code_in_study": {
                    "timestamp": {
                        "custom_data": None
                    }
                }
            }
        },
        jobs_alias="project1_freesurfer")
    print("-------", basename)
    tree(parser.data, level=2, display_content=False)
"""
We obtain 3 FreeSurfer records from 'fsreconall_1.txt':

------- fsreconall_1.txt
 +-project1_freesurfer
 | +-0001
 | | +-2015-11-10T01:33
 | +-0002
 | | +-2015-11-10T01:35
 | +-0003
 | | +-2015-11-10T01:38

And 1 more from 'fsreconall_2.txt':

------- fsreconall_2.txt
 +-project1_freesurfer
 | +-0001
 | | +-2015-11-10T01:33
 | +-0002
 | | +-2015-11-10T01:35
 | +-0003
 | | +-2015-12-03T17:04
 | | +-2015-11-10T01:38

"""

"""
We now parse JSON struct generated from two processings.
"""
for name in ("dtifit_0001", "dtifit_0002"):
    dirfiles = {
        os.path.join(demodir, name, "runtime.json"): True,
        os.path.join(demodir, name, "inputs.json"): False,
        os.path.join(demodir, name, "outputs.json"): False
    }
    parser.parse_logdir(
        logfiles=dirfiles,
        job_name="project1_dtifit",
        timestamp_key="timestamp",
        hierarchy={
            "job_name": {
                "subjectid": {
                    "timestamp": {
                        "custom_data": None
                    }
                }
            }
        },
        extract_keys=["subjectid"])
    print("-------", name)
    tree(parser.data, level=2, display_content=False)
"""
We obtain 2 DTIFit extra records:

------- dtifit_0001
 +-project1_dtifit
 | +-0001
 | | +-2016-07-13T09:20:00.007074
 +-project1_freesurfer
 | +-0001
 | | +-2015-11-10T01:33
 | +-0002
 | | +-2015-11-10T01:35
 | +-0003
 | | +-2015-12-03T17:04
 | | +-2015-11-10T01:38

------- dtifit_0002
 +-project1_dtifit
 | +-0001
 | | +-2016-07-13T09:20:00.007074
 | +-0002
 | | +-2016-07-13T09:16:32.993929
 +-project1_freesurfer
 | +-0001
 | | +-2015-11-10T01:33
 | +-0002
 | | +-2015-11-10T01:35
 | +-0003
 | | +-2015-12-03T17:04
 | | +-2015-11-10T01:38
"""

"""
We show how to organize the presneted parsing in a single Json configuration
file.
"""
descfile = os.path.join(demodir, "pylogparser_demo.json")
modify_descfile = tempfile.NamedTemporaryFile(suffix=".json").name
with open(descfile, "rt") as open_file:
    jbuffer = open_file.read().replace("DEMODIR", demodir)
with open(modify_descfile, "wt") as open_file:
    open_file.write(jbuffer)
LogParser.load(modify_descfile, verbose=0)
print("------- load 'project2' from description")
tree(parser.data, level=2, display_content=False)
"""
The same data are parsed and associated to 'project2':

------- load 'project2' from description
 +-project1_dtifit
 | +-0001
 | | +-2016-07-13T09:20:00.007074
 | +-0002
 | | +-2016-07-13T09:16:32.993929
 +-project2_dtifit
 | +-0001
 | | +-2016-07-13T09:20:00.007074
 | +-0002
 | | +-2016-07-13T09:16:32.993929
 +-project2_freesurfer
 | +-0001
 | | +-2015-11-10T01:33
 | +-0002
 | | +-2015-11-10T01:35
 | +-0003
 | | +-2015-12-03T17:04
 | | +-2015-11-10T01:38
 +-project1_freesurfer
 | +-0001
 | | +-2015-11-10T01:33
 | +-0002
 | | +-2015-11-10T01:35
 | +-0003
 | | +-2015-12-03T17:04
 | | +-2015-11-10T01:38
"""

"""
We now interact with ElasticSearch and save the log parsed data.
"""
print("------- save data in elasticsearch")
dump_log_es(parser.data, "boss", "alpine", url="localhost", port=9200,
            verbose=2)

"""
We now dump all the saved datain elasticsearch and check everything is all
right.
"""
data = load_log_es("boss", "alpine", url="localhost", port=9200, verbose=1)
print("------- load data from elasticsearch")
tree(parser.data, level=2, display_content=False)
record1 = data["project1_dtifit"]["0001"]["2016-07-13T09:20:00.007074"]
record2 = parser.data["project1_dtifit"]["0001"]["2016-07-13T09:20:00.007074"]
assert record1 == record2

"""
All right, now search all jobs final status.
"""
print("------- check status")
status = match(
    match_name="exitcode", match_value=None, login="boss", password="alpine",
    url="localhost", port=9200, index=None, doc_type=None, verbose=1)
"""
------- check status
Matches for 'exitcode=None'...
{u'project1_dtifit': {u'0001': None, u'0002': None},
 u'project1_freesurfer': {u'0001': u'0', u'0002': u'0', u'0003': u'0'},
 u'project2_dtifit': {u'0001': None, u'0002': None},
 u'project2_freesurfer': {u'0001': u'0', u'0002': u'0', u'0003': u'0'}}
"""

"""
Focus now on a specific processing.
"""
print("------- check status of one processing")
status = match(
    match_name="exitcode", match_value=None, login="boss", password="alpine",
    url="localhost", port=9200, index="project1_freesurfer", doc_type=None,
    verbose=1)
"""
------- check status of one processing
Matches for 'exitcode=None'...
{'project1_freesurfer': {u'0001': u'0', u'0002': u'0', u'0003': u'0'}}

"""

"""
Finally search where an error occured during processings.
"""
print("------- check errors")
status = match(
    match_name="exitcode", match_value="1", login="boss", password="alpine",
    url="localhost", port=9200, index=None, doc_type=None, verbose=1)
"""
------- check errors
Matches for 'exitcode=1'...
{u'project1_freesurfer': {u'0003': u'1'},
 u'project2_freesurfer': {u'0003': u'1'}}
"""