Note: We no longer publish the latest version of our code here. We primarily use a kumc-bmi github organization. The heron ETL repository, in particular, is not public. Peers in the informatics community should see MultiSiteDev for details on requesting access.

source: heron_load/log/etl_log_outline.py @ 0:42ad7288920a

heron-michigan tip
Last change on this file since 0:42ad7288920a was 0:42ad7288920a, checked in by Matt Hoag <mhoag@…>, 6 years ago

Merge with demo_concepts_3800

File size: 5.5 KB
Line 
1'''etl_log_outline -- render HERON ETL log in HTML outline
2with font size indicating order of magnitude of duration.
3
4:copyright: Copyright 2014 University of Kansas Medical Center
5            part of the `HERON open source codebase`__;
6            see NOTICE file for license details.
7
8__ http://informatics.kumc.edu/work/wiki/HERON
9'''
10
11from collections import namedtuple
12import csv
13import math
14
15
16def main(argv, stdout, open_arg):
17    log_fn = argv[1]
18    log_lines = open_arg(log_fn)
19
20    stdout.write(PAGE_TOP)
21    root = Event.make(each_entry(log_lines))
22    for chunk in outline(root):
23        stdout.write(chunk)
24    stdout.write(PAGE_BOTTOM)
25
26
27EntrySchema = ('time,usec,level,log_path_name,message,dur'
28               ',skip,parent,seqno,detail').split(',')
29EntryWidth = len(EntrySchema) - 1
30Entry = namedtuple('Entry', EntrySchema)
31
32
33def fmt_entry(e):
34    bg = '<small>%s[%s]%s</small> ' % (
35        e.time, e.level,
36        '' if e.skip.strip() == 'REQ' else '?')
37
38    fg = (
39        '<code>%s</code> <b>%s</b>' % (e.detail[3], e.detail[1])
40        if e.detail[:1] == ['config']
41        else '<b>%s</b> %s' % (e.detail[1], e.message)
42        if e.detail[:1] == ['task']
43        # line,2,result,,rowcount,-1,script,$ensure_dblink
44        else '%s:%s %s rows:\n<pre>%s</pre>' % (
45            e.detail[9], e.detail[3], e.detail[7], e.detail[1])
46        if e.detail[:1] == ['code']
47        else e.message if not e.detail
48        else e.message if e.detail[:1] == ['script']
49        else e.detail[1] if e.detail[:1] == ['access']
50        else '%s chunks' % e.detail[1] if e.detail[:1] == ['chunk_qty']
51        else str(e))
52
53    outcome = e.dur.rsplit('.', 1)[0] if e.dur else ''
54
55    return bg + fg + ' ' + outcome
56
57
58def make_entry(item):
59    return Entry(*(item[:EntryWidth]
60                   + [item[EntryWidth:]]))
61
62
63def each_entry(lines):
64    return (make_entry(item) for item in csv.reader(lines))
65
66
67class Event(object):
68    def __init__(self, start_entry):
69        self.start = start_entry
70        self.children = []
71        self.end = None
72
73    @classmethod
74    def make(cls, entries):
75        entries.next()  # skip header
76
77        root = cls(start_entry=None)
78        by_seqno = {'0': root}
79        ancestor_ids = ['0']
80        parent = root
81
82        for entry in entries:
83            event = by_seqno.setdefault(entry.seqno, cls(entry))
84            if entry is event.start:
85                if [entry.parent] == ancestor_ids[-1:]:
86                    pass
87                elif entry.parent not in ancestor_ids:
88                    ancestor_ids.append(entry.parent)
89                    parent = by_seqno[entry.parent]
90                else:
91                    while entry.parent != ancestor_ids[-1]:
92                        ancestor_ids.pop()
93                    parent = by_seqno[ancestor_ids[-1]]
94                parent.children.append(event)
95            else:
96                assert(event.end is None)
97                event.end = entry
98
99        return root
100
101
102def outline(event):
103    if not event.children:
104        return
105
106    if event.start:
107        yield '<ol compact="compact">\n'
108    else:
109        yield '<ol class="xoxo">\n'  # root
110
111    for child in event.children:
112        yield '<li id="seq_%s" class="dur_%s">' % (
113            child.start.seqno, dur_size(child.end))
114        yield '<a>' + fmt_entry(child.end or child.start) + '</a>'
115        yield '\n'
116
117        for chunk in outline(child):
118            yield chunk
119        yield '</li>\n\n'
120    yield '</ol>\n'
121
122
123def dur_size(entry):
124    r'''Text size ~ log(duration)
125
126    >>> entries = each_entry(TEST_DATA.split('\n'))
127    >>> entry = entries.next()
128    >>> entry.dur
129    '8:47:00.460566'
130    >>> dur_size(entry)
131    4
132
133    Beware very long durations:
134
135    >>> entry = entries.next()
136    >>> entry.dur
137    '1 day, 8:59:01.268949'
138    >>> dur_size(entry)
139    4
140    '''
141    if not entry:
142        return 0
143    dur = entry.dur
144    if not dur:
145        return 0
146    try:
147        h, m, s = [int(nn) for nn in dur.rsplit('.', 1)[0].split(':')]
148        seconds = s + 60 * (m + 60 * h)
149    except ValueError:
150        d = int(dur.split()[0])  # n days
151        seconds = d * 24 * 60 * 60
152    return int(math.log(1 + seconds, 10))
153
154
155PAGE_TOP = '''
156<html>
157<head>
158<title>ETL Log</title>
159<style>
160ol[compact=compact] { display: none }
161//@@ ol[compact=compact]:before { content: '...' }
162
163.dur_ { font-size: xx-small }
164.dur_0 { font-size: x-small }
165.dur_1 { font-size: small }
166.dur_2 { font-size: medium }
167.dur_3 { font-size: large }
168.dur_4 { font-size: x-large }
169.dur_5 { font-size: xx-large }
170
171// .xoxo li > a { text-decoration: underline }
172
173</style>
174</head>
175<body>
176'''
177
178# ack: http://stackoverflow.com/a/11798817
179PAGE_BOTTOM = '''
180<script src="https://code.jquery.com/jquery-1.10.2.min.js">
181</script>
182<script language="javascript">
183$(document).ready(function (){
184
185  $('.xoxo li > a').click(function() {
186        $(this).parent().children('ol').toggle();
187  });
188});
189</script>
190
191</body>
192</html>
193'''
194
195TEST_DATA = '''
1962015-04-22 06:30:28,429,INFO,heron.load_epic_flowsheets.Epic_Nursing_Observations_multiselectflows,run(Epic_Nursing_Observations_multiselectflows),8:47:00.460566,"
197REQ",1567,2400,script,Epic_Nursing_Observations_multiselectflows,upload_id,32,z_loaded,282531368,z_total,282533773
1982015-04-22 06:30:28,430,INFO,heron.load_epic_flowsheets,multi_loader(load_epic_flowsheets),"1 day, 8:59:01.268949","
199REQ",0,1567,task,load_epic_flowsheets
200'''.strip()
201
202
203if __name__ == '__main__':
204    def _with_caps():
205        from sys import argv, stdout
206
207        def open_arg(n):
208            if n not in argv:
209                raise IOError
210            return open(n)
211
212        main(argv[:], stdout, open_arg)
213
214    _with_caps()
Note: See TracBrowser for help on using the repository browser.