Note: We no longer publish the latest version of our code here. We primarily use a kumc-bmi github organization. The heron ETL repository, in particular, is not public. Peers in the informatics community should see MultiSiteDev for details on requesting access.

source: heron_load/log/etl_exc_fmt.py @ 0:42ad7288920a

heron-michigan tip
Last change on this file since 0:42ad7288920a was 0:42ad7288920a, checked in by Matt Hoag <mhoag@…>, 6 years ago

Merge with demo_concepts_3800

File size: 7.8 KB
Line 
1r'''etl_exc_fmt -- format exception report
2=========================================
3
4:copyright: Copyright 2010-2013 University of Kansas Medical Center
5            part of the `HERON open source codebase`__;
6            see NOTICE file for license details.
7
8__ http://informatics.kumc.edu/work/wiki/HERON
9
10
11Test Results from DB
12--------------------
13
14    >>> columns = ['TEST_RESULT', 'TEST_DOMAIN', 'TEST_NAME', 'TEST_VALUE',
15    ...            'RESULT_ID', 'RESULT_DATE', 'LOWER_BOUND', 'UPPER_BOUND',
16    ...            'UNITS', 'TEST_DESCRIPTION', 'MAX_RESULT_ID']
17
18    >>> from datetime import datetime
19    >>> rows = [
20    ...     ('FAIL', 'Demographics', 'hictr_mrn_leading_zero', 0, 25,
21    ...      datetime(2015, 4, 8, 15, 4, 52), 1, None, 'Patients',
22    ...      'Verify 2451: Frontiers Registry participant leading 0s', 25),
23    ...     ('FAIL', 'Flowsheets', 'val_type_seen_before', 1, 26,
24    ...      datetime(2015, 4, 8, 15, 5, 17), None, 0, 'Value Types',
25    ...      "Any value_type_c we haven't seen?", 26),
26    ...     ('FAIL', 'Medications', 'most_meds_mapped', 91, 29,
27    ...      datetime(2015, 4, 8, 15, 9, 53), 98, None, 'Percent',
28    ...      'verify RxNorm ontology covers nearly all ...', 29),
29    ...     ('FAIL', 'Medications', 'few_null_disp_ids', 0.039856516540, 27,
30    ...      datetime(2015, 4, 8, 15, 6, 7), None, 0.01, 'Percent',
31    ...      'Dispense facts that have a null" medication id. ', 27),
32    ...     ('fyi', 'test', 'test_no_bound', 1, 3,
33    ...      datetime(2015, 4, 8, 9, 43, 11), None, None, 'Nothing',
34    ...      'Just show what a "no bound" test looks like.', 3),
35    ...     ('ok', 'test', 'test_success', 1, 2,
36    ...      datetime(2015, 4, 8, 9, 43, 11), 1, 2, 'Nothing',
37    ...      'Just show what a successful test looks like.', 2),
38    ...     ('ok', 'Specimens', 'bsr_mismatch_mrn', 0, 28,
39    ...      datetime(2015, 4, 8, 15, 6, 24), None, 0.02, 'percent',
40    ...      "small number of specimens not linked by MRN", 28),
41    ...     (None, 'test', 'test_not_found_in_csv', 666, 1,
42    ...      datetime(2015, 4, 8, 9, 43, 11),
43    ...      None, None, None, None, 1),
44    ...     (None, 'Frontiers Registry', 'hictr_mrn_leading_zero', 0, 16,
45    ...      datetime(2015, 4, 8, 14, 13, 31),
46    ...      None, None, None, None, 16)]
47
48    >>> lines = ''.join(format_results(columns, rows)).strip().split('\n')
49    >>> lines[:2]
50    ['<!DOCTYPE html>', '<html>']
51    >>> lines[3]
52    '<title>ETL Check Results</title>'
53    >>> [line.strip() for line in lines if '<th>' in line]
54    ... # doctest: +NORMALIZE_WHITESPACE
55    ['<th></th><th>At</th><th>Domain</th><th>Test</th><th>Result</th>',
56     '<th>&gt;=</th><th>Value</th><th>&lt;=</th><th>(unit)</th>']
57
58    >>> lines[38:54]
59    ... # doctest: +NORMALIZE_WHITESPACE
60    ["<tr class='FAIL'>", ' <td class="num">25</td>',
61     ' <td class="datetime">2015-04-08 15:04:52</td>',
62     ' <td>Demographics</td>', ' <td>',
63     '   <strong', "        id='result_id_25' class='dropt'",
64     "        title='Verify 2451: Frontiers Registry participant leading 0s'",
65     '        >hictr_mrn_leading_zero',
66     ' </td>',
67     ' <td>FAIL</td>',
68     ' <td class="num">1.000</td>',
69     ' <td class="num">0.000</td>',
70     ' <td class="num"></td>',
71     ' <td>Patients</td>',
72     '</tr>']
73'''
74
75from ConfigParser import SafeConfigParser
76from collections import namedtuple
77from contextlib import contextmanager
78from xml.sax.saxutils import escape as xml_escape
79import logging
80
81log = logging.getLogger(__name__)
82
83
84def main(stdout, cli_access):
85    dbtrx = cli_access()
86    with dbtrx() as q:
87        q.execute('''
88        select *
89        from etl_tests_report
90        order by test_result desc, result_id
91        ''')
92        cols = [d[0] for d in q.description]
93        results = q.fetchall()
94    log.debug('DB columns: %s', cols)
95    for chunk in format_results(cols, results):
96        stdout.write(chunk)
97
98
99Result = namedtuple('Result', [
100    'test_result', 'test_domain', 'test_name', 'test_value',
101    'result_id', 'result_date', 'lower_bound', 'upper_bound',
102    'units', 'test_description', 'max_result_id'])
103
104
105def format_results(columns, rows):
106    results = [Result(*row) for row in rows]
107
108    yield '''
109<!DOCTYPE html>
110<html>
111<head>
112<title>ETL Check Results</title>
113
114<style type="text/css">
115/* ack: http://www.scientificpsychic.com/etc/css-mouseover.html */
116.dropt {border-bottom: thin dotted; background: #ffeedd;}
117.dropt:hover {text-decoration: none; background: #ffffff; z-index: 6; }
118.dropt span {position: absolute; left: -9999px;
119  margin: 20px 0 0 0px; padding: 3px 3px 3px 3px;
120  border-style:solid; border-color:black; border-width:1px; z-index: 6;}
121.dropt:hover span {left: 2%; background: #ffffff;}
122.dropt span {position: absolute; left: -9999px;
123  margin: 4px 0 0 0px; padding: 3px 3px 3px 3px;
124  border-style:solid; border-color:black; border-width:1px;}
125.dropt:hover span {margin: 20px 0 0 170px; background: #ffffff; z-index:6;}
126
127th { border-bottom: 1px solid }
128.FAIL { color: red }
129.num { text-align: right; border-right: 1px solid }
130.datetime { border-right: 1px solid }
131
132</style>
133
134</head>
135<body>
136<h1>ETL Check Results</h1>
137<p><em>TODO: specific name/title for each report?</em></p>
138<table>
139<thead>
140<tr>
141 <th></th><th>At</th><th>Domain</th><th>Test</th><th>Result</th>
142 <th>&gt;=</th><th>Value</th><th>&lt;=</th><th>(unit)</th>
143</tr>
144<thead>
145<tbody>
146'''
147
148    for r in results:
149        yield '''
150<tr class='%(test_result)s'>
151 <td class="num">%(result_id)s</td>
152 <td class="datetime">%(result_date)s</td>
153 <td>%(test_domain)s</td>
154 <td>
155   <strong
156        id='result_id_%(result_id)s' class='dropt'
157        title='%(test_description)s'
158        >%(test_name)s
159 </td>
160 <td>%(test_result)s</td>
161 <td class="num">%(lower_bound)s</td>
162 <td class="num">%(test_value)s</td>
163 <td class="num">%(upper_bound)s</td>
164 <td>%(units)s</td>
165</tr>
166''' % dict((k, markup(float(v)
167                      if k in ['lower_bound', 'test_value', 'upper_bound']
168                      and v is not None
169                      else v))
170           for (k, v) in r._asdict().items())
171
172    yield "</tbody>\n"
173
174    yield '''
175</table>
176</body>
177</html>
178'''
179
180
181def markup(x):
182    r'''Turn a string into markup to represent the string.
183
184    >>> markup('abc')
185    'abc'
186
187    >>> markup('AT&T')
188    'AT&amp;T'
189
190    >>> markup("haven't seen")
191    'haven&apos;t seen'
192
193    '''
194    return ('' if x is None else
195            '%.3f' % x if isinstance(x, type(0.1)) else
196            xml_escape(str(x), {"'": '&apos;',
197                                '"': '&quot;'}))
198
199
200def dbmgr(connect):
201    '''Make a context manager that yields cursors, given connect access.
202    '''
203    @contextmanager
204    def dbtrx():
205        conn = connect()
206        cur = conn.cursor()
207        try:
208            yield cur
209        except:
210            conn.rollback()
211            raise
212        else:
213            conn.commit()
214        finally:
215            cur.close()
216    return dbtrx
217
218
219def make_ora_connect(environ, cx, get_config):
220    cp = SafeConfigParser()
221    fp, fn = get_config()
222    cp.readfp(fp, fn)
223
224    [host, port, sid, username, password_env] = [
225        cp.get('id', n) for n in
226        'host, port, sid, username, password_env'.split(', ')]
227    port = int(port)
228    password = environ[password_env]
229    ora = cx()
230    dsn = ora.makedsn(host, port, sid)
231
232    def ora_connect():
233        return ora.connect(username, password, dsn)
234
235    return ora_connect
236
237
238if __name__ == '__main__':
239    def _privileged_main():
240        from __builtin__ import open as openf
241        from sys import argv, stdout
242        from os import environ
243
244        def cli_access():
245            logging.basicConfig(level=logging.DEBUG)  # TODO: --debug opt
246            ini = argv[1]
247
248            import cx_Oracle
249
250            getdb = make_ora_connect(environ,
251                                     lambda: cx_Oracle,
252                                     lambda: (openf(ini), ini))
253
254            return dbmgr(getdb)
255
256        main(stdout, cli_access)
257    _privileged_main()
Note: See TracBrowser for help on using the repository browser.