sábado, 17 de agosto de 2013

Where have I been running?

A small project to extract the geographical information from my GPS training device and present it using the Google Maps API. I want to see a map with all the places I've run. To do so I export the activities from "Garmin Training Center" in TCX format version 2. Then I use the first longitude and latitude of the first point of each activity. To consider them to be different they must be separated, at least, for 1 kilometer. Finally I create a html file with the Google Maps API and a marker for each starting point. Easy.

I borrowed some code from pyTrainer.

A file, called startingPoints.py, that creates a data structure to manages list of points:


import math

class Point (object):
    def __init__ (self, lat, lon):
        self._lat = lat
        self._lon = lon

    def getLatitude (self):
        return self._lat

    def getLongitude (self):
        return self._lon

    def __str__ (self):
        retVal = str(self._lat) + ',' + str(self._lon)
        return retVal

    def distanceInKm (self, other):
        d2r = math.pi / 180.0

        dlong = (self._lon - other._lon) * d2r
        dlat = (self._lat - other._lat) * d2r
        a = math.pow (math.sin (dlat / 2.0), 2) + math.cos (self._lat * d2r) * math.cos (other._lat * d2r) \
            * math.pow(math.sin (dlong/2.0), 2)
        c = 2.0 * math.atan2 (math.sqrt(a), math.sqrt(1.0 -a))
        d = 6367.0 * c
        return d

class PointList (object):
    def __init__ (self):
        self._list = []

    def appendPoint (self, point):
        found = False

        for p in self._list:
            if p.distanceInKm (point) < 1.0:
                found = True
                break

        if not found:
            self._list.append (point)

    def __str__ (self):
        retVal = ""
        for p in self._list:
            retVal += str (p) + "\n"
        return retVal

    def __getitem__ (self, key):
        return self._list [key]

    def len (self):
        return len (self._list)

    def getParameters (self):
        # Returns max, min & average
        maxLat = -50000.0
        minLat = 50000.0
        maxLon = -50000.0
        minLon = 50000.0
        avLon = 0.0
        avLat = 0.0
        total = float (len (self._list))

        for s in self._list:
            lon = s.getLongitude ()
            lat = s.getLatitude ()

            if lon > maxLon:
                maxLon = lon
            if lon < minLon:
                minLon = lon
            if lat > maxLat:
                maxLat = lat
            if lat < minLat:
                minLat = lat
            avLon += lon / total
            avLat += lat / total

        return maxLat, minLat, maxLon, minLon, avLat, avLon

Another file to generate the map file:


#!/usr/bin/env python
# -*- coding: iso-8859-1 -*-

class PrintMap (object):
    
    def __init__ (self):
        pass

    def printMap (self, filename, startingPoints):

        maxLat, minLat, maxLon, minLon, avLat, avLon = startingPoints.getParameters ()

        fp = open (filename, "w")

        fp.write (
"""<!DOCTYPE html>
<html>
<head>
    <meta name="viewport" content="initial-scale=1.0, user-scalable=no" />
    <style type="text/css">
        html { height: 100% }
        body { height: 100%; margin: 0; padding: 0 }
        #map_canvas { height: 100% }
    </style>
    <script type="text/javascript" 
        src="https://maps.googleapis.com/maps/api/js?sensor=false">    
    </script>
    <script type="text/javascript">
    function initialize ()
    {
        var mapOptions = {
            center: new google.maps.LatLng (""")
        fp.write (str(avLat) + ',' + str(avLon))
        fp.write ("""),
            zoom: 12,
            mapTypeId: google.maps.MapTypeId.ROADMAP
        };
        var map = new google.maps.Map (document.getElementById ("map_canvas"), mapOptions);""")

        fp.write ("var swlatlng = new google.maps.LatLng (%f, %f);\n" % (minLat, minLon))
        fp.write ("var nelatlng = new google.maps.LatLng (%f, %f);\n" % (maxLat, maxLon))
        fp.write ('''var boundsBox = new google.maps.LatLngBounds(swlatlng, nelatlng );\n
                    map.fitBounds(boundsBox);\n''')

        no_of_points = startingPoints.len ()
        for i in range (0, no_of_points):
            p = startingPoints [i]
            markerName = "point" + str (i)

            content = " var startmarker = new google.maps.Marker ({\n" + \
                "position: new google.maps.LatLng (" + \
                str (p.getLatitude ()) + "," + str (p.getLongitude ()) + \
                "),\n" + \
                "map: map,\n" + \
                'title: "' + markerName + '"})\n\n'
            fp.write (content)

        fp.write ("""
    }
    google.maps.event.addDomListener(window, 'load', initialize);
    </script>
</head>
<body>
    <div id="map_canvas"/></div>
</body>
</html>""")

And finally a file to parse the TCX files:

#!/usr/bin/env python
# -*- coding: iso-8859-1 -*-

import logging
import sys
import traceback
from lxml import etree
from startingPoints import Point, PointList
from printMap import PrintMap


class parseTCX (object):
    def __init__ (self):
        pass

    def validate(self, xmldoc, schema):
        logging.debug(">>")
        xmlschema_doc = etree.parse("./" + schema)
        xmlschema = etree.XMLSchema(xmlschema_doc)
        logging.debug("<<")
        return xmlschema.validate(xmldoc)

    def parseFile (self, filename):
        startingPoints = PointList ()
        logging.debug ("Parsing file " +  filename)
        try:
            xmldoc = etree.parse(filename)
            valid_xml = self.validate(xmldoc, "schemas/GarminTrainingCenterDatabase_v2.xsd")
            if (valid_xml):
                logging.debug ("It is a valid file " +  filename)
                activities = xmldoc.findall(".//{http://www.garmin.com/xmlschemas/TrainingCenterDatabase/v2}Activity")
                for activity in activities:
                    point = activity.find(".//{http://www.garmin.com/xmlschemas/TrainingCenterDatabase/v2}Trackpoint")
                    if point != None:
                        pos = point.find(".//{http://www.garmin.com/xmlschemas/TrainingCenterDatabase/v2}Position")
                        if pos != None:
                            lat = pos.find(".//{http://www.garmin.com/xmlschemas/TrainingCenterDatabase/v2}LatitudeDegrees")
                            lon = pos.find(".//{http://www.garmin.com/xmlschemas/TrainingCenterDatabase/v2}LongitudeDegrees")
                            startingPoints.appendPoint (Point (float(lat.text), float(lon.text)))
        except:
            logging.debug("Traceback: %s" % traceback.format_exc())

        return startingPoints

if __name__ == "__main__":
    logging.basicConfig(format='%(asctime)-6s: %(name)s - %(levelname)s - %(message)s', level=logging.DEBUG,
            filename="parseTCX.log")

    filename = sys.argv[1]

    P = parseTCX ()
    startingPoints = P.parseFile (filename)

    M = PrintMap ()
    M.printMap ("mapa.html", startingPoints)


domingo, 11 de agosto de 2013

A serial terminal in Python using curses

I have written this small program to interact with a device using Python. It's basically a terminal that has been written with python and curses. What is the need to write another terminal program? In my case, to have it extended. It'll be extended to parse the output and control other systems. Here is the code:


#!/usr/bin/env python

import serial
import logging 
import threading
import time
import curses
import Queue

serial_port='/dev/ttyUSB0'
g_exit_threads = False

class SerialPortThread (threading.Thread):
    def __init__ (self, port, logger, queue):
        threading.Thread.__init__ (self)
        self._port = port
        self._logger = logger
        self._queue = queue

    def run (self):
        global g_exit_threads
        self._logger.info ("SerialPortThread.run. ");
        while (not g_exit_threads):
            #line = self._serial_port.readline ()
            #print line,
            c = self._port.read ()
            if (len(c) > 0):
                self._queue.put (c)
        self._logger.info ("SerialPortThread. Exiting ");
            
            
        
class KeyboardThread (threading.Thread):
    def __init__ (self, screen, logger, port):
        threading.Thread.__init__ (self)
        self._screen = screen
        self._logger = logger
        self._port = port

    def run (self):
        global g_exit_threads
        self._logger.info ("KeyboardThread.Run")
        status = 0
        while (not g_exit_threads):
            try: 
                key = self._screen.getkey ()
                self._logger.info ("KeyboardThread.run. Event: >>" + key + "<<")
                self._port.write (key) 
                if status == 0:
                    if ord(key) == 27:
                        status = 1
                elif status == 1:
                    if ord(key) == 79:
                        status = 2
                    else:
                        status = 0
                elif status == 2:
                    if ord(key) == 80:
                        self._logger.info ("KeyboardThread. Exiting")
                        g_exit_threads = True
                        status = 0
                    else:
                        status = 0

            except:
                time.sleep (0.1);
        
class OutputThread (threading.Thread):

    def __init__ (self, screen, logger, queue):
        threading.Thread.__init__ (self)
        self._screen = screen
        self._logger = logger
        self._queue = queue

    def run (self):
        global g_exit_threads
        self._logger.info ("OutputThread.Run")
        y, x = self._screen.getmaxyx ()
        pos = 2
        while (not g_exit_threads):
            try:
                e = self._queue.get (timeout=0.1)
                self._logger.info ("OutputThread: " + str(ord(e[0])) )
                if e == '\n':
                    pos = 1
                    self._screen.scroll ()
                else:
                    self._logger.info ("Char: " + str(ord(e)))
                    if (ord(e) == 8):
                        if pos > 2:
                            pos -= 1
                        self._screen.addstr (y-1, pos, ' ')
                    elif (ord(e) == 7):
                        curses.flash ()
                    else:
                        self._screen.addstr (y-1, pos, e)
                        pos +=1
            except:
                pass

        self._logger.info ("OutputThread. Exiting")



if __name__ == "__main__":
    logging.basicConfig(format='%(asctime)-6s: %(name)s - %(levelname)s - %(message)s', level=logging.DEBUG,
            filename="console.log")

    queue = Queue.Queue ()

    screen = curses.initscr ()
    curses.noecho ()
    curses.curs_set (0)
    curses.raw ()
    screen.keypad (0)
    screen.clear ()
    screen.nodelay (True)
    y, x = screen.getmaxyx ()
    screen.setscrreg (1, y-1)
    screen.scrollok (True)

    port = serial.Serial (serial_port, 115200, timeout= 0.1);

    threads = []
    thread1 = SerialPortThread (port, logging, queue)
    threads.append (thread1)

    thread2 = KeyboardThread (screen, logging, port)
    threads.append (thread2)

    thread3 = OutputThread (screen, logging, queue)
    threads.append (thread3)


    for t in threads:
        t.setDaemon (True)
        t.start ()

    for t in threads:
        t.join ()

    # Wait for all the thread to finish
    #while threading.active_count () > 0:
    #    time.sleep (0.1)

    curses.endwin ()


viernes, 8 de marzo de 2013

Log diff script

I wanted to share a little a script I wrote to make a diff between two log files. The basic idea is to compare the log messages but ignore certain fields that are different every time, like the time stamp. What I do is remove every digit (0-9) and hex digit (a-f A-F) from every line and then call difflib to make the comparison. Then the original lines are presented in and html format. So here is the script:

#!/usr/bin/python



import difflib

import sys

import re

import itertools



def charJunk (ch):

    ch  = re.sub (r"[A-Fa-f0-9]", "", ch)



    if len(ch) == 0:

        return True

    else:

        return False



def filter (line):

    return  re.sub (r"[A-Fa-f0-9]", "", line)



def printResultsHtml (result, file1Name, file2Name):

    text1 = '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" \n\

          "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> \n\

            \n\

            <html> \n\

            \n\

            <head> \n\

                <meta http-equiv="Content-Type" \n\

                    content="text/html; charset=ISO-8859-1" /> \n\

                <title></title> \n\

                <style type="text/css"> \n\

                    table.diff {font-family:Courier; border:medium;} \n\

                    .diff_header {background-color:#e0e0e0} \n\

                    td.diff_header {text-align:right} \n\

                    .diff_next {background-color:#c0c0c0} \n\

                    .diff_add {background-color:#aaffaa} \n\

                    .diff_chg {background-color:#ffff77} \n\

                    .diff_sub {background-color:#ffaaaa} \n\

                </style> \n\

            </head>\n\

            <body>\n'



    table_header = '<table class="diff" id="difflib_chg_to0__top"\n\

           cellspacing="0" cellpadding="0" rules="groups" >\n\

        <colgroup></colgroup> <colgroup></colgroup> <colgroup></colgroup>\n\

        <colgroup></colgroup> <colgroup></colgroup> <colgroup></colgroup>\n\

        <thead><tr><th class="diff_next"><br /></th><th colspan="2"\n\

        class="diff_header">\n'



    print text1



    print table_header, 

    print file1Name,

    print '</th><th class="diff_next"><br /></th><th colspan="2" class="diff_header">',

    print file2Name,

    print '</th></tr></thead>',





    index1 = 0

    index2 = 0

    for line in result:

        if line.startswith ('-'):

            print '<tr><td class="diff_next"></td><td class="diff_header">' + str (index1 +1) + '</td>',

            print '<td nowrap="nowrap"><span class="diff_sub">' + lines1[index1] + '</span></td>',

            print '<td class="diff_next"></td><td class="diff_header">' + '</td>',

            print '<td nowrap="nowrap">' + '</td></tr>',

            index1 += 1

        elif line.startswith ('+'):

            print '<tr><td class="diff_next"></td><td class="diff_header">'  + '</td>',

            print '<td nowrap="nowrap">' + '</td>',

            print '<td class="diff_next"></td><td class="diff_header">' + str(index2 + 1) +  '</td>',

            print '<td nowrap="nowrap"><span class="diff_add">' + lines2[index2] + '</span></td></tr>',

            index2 += 1

        elif line.startswith ('?'):

            pass

        else:

            if index1 >= len (lines1):

                print "Reeeeeeeeediox"

                break

            else:

                print '<tr><td class="diff_next"></td><td class="diff_header">' + str (index1 +1) + '</td>',

                print '<td nowrap="nowrap">' + lines1[index1] + '</td>',

                print '<td class="diff_next"></td><td class="diff_header">' + str (index2 +1) + '</td>',

                print '<td nowrap="nowrap">' + lines2[index2] + '</td></tr>',

                index1 += 1

                index2 += 1

    print text2





def printResultsHtml2 (result, file1Name, file2Name):

    text1 = '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" \n\

          "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> \n\

            \n\

            <html> \n\

            \n\

            <head> \n\

                <meta http-equiv="Content-Type" \n\

                    content="text/html; charset=ISO-8859-1" /> \n\

                <title></title> \n\

                <style type="text/css"> \n\

                    table.diff {font-family:Courier; border:medium;} \n\

                    .diff_header {background-color:#e0e0e0} \n\

                    td.diff_header {text-align:right} \n\

                    .diff_next {background-color:#c0c0c0} \n\

                    .diff_add {background-color:#aaffaa} \n\

                    .diff_chg {background-color:#ffff77} \n\

                    .diff_sub {background-color:#ffaaaa} \n\

                    #one \n\

                {border:1px solid red;overflow:hidden; \n\

                float: left; \n\

                width: 48%;overflow-x: scroll; \n\

                } \n\

            #two \n\

                {border:1px solid red;overflow:hidden; \n\

                float: left; \n\

                width: 48%;overflow-x: scroll; \n\

                } \n\

            .c1 {width: 200px;} \n\

            #wrapper \n\

                { \n\

                float: left; \n\

                float/**/: none; \n\

                } \n\

            /* easy clearing */ \n\

            #wrapper:after \n\

                { \n\

                content: ' + "'.';  \n" +\

                'display: block;  \n\

                height: 0;  \n\

                clear: both;  \n\

                visibility: hidden; \n\

                } \n\

            #wrapper \n\

                { \n\

                display: inline-block; \n\

                } \n\

            /*\*/ \n\

            #wrapper \n\

                { \n\

                display: block; \n\

                } \n\

            /* end easy clearing */ \n\

                </style> \n\

            </head>\n\

            <body>\n'



    text2= '<table class="diff" summary="Legends">\n\

        <tr> <th colspan="2"> Legends </th> </tr>\n\

        <tr> <td> <table border="" summary="Colors">\n\

                      <tr><th> Colors </th> </tr>\n\

                      <tr><td class="diff_add"> Added </td></tr>\n\

                      <tr><td class="diff_chg">Changed</td> </tr>\n\

                      <tr><td class="diff_sub">Deleted</td> </tr>\n\

                  </table></td>\n\

             <td> <table border="" summary="Links">\n\

                      <tr><th colspan="2"> Links </th> </tr>\n\

                      <tr><td>(f)irst change</td> </tr>\n\

                      <tr><td>(n)ext change</td> </tr>\n\

                      <tr><td>(t)op</td> </tr>\n\

                  </table></td> </tr>\n\

    </table>\n\

    </body>\n\

    </html>\n'



    table_header = '<table class="diff" id="difflib_chg_to0__top"\n\

           cellspacing="0" cellpadding="0" rules="groups" >\n\

        <colgroup></colgroup> <colgroup></colgroup> <colgroup></colgroup>\n\

        <colgroup></colgroup> <colgroup></colgroup> <colgroup></colgroup>\n\

        <thead><tr><th class="diff_next"><br /></th><th colspan="2"\n\

        class="diff_header">\n'

    table_header2 = '<table class="diff" id="difflib_chg_to0__top"\n\

           cellspacing="0" cellpadding="0" rules="groups" >\n\

        <colgroup></colgroup> <colgroup></colgroup> <colgroup></colgroup>\n\

        <colgroup></colgroup> <colgroup></colgroup> <colgroup></colgroup>\n\

        <thead><tr><th class="diff_next"><br /></th><th colspan="2"\n\

        class="diff_header">\n'



    print text1



    print '<div id="wrapper"><div id="one">'

    print table_header, 

    print file1Name,

    print '</th></tr></thead>',



    r1, r2 = itertools.tee (result)



    index1 = 0

    index2 = 0

    for line in r1:

        if line.startswith ('-'):

            print '<tr><td class="diff_next"></td><td class="diff_header">' + str (index1 +1) + '</td>',

            print '<td nowrap="nowrap"><span class="diff_sub">' + lines1[index1] + '</span></td></tr>',

            index1 += 1

        elif line.startswith ('+'):

            print '<tr><td class="diff_next"></td><td class="diff_header">  '  + '</td>',

            print '<td nowrap="nowrap">+  ' + '</td></tr>',

            index2 += 1

        elif line.startswith ('?'):

            pass

        else:

            if index1 >= len (lines1):

                print "Reeeeeeeeediox"

                break

            else:

                print '<tr><td class="diff_next"></td><td class="diff_header">' + str (index1 +1) + '</td>',

                print '<td nowrap="nowrap">' + lines1[index1] + '</td></tr>',

                index1 += 1

                index2 += 1

    print '</table></div><div id="two">'

    print table_header2, 

    print file2Name,

    print '</th></tr></thead>',



    index1 = 0

    index2 = 0

    for line in r2:

        if line.startswith ('-'):

            print '<tr><td class="diff_next"></td><td class="diff_header">' + '</td>',

            print '<td nowrap="nowrap">- ' +  '</td></tr>',

            index1 += 1

        elif line.startswith ('+'):

            print '<tr><td class="diff_next"></td><td class="diff_header">'  + str(index2 +1 ) + '</td>',

            print '<td nowrap="nowrap"><span class="diff_add">' + lines2[index2] + '</span></td></tr>',

            index2 += 1

        elif line.startswith ('?'):

            pass

        else:

            if index1 >= len (lines1):

                print "Reeeeeeeeediox"

                break

            else:

                print '<tr><td class="diff_next"></td><td class="diff_header">' + str (index2 +1) + '</td>',

                print '<td nowrap="nowrap">' + lines2[index2] + '</td></tr>',

                index1 += 1

                index2 += 1



    print '</table></div></div>'





def printResults (result):

    index1 = 0

    index2 = 0

    for line in result:

        print (index1, index2),

        if line.startswith ('-'):

            print '- ' + lines1[index1],

            index1 += 1

        elif line.startswith ('+'):

            print '+ ' + lines2[index2],

            index2 += 1

        elif line.startswith ('?'):

            pass

        else:

            if index1 >= len (lines1):

                break

            else:

                print '  ' + lines1[index1],

                index1 += 1

                index2 += 1



def printResults2 (result):

    for line in result:

        print line,





file1 = open (sys.argv[1])

file2 = open (sys.argv[2])

    

lines1 = file1.read().splitlines (1)

lines2 = file2.read().splitlines (1)



filteredLines1 = []

filteredLines2 = []



for line in lines1:

    filteredLines1.append (filter (line))

for line in lines2:

    filteredLines2.append ( filter (line))



d = difflib.Differ ()



results = d.compare (filteredLines1, filteredLines2)



printResultsHtml2 (results, sys.argv[1], sys.argv[2])