sábado, 17 de agosto de 2013

Where have I been running?

A small project to extract the geographical information from my GPS training device and present it using the Google Maps API. I want to see a map with all the places I've run. To do so I export the activities from "Garmin Training Center" in TCX format version 2. Then I use the first longitude and latitude of the first point of each activity. To consider them to be different they must be separated, at least, for 1 kilometer. Finally I create a html file with the Google Maps API and a marker for each starting point. Easy.

I borrowed some code from pyTrainer.

A file, called startingPoints.py, that creates a data structure to manages list of points:


import math

class Point (object):
    def __init__ (self, lat, lon):
        self._lat = lat
        self._lon = lon

    def getLatitude (self):
        return self._lat

    def getLongitude (self):
        return self._lon

    def __str__ (self):
        retVal = str(self._lat) + ',' + str(self._lon)
        return retVal

    def distanceInKm (self, other):
        d2r = math.pi / 180.0

        dlong = (self._lon - other._lon) * d2r
        dlat = (self._lat - other._lat) * d2r
        a = math.pow (math.sin (dlat / 2.0), 2) + math.cos (self._lat * d2r) * math.cos (other._lat * d2r) \
            * math.pow(math.sin (dlong/2.0), 2)
        c = 2.0 * math.atan2 (math.sqrt(a), math.sqrt(1.0 -a))
        d = 6367.0 * c
        return d

class PointList (object):
    def __init__ (self):
        self._list = []

    def appendPoint (self, point):
        found = False

        for p in self._list:
            if p.distanceInKm (point) < 1.0:
                found = True
                break

        if not found:
            self._list.append (point)

    def __str__ (self):
        retVal = ""
        for p in self._list:
            retVal += str (p) + "\n"
        return retVal

    def __getitem__ (self, key):
        return self._list [key]

    def len (self):
        return len (self._list)

    def getParameters (self):
        # Returns max, min & average
        maxLat = -50000.0
        minLat = 50000.0
        maxLon = -50000.0
        minLon = 50000.0
        avLon = 0.0
        avLat = 0.0
        total = float (len (self._list))

        for s in self._list:
            lon = s.getLongitude ()
            lat = s.getLatitude ()

            if lon > maxLon:
                maxLon = lon
            if lon < minLon:
                minLon = lon
            if lat > maxLat:
                maxLat = lat
            if lat < minLat:
                minLat = lat
            avLon += lon / total
            avLat += lat / total

        return maxLat, minLat, maxLon, minLon, avLat, avLon

Another file to generate the map file:


#!/usr/bin/env python
# -*- coding: iso-8859-1 -*-

class PrintMap (object):
    
    def __init__ (self):
        pass

    def printMap (self, filename, startingPoints):

        maxLat, minLat, maxLon, minLon, avLat, avLon = startingPoints.getParameters ()

        fp = open (filename, "w")

        fp.write (
"""<!DOCTYPE html>
<html>
<head>
    <meta name="viewport" content="initial-scale=1.0, user-scalable=no" />
    <style type="text/css">
        html { height: 100% }
        body { height: 100%; margin: 0; padding: 0 }
        #map_canvas { height: 100% }
    </style>
    <script type="text/javascript" 
        src="https://maps.googleapis.com/maps/api/js?sensor=false">    
    </script>
    <script type="text/javascript">
    function initialize ()
    {
        var mapOptions = {
            center: new google.maps.LatLng (""")
        fp.write (str(avLat) + ',' + str(avLon))
        fp.write ("""),
            zoom: 12,
            mapTypeId: google.maps.MapTypeId.ROADMAP
        };
        var map = new google.maps.Map (document.getElementById ("map_canvas"), mapOptions);""")

        fp.write ("var swlatlng = new google.maps.LatLng (%f, %f);\n" % (minLat, minLon))
        fp.write ("var nelatlng = new google.maps.LatLng (%f, %f);\n" % (maxLat, maxLon))
        fp.write ('''var boundsBox = new google.maps.LatLngBounds(swlatlng, nelatlng );\n
                    map.fitBounds(boundsBox);\n''')

        no_of_points = startingPoints.len ()
        for i in range (0, no_of_points):
            p = startingPoints [i]
            markerName = "point" + str (i)

            content = " var startmarker = new google.maps.Marker ({\n" + \
                "position: new google.maps.LatLng (" + \
                str (p.getLatitude ()) + "," + str (p.getLongitude ()) + \
                "),\n" + \
                "map: map,\n" + \
                'title: "' + markerName + '"})\n\n'
            fp.write (content)

        fp.write ("""
    }
    google.maps.event.addDomListener(window, 'load', initialize);
    </script>
</head>
<body>
    <div id="map_canvas"/></div>
</body>
</html>""")

And finally a file to parse the TCX files:

#!/usr/bin/env python
# -*- coding: iso-8859-1 -*-

import logging
import sys
import traceback
from lxml import etree
from startingPoints import Point, PointList
from printMap import PrintMap


class parseTCX (object):
    def __init__ (self):
        pass

    def validate(self, xmldoc, schema):
        logging.debug(">>")
        xmlschema_doc = etree.parse("./" + schema)
        xmlschema = etree.XMLSchema(xmlschema_doc)
        logging.debug("<<")
        return xmlschema.validate(xmldoc)

    def parseFile (self, filename):
        startingPoints = PointList ()
        logging.debug ("Parsing file " +  filename)
        try:
            xmldoc = etree.parse(filename)
            valid_xml = self.validate(xmldoc, "schemas/GarminTrainingCenterDatabase_v2.xsd")
            if (valid_xml):
                logging.debug ("It is a valid file " +  filename)
                activities = xmldoc.findall(".//{http://www.garmin.com/xmlschemas/TrainingCenterDatabase/v2}Activity")
                for activity in activities:
                    point = activity.find(".//{http://www.garmin.com/xmlschemas/TrainingCenterDatabase/v2}Trackpoint")
                    if point != None:
                        pos = point.find(".//{http://www.garmin.com/xmlschemas/TrainingCenterDatabase/v2}Position")
                        if pos != None:
                            lat = pos.find(".//{http://www.garmin.com/xmlschemas/TrainingCenterDatabase/v2}LatitudeDegrees")
                            lon = pos.find(".//{http://www.garmin.com/xmlschemas/TrainingCenterDatabase/v2}LongitudeDegrees")
                            startingPoints.appendPoint (Point (float(lat.text), float(lon.text)))
        except:
            logging.debug("Traceback: %s" % traceback.format_exc())

        return startingPoints

if __name__ == "__main__":
    logging.basicConfig(format='%(asctime)-6s: %(name)s - %(levelname)s - %(message)s', level=logging.DEBUG,
            filename="parseTCX.log")

    filename = sys.argv[1]

    P = parseTCX ()
    startingPoints = P.parseFile (filename)

    M = PrintMap ()
    M.printMap ("mapa.html", startingPoints)


domingo, 11 de agosto de 2013

A serial terminal in Python using curses

I have written this small program to interact with a device using Python. It's basically a terminal that has been written with python and curses. What is the need to write another terminal program? In my case, to have it extended. It'll be extended to parse the output and control other systems. Here is the code:


#!/usr/bin/env python

import serial
import logging 
import threading
import time
import curses
import Queue

serial_port='/dev/ttyUSB0'
g_exit_threads = False

class SerialPortThread (threading.Thread):
    def __init__ (self, port, logger, queue):
        threading.Thread.__init__ (self)
        self._port = port
        self._logger = logger
        self._queue = queue

    def run (self):
        global g_exit_threads
        self._logger.info ("SerialPortThread.run. ");
        while (not g_exit_threads):
            #line = self._serial_port.readline ()
            #print line,
            c = self._port.read ()
            if (len(c) > 0):
                self._queue.put (c)
        self._logger.info ("SerialPortThread. Exiting ");
            
            
        
class KeyboardThread (threading.Thread):
    def __init__ (self, screen, logger, port):
        threading.Thread.__init__ (self)
        self._screen = screen
        self._logger = logger
        self._port = port

    def run (self):
        global g_exit_threads
        self._logger.info ("KeyboardThread.Run")
        status = 0
        while (not g_exit_threads):
            try: 
                key = self._screen.getkey ()
                self._logger.info ("KeyboardThread.run. Event: >>" + key + "<<")
                self._port.write (key) 
                if status == 0:
                    if ord(key) == 27:
                        status = 1
                elif status == 1:
                    if ord(key) == 79:
                        status = 2
                    else:
                        status = 0
                elif status == 2:
                    if ord(key) == 80:
                        self._logger.info ("KeyboardThread. Exiting")
                        g_exit_threads = True
                        status = 0
                    else:
                        status = 0

            except:
                time.sleep (0.1);
        
class OutputThread (threading.Thread):

    def __init__ (self, screen, logger, queue):
        threading.Thread.__init__ (self)
        self._screen = screen
        self._logger = logger
        self._queue = queue

    def run (self):
        global g_exit_threads
        self._logger.info ("OutputThread.Run")
        y, x = self._screen.getmaxyx ()
        pos = 2
        while (not g_exit_threads):
            try:
                e = self._queue.get (timeout=0.1)
                self._logger.info ("OutputThread: " + str(ord(e[0])) )
                if e == '\n':
                    pos = 1
                    self._screen.scroll ()
                else:
                    self._logger.info ("Char: " + str(ord(e)))
                    if (ord(e) == 8):
                        if pos > 2:
                            pos -= 1
                        self._screen.addstr (y-1, pos, ' ')
                    elif (ord(e) == 7):
                        curses.flash ()
                    else:
                        self._screen.addstr (y-1, pos, e)
                        pos +=1
            except:
                pass

        self._logger.info ("OutputThread. Exiting")



if __name__ == "__main__":
    logging.basicConfig(format='%(asctime)-6s: %(name)s - %(levelname)s - %(message)s', level=logging.DEBUG,
            filename="console.log")

    queue = Queue.Queue ()

    screen = curses.initscr ()
    curses.noecho ()
    curses.curs_set (0)
    curses.raw ()
    screen.keypad (0)
    screen.clear ()
    screen.nodelay (True)
    y, x = screen.getmaxyx ()
    screen.setscrreg (1, y-1)
    screen.scrollok (True)

    port = serial.Serial (serial_port, 115200, timeout= 0.1);

    threads = []
    thread1 = SerialPortThread (port, logging, queue)
    threads.append (thread1)

    thread2 = KeyboardThread (screen, logging, port)
    threads.append (thread2)

    thread3 = OutputThread (screen, logging, queue)
    threads.append (thread3)


    for t in threads:
        t.setDaemon (True)
        t.start ()

    for t in threads:
        t.join ()

    # Wait for all the thread to finish
    #while threading.active_count () > 0:
    #    time.sleep (0.1)

    curses.endwin ()


viernes, 8 de marzo de 2013

Log diff script

I wanted to share a little a script I wrote to make a diff between two log files. The basic idea is to compare the log messages but ignore certain fields that are different every time, like the time stamp. What I do is remove every digit (0-9) and hex digit (a-f A-F) from every line and then call difflib to make the comparison. Then the original lines are presented in and html format. So here is the script:

#!/usr/bin/python



import difflib

import sys

import re

import itertools



def charJunk (ch):

    ch  = re.sub (r"[A-Fa-f0-9]", "", ch)



    if len(ch) == 0:

        return True

    else:

        return False



def filter (line):

    return  re.sub (r"[A-Fa-f0-9]", "", line)



def printResultsHtml (result, file1Name, file2Name):

    text1 = '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" \n\

          "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> \n\

            \n\

            <html> \n\

            \n\

            <head> \n\

                <meta http-equiv="Content-Type" \n\

                    content="text/html; charset=ISO-8859-1" /> \n\

                <title></title> \n\

                <style type="text/css"> \n\

                    table.diff {font-family:Courier; border:medium;} \n\

                    .diff_header {background-color:#e0e0e0} \n\

                    td.diff_header {text-align:right} \n\

                    .diff_next {background-color:#c0c0c0} \n\

                    .diff_add {background-color:#aaffaa} \n\

                    .diff_chg {background-color:#ffff77} \n\

                    .diff_sub {background-color:#ffaaaa} \n\

                </style> \n\

            </head>\n\

            <body>\n'



    table_header = '<table class="diff" id="difflib_chg_to0__top"\n\

           cellspacing="0" cellpadding="0" rules="groups" >\n\

        <colgroup></colgroup> <colgroup></colgroup> <colgroup></colgroup>\n\

        <colgroup></colgroup> <colgroup></colgroup> <colgroup></colgroup>\n\

        <thead><tr><th class="diff_next"><br /></th><th colspan="2"\n\

        class="diff_header">\n'



    print text1



    print table_header, 

    print file1Name,

    print '</th><th class="diff_next"><br /></th><th colspan="2" class="diff_header">',

    print file2Name,

    print '</th></tr></thead>',





    index1 = 0

    index2 = 0

    for line in result:

        if line.startswith ('-'):

            print '<tr><td class="diff_next"></td><td class="diff_header">' + str (index1 +1) + '</td>',

            print '<td nowrap="nowrap"><span class="diff_sub">' + lines1[index1] + '</span></td>',

            print '<td class="diff_next"></td><td class="diff_header">' + '</td>',

            print '<td nowrap="nowrap">' + '</td></tr>',

            index1 += 1

        elif line.startswith ('+'):

            print '<tr><td class="diff_next"></td><td class="diff_header">'  + '</td>',

            print '<td nowrap="nowrap">' + '</td>',

            print '<td class="diff_next"></td><td class="diff_header">' + str(index2 + 1) +  '</td>',

            print '<td nowrap="nowrap"><span class="diff_add">' + lines2[index2] + '</span></td></tr>',

            index2 += 1

        elif line.startswith ('?'):

            pass

        else:

            if index1 >= len (lines1):

                print "Reeeeeeeeediox"

                break

            else:

                print '<tr><td class="diff_next"></td><td class="diff_header">' + str (index1 +1) + '</td>',

                print '<td nowrap="nowrap">' + lines1[index1] + '</td>',

                print '<td class="diff_next"></td><td class="diff_header">' + str (index2 +1) + '</td>',

                print '<td nowrap="nowrap">' + lines2[index2] + '</td></tr>',

                index1 += 1

                index2 += 1

    print text2





def printResultsHtml2 (result, file1Name, file2Name):

    text1 = '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" \n\

          "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> \n\

            \n\

            <html> \n\

            \n\

            <head> \n\

                <meta http-equiv="Content-Type" \n\

                    content="text/html; charset=ISO-8859-1" /> \n\

                <title></title> \n\

                <style type="text/css"> \n\

                    table.diff {font-family:Courier; border:medium;} \n\

                    .diff_header {background-color:#e0e0e0} \n\

                    td.diff_header {text-align:right} \n\

                    .diff_next {background-color:#c0c0c0} \n\

                    .diff_add {background-color:#aaffaa} \n\

                    .diff_chg {background-color:#ffff77} \n\

                    .diff_sub {background-color:#ffaaaa} \n\

                    #one \n\

                {border:1px solid red;overflow:hidden; \n\

                float: left; \n\

                width: 48%;overflow-x: scroll; \n\

                } \n\

            #two \n\

                {border:1px solid red;overflow:hidden; \n\

                float: left; \n\

                width: 48%;overflow-x: scroll; \n\

                } \n\

            .c1 {width: 200px;} \n\

            #wrapper \n\

                { \n\

                float: left; \n\

                float/**/: none; \n\

                } \n\

            /* easy clearing */ \n\

            #wrapper:after \n\

                { \n\

                content: ' + "'.';  \n" +\

                'display: block;  \n\

                height: 0;  \n\

                clear: both;  \n\

                visibility: hidden; \n\

                } \n\

            #wrapper \n\

                { \n\

                display: inline-block; \n\

                } \n\

            /*\*/ \n\

            #wrapper \n\

                { \n\

                display: block; \n\

                } \n\

            /* end easy clearing */ \n\

                </style> \n\

            </head>\n\

            <body>\n'



    text2= '<table class="diff" summary="Legends">\n\

        <tr> <th colspan="2"> Legends </th> </tr>\n\

        <tr> <td> <table border="" summary="Colors">\n\

                      <tr><th> Colors </th> </tr>\n\

                      <tr><td class="diff_add"> Added </td></tr>\n\

                      <tr><td class="diff_chg">Changed</td> </tr>\n\

                      <tr><td class="diff_sub">Deleted</td> </tr>\n\

                  </table></td>\n\

             <td> <table border="" summary="Links">\n\

                      <tr><th colspan="2"> Links </th> </tr>\n\

                      <tr><td>(f)irst change</td> </tr>\n\

                      <tr><td>(n)ext change</td> </tr>\n\

                      <tr><td>(t)op</td> </tr>\n\

                  </table></td> </tr>\n\

    </table>\n\

    </body>\n\

    </html>\n'



    table_header = '<table class="diff" id="difflib_chg_to0__top"\n\

           cellspacing="0" cellpadding="0" rules="groups" >\n\

        <colgroup></colgroup> <colgroup></colgroup> <colgroup></colgroup>\n\

        <colgroup></colgroup> <colgroup></colgroup> <colgroup></colgroup>\n\

        <thead><tr><th class="diff_next"><br /></th><th colspan="2"\n\

        class="diff_header">\n'

    table_header2 = '<table class="diff" id="difflib_chg_to0__top"\n\

           cellspacing="0" cellpadding="0" rules="groups" >\n\

        <colgroup></colgroup> <colgroup></colgroup> <colgroup></colgroup>\n\

        <colgroup></colgroup> <colgroup></colgroup> <colgroup></colgroup>\n\

        <thead><tr><th class="diff_next"><br /></th><th colspan="2"\n\

        class="diff_header">\n'



    print text1



    print '<div id="wrapper"><div id="one">'

    print table_header, 

    print file1Name,

    print '</th></tr></thead>',



    r1, r2 = itertools.tee (result)



    index1 = 0

    index2 = 0

    for line in r1:

        if line.startswith ('-'):

            print '<tr><td class="diff_next"></td><td class="diff_header">' + str (index1 +1) + '</td>',

            print '<td nowrap="nowrap"><span class="diff_sub">' + lines1[index1] + '</span></td></tr>',

            index1 += 1

        elif line.startswith ('+'):

            print '<tr><td class="diff_next"></td><td class="diff_header">  '  + '</td>',

            print '<td nowrap="nowrap">+  ' + '</td></tr>',

            index2 += 1

        elif line.startswith ('?'):

            pass

        else:

            if index1 >= len (lines1):

                print "Reeeeeeeeediox"

                break

            else:

                print '<tr><td class="diff_next"></td><td class="diff_header">' + str (index1 +1) + '</td>',

                print '<td nowrap="nowrap">' + lines1[index1] + '</td></tr>',

                index1 += 1

                index2 += 1

    print '</table></div><div id="two">'

    print table_header2, 

    print file2Name,

    print '</th></tr></thead>',



    index1 = 0

    index2 = 0

    for line in r2:

        if line.startswith ('-'):

            print '<tr><td class="diff_next"></td><td class="diff_header">' + '</td>',

            print '<td nowrap="nowrap">- ' +  '</td></tr>',

            index1 += 1

        elif line.startswith ('+'):

            print '<tr><td class="diff_next"></td><td class="diff_header">'  + str(index2 +1 ) + '</td>',

            print '<td nowrap="nowrap"><span class="diff_add">' + lines2[index2] + '</span></td></tr>',

            index2 += 1

        elif line.startswith ('?'):

            pass

        else:

            if index1 >= len (lines1):

                print "Reeeeeeeeediox"

                break

            else:

                print '<tr><td class="diff_next"></td><td class="diff_header">' + str (index2 +1) + '</td>',

                print '<td nowrap="nowrap">' + lines2[index2] + '</td></tr>',

                index1 += 1

                index2 += 1



    print '</table></div></div>'





def printResults (result):

    index1 = 0

    index2 = 0

    for line in result:

        print (index1, index2),

        if line.startswith ('-'):

            print '- ' + lines1[index1],

            index1 += 1

        elif line.startswith ('+'):

            print '+ ' + lines2[index2],

            index2 += 1

        elif line.startswith ('?'):

            pass

        else:

            if index1 >= len (lines1):

                break

            else:

                print '  ' + lines1[index1],

                index1 += 1

                index2 += 1



def printResults2 (result):

    for line in result:

        print line,





file1 = open (sys.argv[1])

file2 = open (sys.argv[2])

    

lines1 = file1.read().splitlines (1)

lines2 = file2.read().splitlines (1)



filteredLines1 = []

filteredLines2 = []



for line in lines1:

    filteredLines1.append (filter (line))

for line in lines2:

    filteredLines2.append ( filter (line))



d = difflib.Differ ()



results = d.compare (filteredLines1, filteredLines2)



printResultsHtml2 (results, sys.argv[1], sys.argv[2])


domingo, 5 de febrero de 2012

Managing tags from a mp3 with eyeD3

Removing all the tags from a mp3 file


I've been using eyeD3 to write some tags in the mp3 files from a python script. Then I had the problem to remove all the tags from a file. According to the documentation you should do something like:
tag.link("/some/file.mp3")
tag.remove()
tag.update()

but it didn't worked for me so I decided to look at the library code and I found the following solution:
tag = eyeD3.Tag()
tag.link (fileName)
frameList = []
for frame in tag.frames:
frameList.append (frame.header.id)
for l in frameList:
tag.frames.removeFramesByID(l)
tag.update()


Converting an iPod podcast file into regular audio file


The iPod uses some special tags to signal the files that are podcast or audio-books and they're stored and played in a different way. I've used eyeD3 to change files I download from certain podcast and use then as regular audio files.
The first thing is to remove an undocumented tag called PCST, then the genre needs to be changed from podcast to something else. Here is the code:
tag = eyeD3.Tag()
tag.link(fileName)
tag.frames.removeFramesByID("PCST")
g = eyeD3.Genre (None, 'Radio 3')
tag.setGenre (g)
tag.update()


Converting an audio file into an iPod podcast


We only have to invert the process. The content of the PCST tag is expected to be
"00 00 00 04 00 00 00 00 00 00" (*). Here is the code:
tag = eyeD3.Tag()
tag.link (fileName)
frameHeader = eyeD3.FrameHeader()
frameHeader.id = 'PCST'
pcstValue = struct.pack ('BBBBBBBBBB', 0, 0, 0, 4, 0, 0, 0, 0, 0, 0)
f = eyeD3.createFrame(frameHeader, pcstValue, eyeD3.TagHeader())
tag.frames.addFrame(f)

g = eyeD3.Genre (None, 'Podcast')
tag.setGenre (g)
tag.update()

domingo, 22 de enero de 2012

Python's urllib2 and podomatic

I'm creating for myself a small tool to download some podcasts. To do so, I use python and the urllib2 library. Everything went well with a number of sites until yesterday.

I discover this great podcast so I wanted to include it in the tool. The problem is that when I try to download the mp3 file I get a "httpError 403: Forbidden". This puzzles me because the web browser can access to file with no problem.

I started wireshark to look into the requests. I couldn't see any significant difference. So after a few tries I discover the issue was the User Agent field of the header. The library was sending something like:

User-agent: Python-urllib/2.6 /r/n

So I decided to change it. This is the code that does the trick:
 
user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
headers ={'User-agent': user_agent}
try:
req = urllib2.Request (url, headers=headers)
response = urllib2.urlopen (req)
except urllib2.URLError, e:
print ("Error: %(e)s with url: %(u)s" % {'e':e , 'u':url})

Now the question is why do they configure the server like that?

sábado, 21 de enero de 2012

Downloading a flash stream audio and convert it to a mp3

I've created this python script to download a flash stream audio and then convert it to mp3. My idea was to download a radio program so I can listen to it later on a not-connected portable device. This script can't be used for live streams.

The script uses 3 programs rtmpdump, ffmpeg and lame. rtmpdump is a util to download a stream using rtmp protocol. It can be easily compiled; it depends on libssl and zlib. ffmpeg and lame should be available for any LInux distribution, I compiled them for MAC.

The first step is to download the contents with rtmpdump. It's not a good idea to download everything in one go; it might take a lot of time. I've implemented a multithread mechanism instead. The number of threads and the number of seconds to download can be configured. Each downloaded audio chunk is stored in a file.

The second step is convert all the chunk to pcm raw format and concatenate all of them in a single file. The ffmpeg program is used to do so.

Finally, the raw file is converted to mp3 using lame.

The script uses a considerable amount of disk space for the temporary files. I think the script can be used, with a little modification to download video streams but I haven't tried that.

Here is the code:


#!/usr/bin/env python

import sys
import subprocess
import os
import threading

def executeCommand (cmd):
p1 = subprocess.Popen (cmd, stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
out = p1.communicate()
return out

class ChunkDownloaderThread (threading.Thread):
def __init__ (self, downloader):
threading.Thread.__init__(self)
self._downloader = downloader

def run (self):
exitLoop = False
while not exitLoop:
cmd = self._downloader.getNextThreadCmd ()
if cmd == None:
exitLoop = True
else:
executeCommand (cmd[0])
sys.stdout.write ( str(cmd[1]) + ' ')
sys.stdout.flush ()

class Downloader (object):
_rtmpDumpProgram = 'rtmpdump'
_ffmpegProgram = 'ffmpeg'
_lameProgram= 'lame'
_tmpFile = 'tempFile'
_tmpExtension = '.flv'
_rawExtension = '.raw'

def __init__ (self, chunkSize, noOfThreads):
""" The class constructor.
chunkSize: the size in seconds of each file chunk.
noOfThreads: the number of simultaneaous threads to use. """
self._chunkSize = chunkSize
self._noOfThreads = noOfThreads



def cleanTempFiles (self):
extensions = [self._tmpExtension, self._rawExtension ]
for e in extensions:
fileName = self._tmpFile + e
if os.path.isfile (fileName):
os.remove (fileName)

index = 0
exitLoop = False
while not exitLoop:
fileName = self._tmpFile + str(index) + self._tmpExtension
if os.path.isfile (fileName):
os.remove (fileName)
else:
exitLoop = True
index += 1



def prepareDownloadCmd (self, url, destinationFile):
cmd = [self._rtmpDumpProgram]
cmd += ['-r', url, '-o', destinationFile]
return cmd

def findDuration (self, url):
duration = 0.0
fileName = self._tmpFile + self._tmpExtension
cmd = self.prepareDownloadCmd (url, fileName)

# Download just one second
cmd += ['-B', '1']
output = executeCommand(cmd)[1]
output = output.split ('\n')
for line in output:
durationStr = 'duration'
infoStr = 'INFO:'
pos = line.find (infoStr)
if pos > -1:
pos = line.find (durationStr)
if pos > -1:
duration = line[pos + len(durationStr):].strip()
duration = float(duration)
break
self.cleanTempFiles()
return duration

def downloadChunk (self, url, tempFile, firstSecond=0.0, lastSecond=0.0):
cmd = self.prepareDownloadCmd (url, tempFile)
if firstSecond != 0.0:
cmd += ['-A', str(firstSecond)]
if lastSecond != 0.0:
cmd += ['-B', str(lastSecond)]
return cmd

def getNextThreadCmd (self):
retVal = None
self.theLock.acquire (True)
if self.currentChunk < self.totalChunks:
auxIndex = self.currentChunk
cmd = self.downloadChunk (self.url, self.chunkList[auxIndex][2],
self.chunkList[auxIndex][0],
self.chunkList[auxIndex][1])
retVal = [cmd, auxIndex]
self.currentChunk += 1

self.theLock.release ()

return retVal

def downloadFile (self, url):
duration = self.findDuration (url)

self.chunkList = []
chSize = float (self._chunkSize)
begin = 0.0
end = chSize
index = 0
while (begin < duration):
if end >= duration:
end = 0.0
fileName = self._tmpFile + str(index) + self._tmpExtension
self.chunkList.append ([begin, end, fileName])
begin += chSize
end += chSize
index += 1

self.totalChunks = index
self.currentChunk = 0
self.url = url
self.theLock = threading.Lock()
index = 0

sys.stdout.write ('Total ' + str(self.totalChunks) + '\n')
sys.stdout.flush ()

threads = []
for i in range (0, self._noOfThreads):
t = ChunkDownloaderThread (self)
threads.append (t)

for t in threads:
t.start()
for t in threads:
t.join()

sys.stdout.write ('\n')
sys.stdout.flush ()
return self.totalChunks

def concatenateFile (self, totalChunks):
# Flv to raw command
cmd = [self._ffmpegProgram, '-i']
completeCmd = ['-vn', '-f', 'u16le', '-acodec', 'pcm_s16le',
'-ac', '2', '-ab', '128k', '-ar', '44100', '-']#, '<', '/dev/null']
tempRawFile = self._tmpFile + self._rawExtension
f = open (tempRawFile, "wb")
sys.stdout.write ('Concatenate: \n')
sys.stdout.flush ()
for i in range (0, totalChunks):
flvFile = self._tmpFile + str(i) + self._tmpExtension
toExe = cmd + [flvFile] + completeCmd

output = executeCommand (toExe)
f.write (output[0])
sys.stdout.write (str (i) + ' ')
sys.stdout.flush ()
# Delete the chunk to save disk space
os.remove (flvFile)
sys.stdout.write ('\n')
sys.stdout.flush ()

f.close ()


def convertToMp3 (self, destination):
tempRawFile = self._tmpFile + self._rawExtension
cmd = [self._lameProgram, '-r', '-s', '44.1', '--preset', 'cd',
tempRawFile, destination]
sys.stdout.write ('Converting to mp3\n')
sys.stdout.flush ()
executeCommand (cmd)
sys.stdout.write ('Done\n')
sys.stdout.flush ()

def downloadAndConvertFile (self, url, destination):
totalChunks = self.downloadFile (url)
self.concatenateFile (totalChunks)
self.convertToMp3 (destination)
self.cleanTempFiles ()


if __name__ == '__main__':
D = Downloader(60, 15)

D.downloadAndConvertFile (sys.argv[1], sys.argv[2])

miércoles, 14 de diciembre de 2011

wxPython example. A jigsaw puzzle (3). MS Windows a double buffer

This is a continuation of the previous two entries. I finally got the chance of testing the puzzle on windows and it didn't show properly it flicked a lot. Then I learn that the Windows platform doesn't implement a double buffer so I have to do it.
The idea is painting to a bitmap and then copy it to the DC on the OnPaint event. This solution improved a lot the way application was shown it didn't removed he flicks completely.
After some research I found the solution by capturing the EVT_ERASE_BACKGROUND event and do nothing with it. This fixed all the issues. It also important to call the Update method after the Refresh.

The new board.py code is:


#!/usr/bin/python

import wx
import stateDB
import rectangle

class Board(wx.Panel):
boardId = 77
def __init__ (self, parent, pieces, count, seconds, allowRotation=True):
wx.Panel.__init__(self, parent, style = wx.NO_FULL_REPAINT_ON_RESIZE)

self.db = stateDB.StateDB()


self.pieces = pieces
self.dragged = None
self.count = count
self.completed = False

self.seconds = seconds
self.timer = wx.Timer (self, wx.ID_ANY)
self.Bind (wx.EVT_TIMER, self.OnTimer, self.timer)
self.timer.Start (1000, False)
self.printStatus ()

self.Bind (wx.EVT_PAINT, self.OnPaint)
self.Bind (wx.EVT_ERASE_BACKGROUND, self.OnEraseBackground)
self.Bind (wx.EVT_SIZE, self.OnSize)
self.Bind (wx.EVT_LEFT_DOWN, self.OnDown)
self.Bind (wx.EVT_LEFT_UP, self.OnUp)
self.Bind (wx.EVT_MOTION, self.OnMouseMotion)
if allowRotation:
self.Bind (wx.EVT_RIGHT_UP, self.OnRight)

def SaveStatus(self):
self.db.initiateDB ()
self.db.saveEverything (Board.boardId, self.count, self.seconds, \
self.pieces)

def printStatus (self):
mins = self.seconds%(60*60)/60
minStr = str(mins)
if (mins < 10):
minStr = '0' + minStr
secs = self.seconds%60
secStr = str(secs)
if (secs < 10):
secStr = '0' + secStr
timeStr = str(self.seconds/(60*60)) + ':' + minStr + ':' + secStr

if self.completed:
status = 'Completed: ' + timeStr
else:
status = str(self.count) + ': ' + timeStr
self.GetParent().statusbar.SetStatusText (status)

def OnTimer (self, e):
if not self.completed:
self.seconds += 1
self.printStatus ()

def OnEraseBackground (self, e):
""" To avoid flickering in windows"""
pass

def OnPaint (self, e):
dc = wx.BufferedPaintDC(self, self._Buffer, wx.BUFFER_VIRTUAL_AREA)

def draw (self, dc):
dc.Clear ()
for p in self.pieces:
p.drawPiece (dc)

def UpdateDrawing (self):
dc = wx.MemoryDC ()
dc.SelectObject (self._Buffer)
self.draw (dc)
del dc
self.Refresh ()
self.Update ()

def OnSize (self, e):
s = self.ClientSize
self._Buffer = wx.EmptyBitmap (*s)
self.UpdateDrawing ()


def OnDown (self, e):
mousePos = e.GetPosition()

# The iteration is reversed so the piece on top is choosen
indexes = range(len(self.pieces))
indexes.reverse()
for i in indexes:
if self.pieces[i].checkPointInPiece (mousePos):
# Put the selected at the end of the list so it's painted the last
p = self.pieces[i]
self.pieces.append(p)
del self.pieces[i]
self.dragged = len(self.pieces) -1
break

def OnUp (self, e):
if self.dragged != None:
mousePos = e.GetPosition()
for i in range(len(self.pieces)):
if (self.dragged != i):
if self.pieces[self.dragged].checkPieceMatch (\
self.pieces[i]):
del self.pieces[i]
#self.Refresh()
self.UpdateDrawing ()
self.count += 1
self.printStatus ()
self.SaveStatus ()
if len(self.pieces) == 1:
if (self.pieces[0].getOrientation() ==\
rectangle.Orientation.or0):
self.db.RemoveDBFile ()
self.completed = True
self.printStatus ()

break
self.dragged = None

def OnMouseMotion (self, e):
if self.dragged != None:
mousePos = e.GetPosition()
self.pieces[self.dragged].movePiece (mousePos)
#self.Refresh()
self.UpdateDrawing ()

def OnRight (self, e):
mousePos = e.GetPosition()
indexes = range(len(self.pieces))
indexes.reverse()
for i in indexes:
if self.pieces[i].checkPointInPiece (mousePos):
self.pieces[i].incrementOrientation()
newPos = self.pieces[i].calculateCenterPosition()
self.pieces[i].changeRelPosition (newPos)
self.pieces[i].checkPointInPiece (mousePos)
#self.Refresh()
self.UpdateDrawing ()
if len(self.pieces) == 1:
if (self.pieces[0].getOrientation() == \
rectangle.Orientation.or0):
self.completed = True
self.printStatus ()
break

class Puzzle (wx.Frame):
def __init__ (self, parent, id, title, pieces, boardSize, count,
seconds, allowRotation=True):
wx.Frame.__init__(self, parent, id, title, size=boardSize)

self.statusbar = self.CreateStatusBar()
self.board = Board(self, pieces, count, seconds, allowRotation)

self.Centre()
self.Show(True)