miércoles, 14 de diciembre de 2011

wxPython example. A jigsaw puzzle (3). MS Windows a double buffer

This is a continuation of the previous two entries. I finally got the chance of testing the puzzle on windows and it didn't show properly it flicked a lot. Then I learn that the Windows platform doesn't implement a double buffer so I have to do it.
The idea is painting to a bitmap and then copy it to the DC on the OnPaint event. This solution improved a lot the way application was shown it didn't removed he flicks completely.
After some research I found the solution by capturing the EVT_ERASE_BACKGROUND event and do nothing with it. This fixed all the issues. It also important to call the Update method after the Refresh.

The new board.py code is:


#!/usr/bin/python

import wx
import stateDB
import rectangle

class Board(wx.Panel):
boardId = 77
def __init__ (self, parent, pieces, count, seconds, allowRotation=True):
wx.Panel.__init__(self, parent, style = wx.NO_FULL_REPAINT_ON_RESIZE)

self.db = stateDB.StateDB()


self.pieces = pieces
self.dragged = None
self.count = count
self.completed = False

self.seconds = seconds
self.timer = wx.Timer (self, wx.ID_ANY)
self.Bind (wx.EVT_TIMER, self.OnTimer, self.timer)
self.timer.Start (1000, False)
self.printStatus ()

self.Bind (wx.EVT_PAINT, self.OnPaint)
self.Bind (wx.EVT_ERASE_BACKGROUND, self.OnEraseBackground)
self.Bind (wx.EVT_SIZE, self.OnSize)
self.Bind (wx.EVT_LEFT_DOWN, self.OnDown)
self.Bind (wx.EVT_LEFT_UP, self.OnUp)
self.Bind (wx.EVT_MOTION, self.OnMouseMotion)
if allowRotation:
self.Bind (wx.EVT_RIGHT_UP, self.OnRight)

def SaveStatus(self):
self.db.initiateDB ()
self.db.saveEverything (Board.boardId, self.count, self.seconds, \
self.pieces)

def printStatus (self):
mins = self.seconds%(60*60)/60
minStr = str(mins)
if (mins < 10):
minStr = '0' + minStr
secs = self.seconds%60
secStr = str(secs)
if (secs < 10):
secStr = '0' + secStr
timeStr = str(self.seconds/(60*60)) + ':' + minStr + ':' + secStr

if self.completed:
status = 'Completed: ' + timeStr
else:
status = str(self.count) + ': ' + timeStr
self.GetParent().statusbar.SetStatusText (status)

def OnTimer (self, e):
if not self.completed:
self.seconds += 1
self.printStatus ()

def OnEraseBackground (self, e):
""" To avoid flickering in windows"""
pass

def OnPaint (self, e):
dc = wx.BufferedPaintDC(self, self._Buffer, wx.BUFFER_VIRTUAL_AREA)

def draw (self, dc):
dc.Clear ()
for p in self.pieces:
p.drawPiece (dc)

def UpdateDrawing (self):
dc = wx.MemoryDC ()
dc.SelectObject (self._Buffer)
self.draw (dc)
del dc
self.Refresh ()
self.Update ()

def OnSize (self, e):
s = self.ClientSize
self._Buffer = wx.EmptyBitmap (*s)
self.UpdateDrawing ()


def OnDown (self, e):
mousePos = e.GetPosition()

# The iteration is reversed so the piece on top is choosen
indexes = range(len(self.pieces))
indexes.reverse()
for i in indexes:
if self.pieces[i].checkPointInPiece (mousePos):
# Put the selected at the end of the list so it's painted the last
p = self.pieces[i]
self.pieces.append(p)
del self.pieces[i]
self.dragged = len(self.pieces) -1
break

def OnUp (self, e):
if self.dragged != None:
mousePos = e.GetPosition()
for i in range(len(self.pieces)):
if (self.dragged != i):
if self.pieces[self.dragged].checkPieceMatch (\
self.pieces[i]):
del self.pieces[i]
#self.Refresh()
self.UpdateDrawing ()
self.count += 1
self.printStatus ()
self.SaveStatus ()
if len(self.pieces) == 1:
if (self.pieces[0].getOrientation() ==\
rectangle.Orientation.or0):
self.db.RemoveDBFile ()
self.completed = True
self.printStatus ()

break
self.dragged = None

def OnMouseMotion (self, e):
if self.dragged != None:
mousePos = e.GetPosition()
self.pieces[self.dragged].movePiece (mousePos)
#self.Refresh()
self.UpdateDrawing ()

def OnRight (self, e):
mousePos = e.GetPosition()
indexes = range(len(self.pieces))
indexes.reverse()
for i in indexes:
if self.pieces[i].checkPointInPiece (mousePos):
self.pieces[i].incrementOrientation()
newPos = self.pieces[i].calculateCenterPosition()
self.pieces[i].changeRelPosition (newPos)
self.pieces[i].checkPointInPiece (mousePos)
#self.Refresh()
self.UpdateDrawing ()
if len(self.pieces) == 1:
if (self.pieces[0].getOrientation() == \
rectangle.Orientation.or0):
self.completed = True
self.printStatus ()
break

class Puzzle (wx.Frame):
def __init__ (self, parent, id, title, pieces, boardSize, count,
seconds, allowRotation=True):
wx.Frame.__init__(self, parent, id, title, size=boardSize)

self.statusbar = self.CreateStatusBar()
self.board = Board(self, pieces, count, seconds, allowRotation)

self.Centre()
self.Show(True)




martes, 6 de diciembre de 2011

wxPython example. A jigsaw puzzle (2)

Continuing with the previous post I created another puzzle, but this time the pieces has the shape of the mainland Spain provinces. I think is a fun way to learn geography and can be used for any map.



The code is basically what described in the previous post. The only difference is the way the pieces masks are created. I did it manually with gimp. For example:



The main file of the application, which I call prov.py, has the following code:



#!/usr/bin/python

import board
import stateDB
import rectangle
import piece
import wx
import random

# Id, image, mask, size, top-left positions, relations
provinceList = [
[1, 'coruna.png', 'corunaMask.png', (106, 96), (0,0), [2, 3] ],
[2, 'lugo.png', 'lugoMask.png', (72, 119), (82,0), [1, 3, 4, 5, 37 ]],
[3, 'pontevedra.png', 'pontevedraMask.png', (70, 79), (17, 73), [1,2,4]],
[4, 'orense.png', 'orenseMask.png', (97, 70), (60,94), [2, 3, 25, 37]],
[5, 'asturias.png', 'asturiasMask.png', (159, 66), (134,11), [2,6, 37]],
[6, 'cantabria.png', 'cantabriaMask.png', (104,64), (274, 25), [5,7, 37,
40, 41]],
[7, 'vizcaya.png', 'vizcayaMask.png', (68, 35), (356,32), [6, 8, 9, 40]],
[8, 'guipuzcoa.png', 'guipuzcoaMask.png', (50, 42), (412,34), [7, 9, 10]],
[9, 'alava.png','alavaMask.png', (62, 60), (372, 52), [8, 7, 10, 39, 40 ]],
[10, 'navarra.png', 'navarraMask.png', (102, 116), (423, 38), [8,9, 11,
12, 39]],
[11, 'huesca.png', 'huescaMask.png', (112, 126), (511,69), [10, 12, 14 ]],
[12, 'zaragoza.png', 'zaragozaMask.png', (160, 147), (441, 85), [11, 10,
13, 14, 17, 18, 38]],
[13, 'teruel.png', 'teruelMask.png', (132, 122), (464, 197), [12, 17, 18,
33, 34, 35]],
[14, 'lleida.png', 'lleidaMask.png', (93,133), (596, 69), [ 11, 12, 15,
16, 17]],
[15, 'girona.png', 'gironaMask.png', (94, 72), (683,89), [ 14, 16 ]],
[16, 'barcelona.png', 'barcelonaMask.png', (85, 96), (662, 108), [14, 15,
17]],
[17, 'tarra.png', 'tarraMask.png', (93,91), (589,169), [16, 14, 13, 12,
34]],
[18, 'guada.png', 'guadaMask.png', (127, 95), (354, 202), [ 13, 12, 19,
35, 38, 43]],
[19, 'madrid.png', 'madridMask.png', (101,100), (287, 219), [18, 20, 35,
43, 44]],
[20, 'avila.png', 'avilaMask.png', (103,91), (212,216), [19, 21, 24, 42,
43, 44]],
[21, 'caceres.png', 'caceresMask.png', (165, 119), (96, 270), [20, 22,
24, 44]],
[22, 'badajoz.png', 'badajozMask.png', (177,125), (106, 352), [21, 23, 26,
44, 45, 47]],
[23, 'huelva.png', 'huelvaMask.png', (91,112), (87, 456), [22, 26 ]],
[24, 'salamanca.png', 'salamancaMask.png',(113, 94), (141,198), [20, 21,
25, 42 ]],
[25, 'zamora.png', 'zamoraMask.png',(111, 84), (136,127), [24, 4, 37, 42 ]],
[26, 'sevilla.png', 'sevillaMask.png',(128, 116), (153, 456), [22, 23,
27, 28, 47 ]],
[27, 'cadiz.png', 'cadizMask.png', (96,84), (154, 555), [26, 28]],
[28, 'malaga.png', 'malagaMask.png', (122,84), (221, 532), [27, 26, 29,
47]],
[29, 'granada.png', 'granadaMask.png', (138,117), (303, 466), [28,30, 31,
36, 46, 47]],
[30, 'almeria.png', 'almeriaMask.png', (105,103), (381, 480), [29, 31]],
[31, 'murcia.png', 'murciaMask.png', (108, 114), (434, 410), [29, 32, 36]],
[32, 'alicante.png', 'alicanteMask.png', (81,92), (520,391), [31, 33, 36]],
[33, 'valencia.png', 'valenciaMask.png', (95,121), (485,292), [32, 13, 34,
35, 36]],
[34, 'castellon.png', 'castellonMask.png', (86,89), (526,242), [33, 13,
17]],
[35, 'cuenca.png', 'cuencaMask.png', (132, 118), (376, 256), [18, 19, 13,
33, 36, 44, 45]],
[36, 'albacete.png', 'albaceteMask.png', (129,119), (397, 354), [35, 29,
36, 32, 33, 31, 45, 46]],
[37, 'leon.png', 'leonMask.png', (143, 102), (138, 50), [2, 4, 25, 5, 6,
41, 42]],
[38, 'soria.png', 'soriaMask.png', (107, 89), (358,136), [12, 18, 39, 40,
43 ]],
[39, 'rioja.png', 'riojaMask.png', (83,55), (383, 96), [38, 9, 10, 40]],
[40, 'burgos.png', 'burgosMask.png', (110,141), (306,52), [39, 38, 6, 7,
9, 41, 42, 43]],
[41, 'palencia.png', 'palenciaMask.png', (70,100), (262,65), [37, 6, 40,
42 ]],
[42, 'valla.png', 'vallaMask.png', (96,98), (235, 125), [37, 40, 41, 20,
25, 24, 43]],
[43, 'segovia.png', 'segoviaMask.png', (92,78), (280, 185), [42, 40, 38,
18, 19, 20]],
[44, 'toledo.png', 'toledoMask.png', (160, 83), (233, 287), [35,21, 22,
19, 20, 45]],
[45, 'creal.png', 'crealMask.png', (158, 103), (255, 344), [44, 36, 35,
22, 46, 47 ]],
[46, 'Jaen.png', 'JaenMask.png', (122, 98), (306, 430), [45, 36, 29, 47]],
[47, 'cordoba.png', 'cordobaMask.png', (109,129), (218,412), [46, 45, 26,
28, 29, 22]]
]


def createPieceList (provinceList, boardSize):
pieces = []

for entry in provinceList:
im = wx.Image (entry[1])
mask = wx.Image (entry[2])
im.SetMaskFromImage (mask, 255, 255, 255)

rels = []
# Calculate the relations
for related in entry[5]:
index = related -1
difX = provinceList [index][4][0] - entry[4][0]
difY = provinceList [index][4][1] - entry[4][1]
rel = [related, (difX, difY), 0, 0]
rels.append (rel)

rec = rectangle.Rectangle (entry[0], (im, mask), entry[3],
rels, 0, 0)
p = piece.Piece (entry[0])
p.addRectangle (rec, (0, 0))
pieces.append (p)

for i in range (len(pieces)):
pieces[i].setPosition ( (random.randint (10, boardSize[0] - 300),\
random.randint(10, boardSize[1] - 300)))
#pieces[i].setOrientation (rectangle.getRandomOrientation())

# Re-order the list so we don't always get the same provinces on top
random.shuffle (pieces)

return pieces

boardSize = (1000, 800)
app = wx.App ()
db = stateDB.StateDB()
count = -1
seconds = 0

if db.isThereADBFile ():
count, seconds, pieces = db.readBoard (board.Board.boardId)

if count <=0:
pieces= createPieceList (provinceList, boardSize)
count = 0

board.Puzzle(None, -1, 'Spain mainland provinces', pieces, boardSize, count,
seconds, False)
app.MainLoop()


lunes, 5 de diciembre de 2011

wxPython example. A jigsaw puzzle

I've been teaching myself a bit about python and sqlite. To do so I have created a puzzle. The thing is pretty basic so far but it's playable. You can use any picture and the pieces are created automatically. The pieces are dragged by pressing the mouse left button and dropped when released. They are rotated with the right button.

I've tested it in Linux and Mac and it should work in Windows as well. You may need to install python and wxWidgets. The Mac version wxPython is only supported for 32bits so you must define the foillowing variable before executing:

export VERSIONER_PYTHON_PREFER_32_BIT=yes





The basic element of the puzzle is a rectangle. A rectangle has an image, a mask and they store the relative position of other rectangles. They can be moved around and rotated. The following code must be included in a file called rectangle.py.


import math
import wx
import random

class Orientation (object):
or0 = 0
or90 = 1
or180 = 2
or270 = 3

def rotateImage (orientation, im):
imAux = im
for i in range (orientation):
imAux = imAux.Rotate90 (True)

return imAux

def rotateRelPosition (orientation, pos, compX, compY):
posAux = pos

if orientation == Orientation.or90:
posAux =(-posAux[1] - compY, posAux[0])
elif orientation == Orientation.or180:
posAux =(-posAux[0] - compX, -posAux[1] - compY)
elif orientation == Orientation.or270:
posAux =(posAux[1], - posAux[0] - compX)

#for i in range (orientation):
# posAux = (-posAux[1], posAux[0])
return posAux

def rotateSize (orientation, size):
sizeAux = size
if (orientation == Orientation.or90 or orientation == Orientation.or270):
sizeAux = (sizeAux[1], sizeAux[0])
return sizeAux


def nextOrientation (orientation):
if orientation == Orientation.or270:
orientation = Orientation.or0
else:
orientation +=1
return orientation

def getRandomOrientation ():
return random.randint (Orientation.or0, Orientation.or270)

class Rectangle (object):
def __init__ (self, id, im, size, relations, compX, compY):
self.id = id
self.image = im[0]
self.mask = im[1]
self.size = size
self.relations = relations
self.position = (0,0)
self.orientation = Orientation.or0
self.compX = compX
self.compY = compY

def __str__ (self):
rectStr = "Rectangle " + str(self.id) + " size: " + str(self.size) + \
", pos: " + str(self.position) + ", comp: " + \
str((self.compX, self.compY))+ ", or: " + \
str(self.orientation) + ", rel: " + str(self.relations)
return rectStr

def setPosition (self, pos):
self.position = pos

def setOrientation (self, ort):
self.orientation = ort

def getCompX (self):
self.compX

def getCompY (self):
self.compY

def incrementOrientation (self):
self.orientation = nextOrientation (self.orientation)

def removeRelations(self, id):
index = 0
for index in range(len(self.relations)):
if self.relations[index][0] == id:
del self.relations[index]
break

def getId (self):
return self.id

def getImage (self):
return self.image

def getMask (self):
return self.mask

def getSize (self):
return self.size

def getPosition (self):
return self.position

def getOrientation (self):
return self.orientation

def getAbsolutePosition (self, absPosition):
relPos = rotateRelPosition (self.orientation, self.position,
self.compX, self.compY)
return (( absPosition[0] + relPos[0],\
absPosition[1] + relPos[1]))

def drawRectangle (self, absPosition, dc):
absPos = self.getAbsolutePosition (absPosition)
im = rotateImage (self.orientation, self.image)
dc.DrawBitmap ( wx.BitmapFromImage(im), absPos[0],\
absPos[1], True)

def changeRelPosition (self, shift):
self.position = ( self.position[0] - shift[0], \
self.position[1] - shift[1])

def checkPointInRectangle (self, absPosition, point):
""" Returns the relative position of the point. """
retVal = False

relPos = (0, 0)
absPos = self.getAbsolutePosition (absPosition)

rotatedSize = rotateSize (self.orientation, self.size)

if (point[0] >= absPos[0]) and \
(point[0] < absPos[0] + rotatedSize[0]) and \
(point[1] >= absPos[1]) and \
(point[1] < absPos[1] + rotatedSize[1]):
retVal = True
relPos = (point[0] - absPos[0] + self.position[0],
point[1] - absPos[1] + self.position[1])

return (retVal, relPos)

def checkRectagleMatch (self, absPosition, id, pos):
""" Checks if another rectangle matches.
id, the id of the other rectangle
pos, the absolute position of the other rectangle.
The function returns the relative position of the other rectangle
with respect to this piece."""
retVal = False
relPos = (0, 0)
absPos = self.getAbsolutePosition (absPosition)
for rel in self.relations:
if rel[0] == id:
rotatedRel = rotateRelPosition (self.orientation, rel[1],\
rel[2], rel[3])
absPos = (absPos[0] + rotatedRel[0], absPos[1] + rotatedRel[1])

distX = pos[0] - absPos[0]
distY = pos[1] - absPos[1]

if (math.sqrt((distX * distX) + (distY * distY)) < 4.0):
retVal = True
relPos= (self.position[0] + rel[1][0],\
self.position[1] + rel[1][1])

break
return retVal, relPos




The next element of the puzzle are the pieces. A piece contains one or more rectangles. The number of pieces gets reduces as the puzzle is solved. The code should be put in a file called piece.py.


import rectangle
import wx

class Piece(object):
def __init__ (self, id):
self.id = id
self.pos = (0,0)
self.mousePos = (0, 0)
self.rectangles = []
self.orientation = rectangle.Orientation.or0

def __str__ (self):
pieceStr = 'Piece ' + str(self.id) + ' pos: ' + str(self.pos) + \
' mousePos: ' + str(self.mousePos) + ' or: '+\
str(self.orientation) + ' , rect:\n'
for rect in self.rectangles:
pieceStr += str(rect) + '\n'
return pieceStr

def getId (self):
return self.id

def getPosition (self):
return (self.pos)

def setPosition (self, pos):
self.pos = pos

def setOrientation (self, ort):
self.orientation = ort
for index in range(len(self.rectangles)):
self.rectangles[index].setOrientation (ort)

def addRectangle (self, rect, relPos):
for index in range(len(self.rectangles)):
self.rectangles[index].removeRelations (rect.getId())

for index in range(len(self.rectangles)):
rect.removeRelations(self.rectangles[index].getId())

rect.setPosition (relPos)
self.rectangles.append (rect)

def getNoOfRectangles (self):
return len (self.rectangles)

def getRectangle (self, i):
return self.rectangles[i]

def getRectangleId (self, i):
return self.rectangles[i].getId ()

def getRectangleAbsPos (self, i):
return self.rectangles[i].getAbsolutePosition (self.pos)

def changeRelPosition (self, shift):
for rect in self.rectangles:
rect.changeRelPosition (shift)

def calculateCenterPosition (self):
maxLeft = 0
maxRight = 0
maxTop = 0
maxBottom = 0
for rect in self.rectangles:
x = rect.getPosition()[0]
if maxLeft > x:
maxLeft = x
elif maxRight < x:
maxRight = x
y = rect.getPosition()[1]
if maxTop > y:
maxTop = y
elif maxBottom < y:
maxBottom = y

c = ((maxLeft + maxRight) / 2, (maxTop + maxBottom)/2)
return c



def drawPiece (self, dc):
for rect in self.rectangles:
rect.drawRectangle (self.pos, dc)

def checkPointInPiece (self, point):
retVal = False
for rect in self.rectangles:
ret, relPos = rect.checkPointInRectangle (self.pos, point)
if ret:
retVal = True
self.mousePos = relPos
break

return retVal

def checkPieceMatch (self, otherPiece):
retVal = False

if (self.orientation == otherPiece.getOrientation()):
for i in range(otherPiece.getNoOfRectangles()):
otherRect = otherPiece.getRectangle (i)
for rect in self.rectangles:
ret, relPos = rect.checkRectagleMatch (self.pos,
otherRect.getId(),
otherPiece.getRectangleAbsPos (i))
if ret:
retVal = True
# Calculate the relative position of the other piece
posOtherPiece = (relPos[0] - otherRect.getPosition()[0],
relPos[1] - otherRect.getPosition()[1])

# Include the rectangles in this piece
for j in range(otherPiece.getNoOfRectangles()):
otherRect = otherPiece.getRectangle(j)
self.addRectangle (otherRect,
(posOtherPiece[0] + otherRect.getPosition()[0],
posOtherPiece[1] + otherRect.getPosition()[1]))


break

return retVal

def movePiece (self, newPos):
self.pos = (newPos[0] - self.mousePos[0],
newPos[1] - self.mousePos[1])

def incrementOrientation (self):
self.orientation = rectangle.nextOrientation (self.orientation)
for i in range(len(self.rectangles)):
self.rectangles[i].setOrientation(self.orientation)

def getOrientation (self):
return self.orientation


The board.py file contains the Board class that creates the gui panel and manages the events.


import wx
import stateDB
import rectangle

class Board(wx.Panel):
boardId = 77
def __init__ (self, parent, pieces, count, seconds, allowRotation=True):
wx.Panel.__init__(self, parent)

self.db = stateDB.StateDB()

self.pieces = pieces
self.dragged = None
self.count = count
self.completed = False

self.seconds = seconds
self.timer = wx.Timer (self, wx.ID_ANY)
self.Bind (wx.EVT_TIMER, self.OnTimer, self.timer)
self.timer.Start (1000, False)
self.printStatus ()

self.Bind (wx.EVT_PAINT, self.OnPaint)
self.Bind (wx.EVT_LEFT_DOWN, self.OnDown)
self.Bind (wx.EVT_LEFT_UP, self.OnUp)
self.Bind (wx.EVT_MOTION, self.OnMouseMotion)
if allowRotation:
self.Bind (wx.EVT_RIGHT_UP, self.OnRight)

def SaveStatus(self):
self.db.initiateDB ()
self.db.saveEverything (Board.boardId, self.count, self.seconds, \
self.pieces)

def printStatus (self):
mins = self.seconds%(60*60)/60
minStr = str(mins)
if (mins < 10):
minStr = '0' + minStr
secs = self.seconds%60
secStr = str(secs)
if (secs < 10):
secStr = '0' + secStr
timeStr = str(self.seconds/(60*60)) + ':' + minStr + ':' + secStr

if self.completed:
status = 'Completed: ' + timeStr
else:
status = str(self.count) + ': ' + timeStr
self.GetParent().statusbar.SetStatusText (status)

def OnTimer (self, e):
if not self.completed:
self.seconds += 1
self.printStatus ()

def OnPaint (self, e):
dc = wx.PaintDC(self)

for p in self.pieces:
p.drawPiece (dc)

def OnDown (self, e):
mousePos = e.GetPosition()

# The iteration is reversed so the piece on top is choosen
indexes = range(len(self.pieces))
indexes.reverse()
for i in indexes:
if self.pieces[i].checkPointInPiece (mousePos):
# Put the selected at the end of the list so it's painted the last
p = self.pieces[i]
self.pieces.append(p)
del self.pieces[i]
self.dragged = len(self.pieces) -1
break

def OnUp (self, e):
if self.dragged != None:
mousePos = e.GetPosition()
for i in range(len(self.pieces)):
if (self.dragged != i):
if self.pieces[self.dragged].checkPieceMatch (\
self.pieces[i]):
del self.pieces[i]
self.Refresh()
self.count += 1
self.printStatus ()
self.SaveStatus ()
if len(self.pieces) == 1:
if (self.pieces[0].getOrientation() ==\
rectangle.Orientation.or0):
self.db.RemoveDBFile ()
self.completed = True
self.printStatus ()

break
self.dragged = None

def OnMouseMotion (self, e):
if self.dragged != None:
mousePos = e.GetPosition()
self.pieces[self.dragged].movePiece (mousePos)
self.Refresh()

def OnRight (self, e):
mousePos = e.GetPosition()
indexes = range(len(self.pieces))
indexes.reverse()
for i in indexes:
if self.pieces[i].checkPointInPiece (mousePos):
self.pieces[i].incrementOrientation()
newPos = self.pieces[i].calculateCenterPosition()
self.pieces[i].changeRelPosition (newPos)
self.pieces[i].checkPointInPiece (mousePos)
self.Refresh()
if len(self.pieces) == 1:
if (self.pieces[0].getOrientation() == \
rectangle.Orientation.or0):
self.completed = True
self.printStatus ()
break

class Puzzle (wx.Frame):
def __init__ (self, parent, id, title, pieces, boardSize, count,
seconds, allowRotation=True):
wx.Frame.__init__(self, parent, id, title, size=boardSize)

self.statusbar = self.CreateStatusBar()
self.board = Board(self, pieces, count, seconds, allowRotation)

self.Centre()
self.Show(True)


The state of the puzzle is stored in a sqlite database. The access to database is implemented in a file called stateDB.py.



import sqlite3
import os
import piece
import rectangle
import wx


class StateDB (object):
# Names of the tables
boardTable = 'board'
pieceTable = 'piece'
rectangleTable = 'rectangle'
relationTable = 'relation'

# The name of the DB file
DBFileName = '.puzzle.db'

# field names
ekey= 'key'
BTcount = 'count'
BTseconds = 'seconds'

PTboard = 'board'
PTposx = 'posx'
PTposy = 'posy'
PTorientation = 'orientation'

RTpiece = 'piece'
RTsizex = 'sizex'
RTsizey = 'sizey'
RTposx = 'posx'
RTposy = 'posy'
RTcompx = 'compx'
RTcompy = 'compy'
RTimage = 'image'
RTmask = 'mask'

RTsource = 'source'
RTdest = 'dest'
RTrelx = 'relx'
RTrely = 'rely'
RelTCompX = 'compX'
RelTCompY = 'compY'

def __init__ (self):
self.con = None
self.cur = None

def OpenDB (self):
self.con = sqlite3.connect (StateDB.DBFileName)
self.cur = self.con.cursor ()

def CloseDB (self):
self.cur.close()
self.con.commit()
self.con.close()
self.cur = None
self.con = None

def RemoveDBFile (self):
if os.path.exists (StateDB.DBFileName):
os.remove (StateDB.DBFileName)

def isThereADBFile (self):
return os.path.exists (StateDB.DBFileName)

def CreateTables (self):
createBT = 'CREATE TABLE ' + StateDB.boardTable + ' (' + \
StateDB.ekey + ' INTEGER PRIMARY KEY, ' + \
StateDB.BTcount + ' INTEGER,' +\
StateDB.BTseconds + ' INTEGER)'
self.cur.execute (createBT)

createPT = 'CREATE TABLE ' + StateDB.pieceTable + ' (' + \
StateDB.ekey + ' INTEGER PRIMARY KEY, ' + \
StateDB.PTboard + ' INTEGER, ' + \
StateDB.PTposx + ' INTEGER, ' + \
StateDB.PTposy + ' INTEGER, ' + \
StateDB.PTorientation + ' INTEGER, ' + \
'FOREIGN KEY (' + StateDB.PTboard + ') REFERENCES ' + \
StateDB.boardTable + '(' + StateDB.ekey + '))'
self.cur.execute (createPT)

createRT = 'CREATE TABLE ' + StateDB.rectangleTable + '(' + \
StateDB.ekey + ' INTEGER PRIMARY KEY, ' + \
StateDB.RTpiece + ' INTEGER, ' + \
StateDB.RTsizex + ' INTEGER, ' + \
StateDB.RTsizey + ' INTEGER, ' + \
StateDB.RTposx + ' INTEGER, ' + \
StateDB.RTposy + ' INTEGER, ' + \
StateDB.RTcompx + ' INTEGER, ' + \
StateDB.RTcompy + ' INTEGER, ' + \
StateDB.RTimage + ' BLOB, ' + \
StateDB.RTmask + ' BLOB, ' + \
'FOREIGN KEY (' + StateDB.RTpiece + ') REFERENCES ' + \
StateDB.pieceTable + '(' + StateDB.ekey + '))'
self.cur.execute (createRT)

createRT = 'CREATE TABLE ' + StateDB.relationTable + '(' + \
StateDB.ekey + ' INTEGER PRIMARY KEY, ' + \
StateDB.RTsource + ' INTEGER, ' + \
StateDB.RTdest + ' INTEGER, ' + \
StateDB.RTrelx + ' INTEGER, ' + \
StateDB.RTrely + ' INTEGER, ' + \
StateDB.RelTCompX + ' INTEGER, ' + \
StateDB.RelTCompY + ' INTEGER, ' + \
'FOREIGN KEY (' + StateDB.RTsource + ') REFERENCES ' + \
StateDB.rectangleTable + '(' + StateDB.ekey + '), ' + \
'FOREIGN KEY (' + StateDB.RTdest + ') REFERENCES ' + \
StateDB.rectangleTable + '(' + StateDB.ekey + '))'
self.cur.execute (createRT)

def initiateDB (self):
self.RemoveDBFile ()
self.OpenDB ()
self.CreateTables ()
self.CloseDB ()

def saveBoard (self, id, count, seconds):
query = 'INSERT INTO ' + StateDB.boardTable + ' VALUES (?,?,?)'
self.cur.execute (query, (id, count, seconds))

def saveRelation (self, rectId, rel):
query = ' INSERT INTO ' + StateDB.relationTable + '(' + \
StateDB.RTsource + ',' + StateDB.RTdest + ',' + \
StateDB.RTrelx + ',' + StateDB.RTrely + ', ' + \
StateDB.RelTCompX + ', ' + StateDB.RelTCompY + ') ' + \
' VALUES (?,?,?,?,?,?) '
entry = (rectId, rel[0], rel[1][0], rel[1][1],rel[2],rel[3])
self.cur.execute (query, entry)


def saveRectangle (self, pieceId, rect):
query = ' INSERT INTO ' + StateDB.rectangleTable + \
' VALUES (?,?,?,?,?,?,?,?,?,?) '
size = rect.getSize ()
pos = rect.getPosition()
imData = rect.getImage().GetData()
maskData = rect.getMask().GetData()
entry = (rect.getId(), pieceId, size[0], size[1], pos[0], pos[1],
rect.compX, rect.compY, sqlite3.Binary(imData),
sqlite3.Binary(maskData))
self.cur.execute (query, entry)
for rel in rect.relations:
self.saveRelation (rect.getId(), rel)

def savePiece (self, boardId, piece):
query = ' INSERT INTO ' + StateDB.pieceTable + ' VALUES (?,?,?,?,?) '
pos = piece.getPosition()
entry = (piece.getId(), boardId,pos[0], pos[1], piece.getOrientation())
self.cur.execute (query, entry)
for i in range(piece.getNoOfRectangles()):
self.saveRectangle (piece.getId(), piece.getRectangle(i))

def saveEverything (self, id, count, seconds, pieces):
self.OpenDB ()
self.saveBoard (id, count, seconds)
for piece in pieces:
self.savePiece (id, piece)
self.CloseDB ()

def readRelations (self, rectId):
relations = []
query = 'SELECT * FROM ' + StateDB.relationTable + ' WHERE ' + \
StateDB.RTsource + '=' + str(rectId)
self.cur.execute(query)
relList = self.cur.fetchall ()
for entry in relList:
r = [entry[2], (entry[3], entry[4]), entry[5], entry[6]]
relations.append (r)
return relations

def readRectangles (self, pieceId):
rectangles = []
query = 'SELECT * FROM ' + StateDB.rectangleTable + ' WHERE ' + \
StateDB.RTpiece + '=' + str(pieceId)
self.cur.execute(query)
rectList = self.cur.fetchall ()
for entry in rectList:
im = wx.ImageFromData (entry[2], entry[3], entry[8])
mask = wx.ImageFromData (entry[2], entry[3], entry[9])
im.SetMaskFromImage (mask, 255, 255, 255)
rels = self.readRelations (entry[0])
r = rectangle.Rectangle (entry[0], (im, mask),\
(entry[2], entry[3]), rels, entry[6], entry[7])
pos = (entry[4], entry[5])
rectangles.append((r, pos))
return rectangles

def readPieces (self, boardId):
pieces = []
query = 'SELECT * FROM ' + StateDB.pieceTable + ' WHERE ' + \
StateDB.PTboard + '=' + str(boardId)
self.cur.execute (query)
pieceList = self.cur.fetchall ()
for entry in pieceList:
p = piece.Piece (entry[0])
p.setPosition ((entry[2], entry[3]))
rectangles = self.readRectangles (entry[0])
for r in rectangles:
p.addRectangle (r[0], r[1])
p.setOrientation (entry[4])
pieces.append (p)
return pieces

def readBoard (self, id):
self.OpenDB()
count = -1
pieces = []

query = 'SELECT ' + StateDB.BTcount + ', ' + StateDB.BTseconds +\
' FROM ' + StateDB.boardTable +\
' WHERE ' + StateDB.ekey + '=' + str(id)
self.cur.execute (query)
countList = self.cur.fetchall ()
if countList != None:
count = countList[0][0]
seconds = countList[0][1]
pieces = self.readPieces(id)
self.CloseDB()

return count, seconds, pieces


Finally, the main file, puzzle.py, takes an image creates all the pieces automatically and then starts the GUI.


#!/usr/bin/python

import wx
import random
import piece
import rectangle
import stateDB
import board


def createMask (sizeX, sizeY, overlap, dims, pos):
theBuffer = bytearray()

w = 255
b = 0

leftLimit = (sizeX/2) - overlap/2
rightLimit = (sizeX/2) +overlap/2
topLimit = (sizeY/2) - overlap/2
bottomLimit = (sizeY/2) + overlap/2


even = (((pos[0] + pos[1])% 2) == 0)

circleShift = 2
radiusSquare = ((overlap+circleShift)*(overlap+circleShift))/4

for j in range(sizeY ):
for i in range (sizeX ):
setWhite = False

setWhite = False
if (pos[0] < dims[0] -1):
if even:
if (i >= sizeX - overlap):
newJ = topLimit + (overlap /2) - j
newI = sizeX - (overlap/2) - circleShift -i
if (newI * newI + newJ * newJ) > radiusSquare:
setWhite = True
else:
if (i >= sizeX - overlap):
setWhite = True
if (i < sizeX - overlap) and (i >= sizeX - overlap*2):
newJ = topLimit + (overlap /2) - j
newI = sizeX - (3*overlap/2) + circleShift -i
if (newI * newI + newJ * newJ) <= radiusSquare:
setWhite = True

if (pos[0] > 0):
if not even:
if (i < overlap):
setWhite = True
if (i >= overlap) and (i < overlap *2):
newI = (3*overlap/2) - circleShift -i
newJ = topLimit + (overlap/2) - j
if (newI * newI + newJ * newJ) <= radiusSquare:
setWhite = True
else:
if (i < overlap):
newI = (overlap/2) + circleShift - i
newJ = topLimit + (overlap/2) -j
if (newI * newI + newJ * newJ) > radiusSquare:
setWhite = True

if (pos[1] < dims[1] - 1):
if not even:
if (j >= sizeY - overlap):
newI = leftLimit + (overlap/2) -i
newJ = sizeY - (overlap/2) - circleShift - j
if (newI * newI + newJ * newJ) > radiusSquare:
setWhite = True
else:
if (j >= sizeY - overlap):
setWhite = True
if (j < sizeY - overlap) and (j >= sizeY - overlap*2):
newI = leftLimit + (overlap/2) -i
newJ = sizeY - (3*overlap/2) + circleShift - j
if (newI * newI + newJ * newJ) <= radiusSquare:
setWhite = True

if (pos[1] > 0):
if not even:
if (j < overlap):
newI = leftLimit + (overlap/2) -i
newJ = (overlap/2) +circleShift - j
if (newI * newI + newJ * newJ) > radiusSquare:
setWhite = True
else:
if (j < overlap):
setWhite = True
if (j >= overlap) and (j < overlap * 2):
newI = leftLimit + (overlap/2) -i
newJ = (3*overlap/2) - circleShift - j
if (newI * newI + newJ * newJ) <= radiusSquare:
setWhite = True
if setWhite:
color = w
else:
color = b
theBuffer.append (color)
theBuffer.append (color)
theBuffer.append (color)

return wx.ImageFromData (sizeX , sizeY, theBuffer)

def createPieceList (imFile, boardSize):
sizeX = 50
sizeY = 50
overlap = 12
im = wx.Image (imFile)
imW = im.GetWidth()
imH = im.GetHeight()
pieces = []

noOfRows = imH/sizeY
noOfCols = imW/sizeX


id = 0
for j in range (noOfRows):
for i in range (noOfCols):
# SubImage for each rectangle
iPos = 0
if i > 0:
iPos = i * sizeX - overlap
jPos = 0
if j > 0:
jPos = j * sizeY - overlap

curSizeX = sizeX + overlap
if i > 0 and i < noOfCols -1:
curSizeX += overlap
curSizeY = sizeY + overlap
if j > 0 and j < noOfRows -1:
curSizeY += overlap


r = wx.Rect (iPos, jPos, curSizeX, curSizeY)
imAux = im.GetSubImage (r)

mask = createMask (curSizeX, curSizeY, overlap, \
(noOfCols, noOfRows), (i, j))
imAux.SetMaskFromImage (mask, 255, 255, 255)


rels = []
if (i > 0):
if (i==1):
rels.append ([id-1,(-(sizeX-overlap), 0), -overlap, 0])
elif (i == (noOfCols - 1)):
rels.append ([id-1,(-(sizeX), 0), overlap, 0])
else:
rels.append ([id-1,(-(sizeX), 0), 0, 0])
if (i < (noOfCols -1)):
if (i==0):
rels.append ([id+1, (sizeX-overlap, 0), overlap, 0])
elif (i == (noOfCols -2)):
rels.append ([id+1, (sizeX, 0), -overlap, 0])
else:
rels.append ([id+1, (sizeX, 0), 0, 0])

if (j > 0):
if (j==1):
rels.append([id-noOfCols, (0, -(sizeY-overlap)), 0,
-overlap])
elif (j == (noOfRows -1)):
rels.append([id-noOfCols, (0, -(sizeY)), 0, overlap])
else:
rels.append([id-noOfCols, (0, -(sizeY)), 0, 0])
if (j < (noOfRows - 1)):
if (j==0):
rels.append([id+noOfCols, (0, sizeY-overlap), 0, overlap])
elif (j == (noOfRows -2)):
rels.append([id+noOfCols, (0, sizeY), 0, -overlap])
else:
rels.append([id+noOfCols, (0, sizeY), 0, 0])

compX, compY = calculateCompensantion (i, j, noOfRows, noOfCols,
overlap)


rec = rectangle.Rectangle (id, (imAux, mask),
(curSizeX, curSizeY), rels, compX, compY)
p = piece.Piece (id)
p.addRectangle (rec, (0,0))
pieces.append (p)
id +=1

for i in range (len(pieces)):
pieces[i].setPosition ( (random.randint (10, boardSize[0] -300),\
random.randint(10, boardSize[1] -300)))
pieces[i].setOrientation (rectangle.getRandomOrientation())

return pieces

def calculateCompensantion (i, j, noOfRows, noOfCols, overlap):
compX = 0
compY = 0
if i==0:
compX = -overlap
elif i== (noOfCols -1):
compX = -overlap

if j==0:
compY = -overlap
elif j == (noOfRows -1):
compY = -overlap

return compX, compY

boardSize = (1000, 800)
app = wx.App ()
db = stateDB.StateDB()
count = -1
seconds = 0


if db.isThereADBFile ():
count, seconds, pieces = db.readBoard (board.Board.boardId)

if count <=0:
pieces= createPieceList ('im2.png', boardSize)
count = 0


board.Puzzle(None, -1, 'Puzzle', pieces, boardSize, count, seconds)
app.MainLoop()

domingo, 5 de junio de 2011

Monitoring processes with Python

We've been working on some scripts to monitor the behavior of a process. The idea is run a process for days and register its memory and CPU usage. We also want to run in both Linux and Windows.

In the Linux environment the top command is used to find out CPU usage and the proc filesystem to obtain the memory usage. The ps command is used to obtain the list of processes.

The win32 extensions for python library is being used for Windows. I've found a bit difficult to work with this library and that's is the main reason I'm writing this post.

Finding out the process ID


The first thing is to find out process to monitor. We are assuming the process to monitor is the lastest descendent of the current process. We have created a function to list all the processes along with their parent proces and then another to obtain the last descendent.

def getListOfProcessesLinux (logger):
processList = []
cmd = 'ps -eo pid,ppid'
output = executeCommand (cmd, logger, True)
lines = output.split ('\n')
skipFirstLine = True
for line in lines:
if skipFirstLine:
skipFirstLine = False
else:
tempList = line.split ();
if (len (tempList) >= 2):
pid = int (tempList[0].strip())
ppid = int (tempList[1].strip())
processList.append ([pid, ppid]);
return processList

def getListOfProcessesWindows (logger):
processList = []
object = win32pdhutil.find_pdh_counter_localized_name("Process")
object2 = win32pdhutil.find_pdh_counter_localized_name('ID Process')
object3 = win32pdhutil.find_pdh_counter_localized_name('Creating Process ID')

# Get the list of processes
try:
junk, instances = win32pdh.EnumObjectItems(None,None, object, win32pdh.PERF_DETAIL_WIZARD)
except Exception, e:
print ('exception' + str(e))

for instance in instances:
hq = win32pdh.OpenQuery() # initializes the query handle
path = win32pdh.MakeCounterPath( (None,object,instance, None, 0,object2) )
counter_handle=win32pdh.AddCounter(hq, path)
win32pdh.CollectQueryData(hq) #collects data for the counter
type, pid = win32pdh.GetFormattedCounterValue(counter_handle, win32pdh.PDH_FMT_LONG)

path2 = win32pdh.MakeCounterPath( (None, object, instance, None, 0, object3) )
counter_handle2 = win32pdh.AddCounter (hq, path2)
win32pdh.CollectQueryData(hq) #collects data for the counter
type, ppid = win32pdh.GetFormattedCounterValue(counter_handle2, win32pdh.PDH_FMT_LONG)
win32pdh.CloseQuery(hq)
processList.append ([pid, ppid])
return processList

def getListOfProcesses (logger, isLinux):
""" This function returns a list of pairs pid ppid for all the processes in the system.
The function has different implementations for Linux and windows. """
if isLinux:
return getListOfProcessesLinux (logger);
else:
return getListOfProcessesWindows (logger);

def getLastDescendent (pid, logger, isLinux):
""" This function returns the last descendent of the given process. """
descendent = pid
processes = getListOfProcesses (logger, isLinux)
exitLoop = False
while (not exitLoop):
# Look for the first descendent of the process. The case of more than one descendent is not considered
found = False
for entry in processes:
if entry[1] == descendent:
found = True
descendent = entry[0]
break

if not found:
exitLoop = True
return descendent


Monitoring the memory usage


This is the code for the memory usage:

def getVmPeakLinux (pid, logger): try:
cmd = 'cat /proc/' + pid + '/status'
out = executeCommand (cmd, logger, True) pos = out.find('VmPeak:')

memStr = out [pos + 7:pos + 17]
memStr = memStr.strip() except Exception, detail:
memStr = '0'
logger.error ('getVmPeakLinux. Cannot get memory information. ' + str(deta
il))
return memStr

def getVmPeakWindows (pid, logger):
try:
pid = int (pid)
han = win32api.OpenProcess (win32con.PROCESS_ALL_ACCESS, False, pid)
info = win32process.GetProcessMemoryInfo (han)

memStr = str(info['PeakPagefileUsage'])
except Exception,detail:
memStr = '0'
logger.error ('getVmPeakWindows. Cannot get memory information. ' + str(de
tail))

return memStr

def getVmPeak (pid, logger, isLinux):
""" This function receives a pid and returns a string indicating the peak of t
he process virtual memory
size.
The function has different implementations for Linux and windows."""
if isLinux:
return (getVmPeakLinux (pid, logger))
else:
return (getVmPeakWindows (pid, logger))


def getVmSizeLinux (pid, logger): try:
cmd = 'cat /proc/' + pid + '/status'
out = executeCommand (cmd, logger, True)
pos = out.find('VmSize:')
memStr = out [pos + 7:pos + 17]
memStr = memStr.strip()
except Exception, detail:
memStr = '0'
logger.error ('getVmSizeLinux. Cannot get memory information. ' + str(deta
il)) return memStr


def getVmSizeWindows (pid, logger):
try:
pid = int(pid)
han = win32api.OpenProcess (win32con.PROCESS_ALL_ACCESS, False, pid)
info = win32process.GetProcessMemoryInfo (han)

memStr = str(info['PagefileUsage'])
except Exception,detail:
memStr = '0'
logger.error ('getVmSizeWindows. Cannot get memory information. ' + str(de
tail) )
return memStr

def getVmSize (pid, logger, isLinux):
""" This function receives a pid and returns a string indicating the current p
rocess virtual memory size.
The function has different implementations for Linux and windows."""
if isLinux:
return (getVmSizeLinux (pid, logger))
else:
return (getVmSizeWindows (pid, logger))

Monitoring the CPU usage


This is the actual code:

def getIdleTimeLinux (logger):
retVal = 110.0
cmd = 'top -b -d 2 -n 2'

output = executeCommand (cmd, logger, True)
pos = output.find ('%id')
if pos > 0:
output = output [pos + 3:]
pos = output.find ('%id')
if pos > 0:
percentStr = output [pos - 5:pos]
percentStr = percentStr.strip()
retVal = float(percentStr)
else:
logger.error ('Cannot find the second idle')
else:
logger.error ('Cannot find the first idle')

return retVal

def getIdleTimeWindows (logger):
retVal = 110.0
object = win32pdhutil.find_pdh_counter_localized_name("Processor")

# I can't translate with find_pdh_counter_localized_name the name "% Processor time"
items, instances = win32pdh.EnumObjectItems (None, None, object, win32pdh.PERF_DETAIL_WIZARD)

hq = win32pdh.OpenQuery()
path = win32pdh.MakeCounterPath ((None, object, '_Total', None, -1, items[0]))
counter_handle = win32pdh.AddCounter (hq, path)
win32pdh.CollectQueryData(hq)
time.sleep (2)
win32pdh.CollectQueryData(hq)
type, retVal = win32pdh.GetFormattedCounterValue (counter_handle, win32pdh.PDH_FMT_LONG)
win32pdh.CloseQuery(hq)

return (100.0 - retVal)

def getIdleTime (logger, isLinux):
""" This function measures the current CPU usage. It returns the percentage of idle time for a certain
time period.
The function has different implementations for Linux and windows. """
if isLinux:
return getIdleTimeLinux (logger)
else:
return getIdleTimeWindows (logger)


The way to find out the platform we're running is:

if os.name=='posix':
isLinux=True
else:
isLinux=False

sábado, 12 de marzo de 2011

Accesing Java enums values from C++ with JNI

Imagine we have to acces the enum values of a Java class. For example:

public enum ComponentCode {
VAD_CODE ((byte)1),
DIARIZATOR_CODE ((byte)2),
FEATURES_EXTRACTOR_CODE((byte)3),
UBM_CODE((byte)4),
HYPERPARAMS_CODE((byte)5),
GENDER_ID_CODE((byte)6),
MODEL_TRAINER_CODE((byte)7),
STATS_GENERATOR_CODE((byte)8);

private byte code;

ComponentCode (byte code) {
this.code = code;
}

public byte getCode () {
return code;
}

@Override
public String toString () {
return "" + (int)code;
}

};


The following C function can be used to acces the enum values:

jobject getStaticFieldID(JNIEnv *env, jclass c, const char *name, const char *signature)
{
jfieldID field = env->GetStaticFieldID (c, name, signature);
if (NULL == field) {
std::string msg = std::string("Error finding field ") + std::string(name) +
std::string(" (") + std::string(signature) + std::string(")");
throwFrontEndException(env, msg.c_str());
}
jobject obj = env->GetStaticObjectField(c, field);
if (NULL == obj) {
std::string msg = std::string("Error finding field ") + std::string(name) +
std::string(" (") + std::string(signature) + std::string(")");
throwFrontEndException(env, msg.c_str());
}

return obj;
}

sábado, 19 de febrero de 2011

Using native libraries with gradle

I was looking for a way to use native libraries in a java project managed with gradle but I couldn't find any. That why I'm publishing the plugin I created to do so.

We have a multi-platform java project that uses natives JNI libraries to perform some computing intensive tasks. We use maven deploy:deploy-file to package the natives libraries in a jar and upload them to our server. The jar name structure is:
artefactId-x.x.x-platform.jar

The platform tag can be linux, win32 or win64.

So here comes the plugi I created to integrate the native library artifact in the whole gradle project. I'm learning gradle and groovy while doing this so don't expect a very elegant code.
A convention called nativeLib is used to provide the actual native artifact and the libraries are extracted to a directory called build/natives.

import org.gradle.api.DefaultTask
import org.gradle.api.tasks.TaskAction
import org.gradle.api.Project
import org.gradle.api.Plugin
import org.gradle.api.Task
import java.util.zip.*

class GetNatives implements Plugin {

def String url = "server_url"
def String nativeDirName = 'build/natives/'

/**
* This method extracts the jar provided in the URL and uncompress it in the given destination directory.
*/
def void getNativeJar (String addr, String dstDir) {

URL url = new URL (addr)
URLConnection uc = url.openConnection()

BufferedOutputStream dest = null;
ZipInputStream zis = new ZipInputStream (uc.getInputStream());
ZipEntry entry;
final int BUFFER=2048
while((entry = zis.getNextEntry()) != null) {
String fullName = dstDir + "/" + entry.getName();

// We ignore this directory
if (!(entry.getName().startsWith ('META-INF')))
{

if (entry.isDirectory ()) {

File theFile = new File (fullName)
theFile.mkdir ()
} else {
int count;
byte []data = new byte[BUFFER];
// write the files to the disk
FileOutputStream fos = new FileOutputStream(fullName);
dest = new
BufferedOutputStream(fos, BUFFER);
while ((count = zis.read(data, 0, BUFFER))
!= -1) {
dest.write(data, 0, count);
}
dest.flush();
dest.close();
}
}
}

zis.close ()


}

def void getNativeLibraries (String theBinString, File nativeDir) {
if (theBinString != 'none') {

nativeDir.mkdirs ()

String platformStr = GetPlatform.getPlatform()

String nativeLib = theBinString
List dependencyList = nativeLib.tokenize(',')
for (entry in dependencyList) {

List theList = entry.tokenize (':')
if ( theList.size() == 3) {
String path = theList [0].replaceAll ("\\.", '/') + '/' + theList[1] + '/' \
+ theList[2] + '/'
String jarName = theList[1] + '-' + theList[2] + '-natives-' \
+ platformStr + '.jar'
getNativeJar ( url + path + jarName, nativeDir.toString())
} else {
println 'Invalid native lib ' + entry
throw new Exception ()
}
}
}
}

def processProject (Project theProject, File nativeDir) {
// Process the project
getNativeLibraries (theProject.convention.plugins.nativeLib.bin, nativeDir)

// Process the children
for (entry in theProject.configurations.default.allDependencies) {

// Avoid the external dependencies
if ( entry.group.tokenize("\\.")[0] == theProject.group.tokenize("\\.")[0]) {
processProject (entry.dependencyProject.project, nativeDir)
}
}

}

def void apply(Project project) {
File nativeDir = project.file (nativeDirName)
project.convention.plugins.nativeLib = new GetNativesPluginConvention ()
project.task ('getNatives') {
outputs.dir nativeDir

doLast {
processProject (project, nativeDir)
}
}
}

}

class GetNativesPluginConvention {
def String bin = 'none'

def nativeLib (Closure closure) {
closure.delegate = this
closure()
}
}


class GetPlatform {
def static String OS_NAME=System.properties ['os.name']
def static String LINUX_NAME='Linux'
def static String OS_ARCH=System.properties ['os.arch']
def static String X86_NAME='x86'

// This method returns the platform we're running according to the system properties
static String getPlatform () {
String platform = "linux"

if (OS_NAME != LINUX_NAME) {
if (OS_ARCH == X86_NAME) {
platform = "win32"
} else {
platform = "win64"
}
}
return platform
}

static boolean isWindows () {
return (OS_NAME != LINUX_NAME)
}

static boolean isLinux () {
return (OS_NAME == LINUX_NAME)
}
}


In order to use this plugin from a build.gradle script the following lines must be added:
apply plugin: 'GetNatives'

nativeLib {
bin = 'org.foo:bar:2.1.1'
}

// Tell the JVM where to find the native libraries
test {
jvmArgs = ['-Djava.library.path=./build/natives']
}


nativeLib {
bin = 'groupId:artifactId:x.x.x'
}


lunes, 14 de febrero de 2011

Dependencies between maven artifacts

The number of artifacts that we are maintaining keeps growing and sometimes is difficult to figure out the dependencies. I know there are ways to see all the artifacts one artifact is dependent on but I couldn't find anything the other way round. So I created this python script that do both.

The scripts analyzes the POM in the provided repositories and then generates and html page showing all the dependencies. The used algorithms are a bit brute force and can be optimized but the times are OK for me so far. I'm also sure the python code can be made more elegant.


#! /usr/bin/env python
#
# This script calculates the dependencies among artifacts in the whole repository
#

import os
import xml.dom.minidom


# The following list contains the directories in the repository that contain artifacts
theRepository = "https://xxx.xxx.xxx.xxx/svn/"
repositoryList = [
theRepository + 'javaprojects/',
theRepository + 'projects/'
]

# Some artifacts are in a "special" structure
repositoryList2= [
theRepository + 'javaprojects/MavenPlugins/trunk/'
]

def executeCommand (cmd, logger, isLinux):
logger.info ("Execute command " + cmd)
startTime = datetime.datetime.now ()

proc = subprocess.Popen(cmd,
shell=isLinux,
stdout=subprocess.PIPE,
)
cmdOutput = proc.communicate()[0]
logger.info ("Command result " + cmdOutput)
endTime = datetime.datetime.now ()
diff = endTime - startTime
logger.info ("Overall time: " + str(diff));
return cmdOutput

def initializeLogger (name, fileName):
logger = logging.getLogger (name)
logger.setLevel(logging.INFO)
hdlr = logging.FileHandler(fileName)
hdlr.setFormatter (logging.Formatter ( "%(message)s"))
#"%(asctime)s %(name)s:%(lineno)d %(levelname)s %(message)s"))
logger.addHandler(hdlr)
return (logger)


class Artifact:
""" The class that contains the Artifact information. """
def __init__ (self, artifactId, groupId):
self.artifactId = artifactId;
self.groupId = groupId
self.dependencies = []
self.indirectDependencies = []
self.directUsers = []
self.indirectUsers = []
self.depTag = '_DEP'
self.userTag = '_USER'

def appendDependency (self, artifact):
if not self.isDependency (artifact):
self.dependencies.append (artifact)

def noOfDependencies (self):
return len (self.dependencies )

def getDependency (self, i):
return self.dependencies[i]

def noOfIndirectDependencies (self):
return len(self.indirectDependencies)

def getIndirectDependency (self, i):
return self.indirectDependencies[i]

def __str__ (self):
retVal = '(' + str (self.groupId) + "." + str(self.artifactId) + ') '
return retVal

def printDependencies (self):
""" Returns a string with all the dependencies """
retVal = str(self) + 'has ' + str(self.noOfDependencies()) + ' dependencies:\n'
for i in range (0, self.noOfDependencies()):
retVal += '\t' + str (self.getDependency(i)) + '\n'

retVal += ' It has ' + str(len(self.indirectDependencies)) + ' indirect dependencies:\n'
for entry in self.indirectDependencies:
retVal += '\t' + str(entry) + '\n'

return retVal

def printUsers (self):
""" Returns a printable string with all the users of this artifact """
retVal = str(self) + 'has ' + str(len(self.directUsers)) + ' direct users:\n'
for entry in self.directUsers:
retVal += '\t' + str(entry) + '\n'

retVal += ' It has ' + str(len(self.indirectUsers)) + ' indirect users:\n'
for entry in self.indirectUsers:
retVal += '\t' + str(entry) + '\n'

return retVal

def isDirectUser (self, user):
""" This function returns whether the given artifact is a direct user of this object or not."""
found = False
for entry in self.directUsers:
if entry == user:
found = True
return found

def isIndirectUser (self, user):
""" This function returns whether the given artifact is an indirect user of this object or not."""
found = False
for entry in self.indirectUsers:
if entry == user:
found = True
return found

def isUser (self, user):
""" This function returns whether the given artifact is a user of this object or not."""
return (self.isDirectUser (user) or self.isIndirectUser (user))

def addDirectUser (self, user):
if not self.isUser (user):
self.directUsers.append (user)

def addIndirectUser (self, user):
if not self.isUser (user):
self.indirectUsers.append (user)

def isDirectDependency (self, dep):
""" This method returns whether the given artifact is a direct dependency of this object or not."""
found = False
for entry in self.dependencies:
if entry == dep:
found = True
break

return found

def isIndirectDependency (self, dep):
""" This method returns whether the given artifact is an indirect dependency of this object or not."""
found = False

if not found:
for entry in self.indirectDependencies:
if entry == dep:
found = True
break

return found

def isDependency (self, dep):
""" This method returns whether the given artifact is a dependency of this object or not."""
return self.isDirectDependency (dep) or self.isIndirectDependency (dep)



def addIndirectDependency (self, dep):
""" This method adds an artifact in the indirect dependency list if it's not already on the list.
It returns True in case it's inserted."""

retVal = False
# Check the dependency list
if not self.isDependency (dep):
self.indirectDependencies.append (dep)
retVal = True

return retVal

def addListOfDependencies (self, list):
""" This function adds a list of dependencies if they are not already in the list. """
modified = False
for entry in list:
if self.addIndirectDependency (entry):
modified = True

return modified


def getListDependencies (self):
""" This method returns the complete list of dependencies."""
return self.dependencies + self.indirectDependencies

def getHTMLName (self):
return self.groupId + "." + self.artifactId

def getHTMLTag (self):
return 'ARTIFACT_' + self.getHTMLName ()

def getHTMLDepTag (self):
return self.getHTMLTag () + self.depTag

def getHTMLUserTag (self):
return self.getHTMLTag () + self.userTag

def printHTMLIndex (self, filePtr):
""" This method prints the index line for the artifact."""
filePtr.write ('<a href="#' + self.getHTMLTag() + '">' + self.getHTMLName() + '</a> ')
filePtr.write ('<a href="#' + self.getHTMLDepTag() + '">(dependencies</a>, ')
filePtr.write ('<a href="#' + self.getHTMLUserTag() + '">users)</a>, ')
filePtr.write ('<p>')

def printHTMLDependencies (self, filePtr):
filePtr.write ('<a name="' + self.getHTMLTag() + '"></a>')
filePtr.write ('<h2> Artifact ' + self.getHTMLName() + '</h2>')
filePtr.write ('<p>')
filePtr.write ('<a name="' + self.getHTMLDepTag() + '"></a>')
filePtr.write ('<h3> ' + self.getHTMLName() + ' dependencies </h3>')
self.printHTMLDependenciesTable (filePtr);
filePtr.write ('<p>')
filePtr.write ('<a name="' + self.getHTMLUserTag() + '"></a>')
filePtr.write ('<h3> ' + self.getHTMLName() + ' users</h3>')
self.printHTMLUsersTable (filePtr);
filePtr.write ('<p>')

def printHTMLDependenciesTable (self,filePtr):
filePtr.write ('<table border="0">\n<tr>')
filePtr.write ('<td width = 400> <b>' + 'Direct dependencies (' + str(len(self.dependencies)) +\
') </b></td>\n')
filePtr.write ('<td width = 400><b>' + 'Indirect dependencies (' +\
str(len(self.indirectDependencies)) + ') </b></td></tr>\n<tr>')
filePtr.write ('<td VALIGN=TOP>')

for entry in self.dependencies:
filePtr.write (entry.getHTMLName() + ' <a href="#' + entry.getHTMLTag() + '">(ref)</a> <p>\n')

filePtr.write ('</td>\n<td VALIGN=TOP>')

for entry in self.indirectDependencies:
filePtr.write (entry.getHTMLName() + ' <a href="#' + entry.getHTMLTag() + '">(ref)</a> <p>\n')

filePtr.write ('</td>\n</tr>\n')
filePtr.write ('</table>\n')
filePtr.write ('<p>')

def printHTMLUsersTable (self, filePtr):
filePtr.write ('<table border="0">\n<tr>')
filePtr.write ('<td width = 400> <b>' + 'Direct users (' + \
str(len(self.directUsers)) + ') </b></td>\n')
filePtr.write ('<td width = 400><b>' + 'Indirect users (' + str(len(self.indirectUsers)) + \
') </b></td></tr>\n<tr>')
filePtr.write ('<td VALIGN=TOP>')

for entry in self.directUsers:
filePtr.write (entry.getHTMLName() + ' <a href="#' + entry.getHTMLTag() + '">(ref)</a> <p>\n')

filePtr.write ('</td>\n<td VALIGN=TOP>')

for entry in self.indirectUsers:
filePtr.write (entry.getHTMLName() + ' <a href="#' + entry.getHTMLTag() + '">(ref)</a> <p>\n')

filePtr.write ('</td>\n</tr>\n')
filePtr.write ('</table>\n')
filePtr.write ('<p>')


def __eq__ (self, other):
return ((self.artifactId == other.artifactId) and (self.groupId == other.groupId))

def __ne__ (self, other):
return ((self.artifactId != other.artifactId) or (self.groupId != other.groupId))

def listDirectory (directory):
""" This function returns two lists with the contents of the given repository directory.
One containing the files and the other the directories."""
fileList = []
dirList = []

listCmd = 'svn list ' + directory

output = executeCommand (listCmd, logger, isLinux)

if len (output) > 0:
listAll = output.split ('\n');
for entry in listAll:
entry = entry.strip()
if len(entry) > 0:
if entry.endswith ('/'):
dirList.append (entry)
else:
fileList.append (entry)

return fileList, dirList


def listArtifact ():
""" This function looks for all the directories that contain a pom.xml file within the trunk directory.
We suppose these directories are artifacts.
The returned list contains pair of name and directory. """
artifactList = []
for dir in repositoryList:
fileList, dirList = listDirectory (dir)

for subDir in dirList:
fileList, dir2List = listDirectory (dir + subDir)
if dir2List.count ("trunk/") > 0:
fileList, dir3List = listDirectory (dir + subDir + 'trunk/')
if fileList.count ("pom.xml") > 0:
name = subDir [: len(subDir) -1]
artifactList.append ([name , dir + subDir + 'trunk/'])
return artifactList

def listArtifact2 ():
""" This function looks for all the directories that contain a pom.xml file within the subdirectories.
We suppose these directories are artifacts."""
artifactList = []
for dir in repositoryList2:
fileList, dirList = listDirectory (dir)

for subDir in dirList:
fileList, dir2List = listDirectory (dir + subDir)
if fileList.count ("pom.xml") > 0:
name = subDir [: len(subDir) -1]
artifactList.append ([name , dir + subDir ])

return artifactList

def listAllArtifacts ():
""" This function returns a list containing the name of all the artifacts in the repository along with
their path."""
artifactList = listArtifact ()
artifactList += listArtifact2 ()
return artifactList

def extractPomFile (repositoryDir):
""" This function downloads the pom file associated with an artifact. The path in the repository of the
artifact must be provided. The function returns true if the file has been downloaded. """
retVal = False

# Delete the pom.xml file if already exists
path = os.path.abspath ('pom.xml')
if os.path.exists (path):
os.remove (path)

exportCmd = 'svn export ' + repositoryDir + 'pom.xml'
output = executeCommand (exportCmd, logger, isLinux)

retVal = os.path.exists (path)


return retVal

def parsePomFile ():
""" This function returns and Artifact object containing the artifact information and its dependencies"""
domObject = xml.dom.minidom.parse ('pom.xml')

project = domObject.getElementsByTagName ('project')
projectChildren = project[0].childNodes

for i in range (0, projectChildren.length):
if projectChildren.item(i).nodeName == 'groupId':
groupName = projectChildren.item(i).firstChild.data
if projectChildren.item(i).nodeName == 'artifactId':
artifactName = projectChildren.item(i).firstChild.data
arte = Artifact (artifactName, groupName)

# Look for the dependencies in the parent section
parents = project[0].getElementsByTagName ('parent')
for i in range (0, parents.length):
parent = parents.item(i)
groupIdDep = parent.getElementsByTagName ('groupId')[0].firstChild.data
artifactIdDep = parent.getElementsByTagName ('artifactId')[0].firstChild.data
if groupIdDep.startswith ('org.example'):
art2 = Artifact (artifactIdDep, groupIdDep)
arte.appendDependency (art2)

# Look for the dependencies in the depencies section
dependencies = project[0].getElementsByTagName ('dependency')
if dependencies.length > 0:
for i in range (0, dependencies.length):
dependency = dependencies.item(i)
groupIdDep = dependency.getElementsByTagName ('groupId')[0].firstChild.data
artifactIdDep = dependency.getElementsByTagName ('artifactId')[0].firstChild.data
if groupIdDep.startswith ('org.example'):
art2 = Artifact (artifactIdDep, groupIdDep)
arte.appendDependency (art2)

return arte

def getDependencyList (artifactList, dep):
""" This function returns the list of dependencies of the artifacts of the given types."""

for entry in artifactList:
if dep == entry:
return entry.getListDependencies ()

return []


def findIndirectDependencies (artifactList, index):
""" This function finds all the indirect dependencies of an artifact. The list gets modified"""
baseArtifact = artifactList[index]
listModified = True

# Include the dependencies of the direct dependencies
for i in range (0, baseArtifact.noOfDependencies()):
dep = baseArtifact.getDependency (i)
if baseArtifact.addListOfDependencies (getDependencyList (artifactList, dep)):
listModified = True

while (listModified):
listModified=False
for i in range (0, baseArtifact.noOfIndirectDependencies()):
dep = baseArtifact.getIndirectDependency (i)
if baseArtifact.addListOfDependencies (getDependencyList (artifactList, dep)):
listModified = True

def findUsers (artifactList, index):
""" This function update the user list in the object. The list gets modified"""
baseArtifact = artifactList[index]

for i in range (0, len(artifactList)):
if i != index:
if artifactList[i].isDirectDependency (baseArtifact):
baseArtifact.addDirectUser (artifactList[i]);
elif artifactList[i].isIndirectDependency (baseArtifact):
baseArtifact.addIndirectUser (artifactList[i])

def lookArtifactDependencies ():
""" This function looks for all the artifacts in the repositories and then look for their dependencies.
The function returns a list of artifact."""
artifactNameList = listAllArtifacts ()
artifactList = []

for entry in artifactNameList:
if extractPomFile (entry[1]):
arte = parsePomFile ()
artifactList.append (arte)

# Calculate all the dependencies
for i in range (0, len (artifactList)):
findIndirectDependencies (artifactList, i)

# Calculate all the users
for i in range (0, len (artifactList)):
findUsers (artifactList, i)

return artifactList

def generateArtifactDependenciesPage (artifactList):
""" This function generates the artifact dependencies html page. """
htmlFile = './artifactDepends.html'
filePtr = open(htmlFile, 'w')
filePtr.write ('<!DOCTYPE html PUBLIC "-//IETF//DTD HTML 2.0//EN">\n <html>\n <head>\n <title>\n')
filePtr.write ('Artifact dependencies\n </title>\n </head>\n <body>\n')


filePtr.write ('The following list shows all the artifact that has been located in the repositories.')
filePtr.write ('For each artifact the list its dependencies and the list of artifact using them are')
filePtr.write (' printed.')
filePtr.write ('<h2>Index</h2>')
filePtr.write ('<p>')

for entry in artifactList:
entry.printHTMLIndex (filePtr)
for entry in artifactList:
entry.printHTMLDependencies (filePtr)

logger = initializeLogger ("artifactDependency", "artifactDependency.log")

if os.name=='posix':
isLinux=True
else:
isLinux=False


artifactList = lookArtifactDependencies ()
generateArtifactDependenciesPage (artifactList)




miércoles, 9 de febrero de 2011

Removing the .svn directories

I usually have this problem after coping a subversion downloaded directory tree. The .svn directories that are created in each directory are copied too. I created the following python script to get rid of them. You must be careful when using it because it deletes with no warning.

#!/usr/bin/env python

import shutil
import os

for root, dirs, files in os.walk (".", topdown=True):
for name in dirs:
if name == (".svn"):
fileName = os.path.join (root, name)
print fileName
shutil.rmtree (fileName)

jueves, 20 de enero de 2011

Compiling libsndfile for Android NDK r5

We've been trying to compile the libsndfile 1.0.23 library for Android and it was a bit tricky. I have described here how we did it.

The first step was to generate the config.h and sndfile.h that are generated by the './configure' script. This is the command line we used is (as described in here):

export NDK=path_to_ndk
export PATH=$PATH:$NDK/toolchains/arm-eabi-4.4.0/prebuilt/linux-x86/bin

./configure \
--host=arm-eabi \
CC=arm-eabi-gcc \
CPPFLAGS="-I $NDK/platforms/android-9/arch-arm/usr/include/" \
CFLAGS="-nostdlib" \
LDFLAGS="-Wl,-rpath-link=$NDK/platforms/android-9/arch-arm/usr/lib/ -L $NDK/platforms/android-9/arch-arm/usr/lib/" \
LIBS="-lc "

The next step is to create a Android.mk file for the library source file. All the source files are copied to a directory along with the Android.mk file. Please note all the files with a 'main' function have been excluded:


LOCAL_PATH := $(call my-dir)

include $(CLEAR_VARS)

LOCAL_MODULE := sndfile
LOCAL_SRC_FILES := mat5.c windows.c G72x/g723_24.c G72x/g72x.c \
G72x/g723_40.c G72x/g721.c G72x/g723_16.c \
float32.c chanmap.c test_endswap.c rf64.c sndfile.c htk.c dither.c \
test_log_printf.c txw.c ms_adpcm.c ima_adpcm.c flac.c aiff.c \
wav.c macbinary3.c mat4.c pcm.c caf.c \
audio_detect.c id3.c alaw.c macos.c file_io.c broadcast.c double64.c \
raw.c test_broadcast_var.c \
g72x.c command.c chunk.c avr.c sd2.c voc.c test_audio_detect.c \
mpc2k.c gsm610.c dwd.c \
interleave.c common.c test_strncpy_crlf.c sds.c pvf.c paf.c au.c \
test_float.c \
vox_adpcm.c ulaw.c strings.c svx.c test_conversions.c rx2.c nist.c \
GSM610/code.c GSM610/gsm_destroy.c \
GSM610/gsm_decode.c GSM610/short_term.c GSM610/gsm_create.c \
GSM610/decode.c GSM610/gsm_option.c \
GSM610/long_term.c GSM610/table.c GSM610/rpe.c GSM610/preprocess.c \
GSM610/gsm_encode.c GSM610/lpc.c \
GSM610/add.c dwvw.c wav_w64.c wve.c ogg.c w64.c test_file_io.c\
ircam.c xi.c ima_oki_adpcm.c
LOCAL_LDLIBS := -llog

include $(BUILD_SHARED_LIBRARY)

This shared library can be compiled as is or can be included in a larger project. For example, if we have the main library in a directory called jni. This library uses the libsndfile. The sndfile code in jni/sndfile the jni/Android.mk file should look like:


LOCAL_PATH := $(call my-dir)

include $(CLEAR_VARS)

LOCAL_MODULE := main
LOCAL_SRC_FILES := main.cpp
LOCAL_SHARED_LIBRARIES := sndfile

include $(BUILD_SHARED_LIBRARY)

include $(LOCAL_PATH)/sndfile/Android.mk

In order to use the libmain.so library from a Java application both libraries has to be included with:


System.loadLibrary ("sndfile");
System.loadLibrary ("main");

lunes, 3 de enero de 2011

Using CDash witn SMTP

Este resumen no está disponible. Haz clic en este enlace para ver la entrada.