The GITS Blog » programming

Counting words (etc.) in an HTML file with Python

Ryan Ginstrom — Sat, 17 May 2008 00:50:38 +0000

In a previous post, I wrote about how to count words, characters, and Asian characters using python.

In this post I want to pull that together with code to get a word count from an HTML file.

What needs counting

What needs counting depends to some extent on what you need the word count for, but here I'm going to be assuming that the word count is going to be used to count billable/localizable content.

In that scenario, you've got to count the text in the title tag, as well as the visible text in the body, and certain other localizable content: img alt attributes, a title attributes, and input value attributes (am I missing any?).

The Code

The code for counting the actual text is in the above link. Here we need code to extract the text from the HTML file, and to accumulate the counts for all the chunks we've extracted.

Here's the Segment class for accumulating counts:

class Segment(object):
"""Represents a text segment.
(For bookkeeping)
"""

def __init__(self, text=""):
""" text is the segment of text we will calculate.
Leave it empty if this will be a master count for a document

@param text: The text of the segment
"""

self.characters = len(text)

num_spaces = len([x for x in text if x.isspace()])
self.chars_no_spaces = self.characters - num_spaces

self.asian_chars = len([x for x in text if is_asian(x)])

self.non_asian_words = non_j_len(text)

self.words = self.non_asian_words + self.asian_chars

def accumulate(self, seg):
"""Add the stats from to this one.
Use this to keep a count for the entire document;
use another for the whole batch of documents

@param seg: The segment to accumulate

>>> seg = Segment(u"")
>>> seg2 = Segment(u"abc")
>>> seg.accumulate(seg2)
>>> seg.words
1
>>> seg.characters
3
"""

self.words += seg.words
self.characters += seg.characters
self.chars_no_spaces += seg.chars_no_spaces
self.asian_chars += seg.asian_chars
self.non_asian_words += seg.non_asian_words

Next, the code for extracting (segmenting) the text from an HTML file. For this, you'll need the excellent Beautiful Soup module.

#coding: UTF8
"""Html segmenter"""

from BeautifulSoup import BeautifulSoup as bsoup
from BeautifulSoup import BeautifulStoneSoup
import re

def normalize(text):
"""Normalize whitepace in C{text}.

>>> normalize(u" spam\\n\\tspam SPAM")
u'spam spam SPAM'
"""

return u' '.join(text.split())

class Segmenter(object):
"""Html segmenter
Retrieves the editable/translatable text from an HTML document.
"""

def __init__(self):
"""Set up various regular expressions for splitting the text"""

self.pre_parse_stripper = re.compile(u"|".join([u"|",
u"\s\S]*?>|",
u"\s\S]*?>|",
u"\s\S]*?>|",
u"[\s\S]*?",
u"\s\S]*?>|"]),
re.I | re.M)
"""Strip out unsightly tags before heading to the splitter"""

self.splitter = re.compile(u'|'.join([u"|

",
u"|

",
u"|",
u"|",
u"\d*?>|\d>",
u"|",
u"|",
u""]),
re.I | re.M)
"""Split segments by certain tags (removing tags in bargain)
These tags indicate a segment boundary"""

self.charset_finder = re.compile(u'[\s\S]*\s\S]*?charset\s*=\s*([\S]+)"[\s\S]*?>[\s\S]*', re.I)
"""Find the charset if necessary"""

self.soup = None

def __str__(self):
"""So we can tell which segger we have (assuming multiple segmenter classes)"""
return "HTML"

def get_chunks(self, html_text):
"""Extract the text from the HTML file"""

self.soup = bsoup(html_text, fromEncoding=self.getEncoding(html_text))

# document title
if self.soup.head:
title = self.soup.head.title
if title:
yield title.string

# image alt attributes, anchor title attributes, input value attributes
for tag, attr in ((u"img", u"alt"),
(u"a", u"title"),
(u"input", u"value")):
for chunk in self.getAttributes(tag, attr):
if chunk:
yield chunk

# Parse the body text
if self.soup.body:
text = self.pre_parse_stripper.sub(u"", unicode(self.soup.body))
for chunk in self.splitter.split(text):
normal = normalize(html2plain(chunk))
if normal:
yield normal

def getAttributes(self, tagName, attrName):
"""Get all attrName values for tagName tags"""

attrs = []

tags = self.soup.findAll(tagName)

for tag in tags:
try:
attr = tag[attrName]
if attr:
attrs.append(attr)
except KeyError, e:
#print "Tag %s does not have attribute %s" % (tagName, attrName)
pass

return attrs

def getEncoding(self, text):
"""Retrieve the encoding META tag, if present"""

m = self.charset_finder.match(text)
if m:
return m.groups(0)[0]
return None

TAG_STRIPPER = re.compile(u"<[!\w/][\s\S]*?>", re.I | re.M)

def strip_tags(line):
"""strip the HTML tags from the line

>>> strip_tags(u"spam")
u'spam'

"""

return TAG_STRIPPER.sub(u"", line)

def html2plain(text):
"""Strips out tags from HTML text

>>> html2plain('spam eggs')
u'spam\\xa0eggs'
>>> html2plain('–>')
u'–>'
"""

entities = BeautifulStoneSoup.HTML_ENTITIES
text = unicode(BeautifulStoneSoup(strip_tags(text),
convertEntities=entities))
return text.replace(u">", ">").replace(u"<", "<")

And here's some code to get the actual wordcount:

wordcount = docstats.Segment()
segger = htmlseg.Segmenter()

for chunk in segger.get_chunks(open("thefile.html").read()):
wordcount.accumulate(docstats.Segment(chunk))

Here are the docstats and htmlseg modules, and here is an online tool using the code for the HTML word counts.

Speeding up search on Honyaku archive site

Ryan Ginstrom — Tue, 29 Apr 2008 11:28:48 +0000

Last summer, I launched a new archive site for the Honyaku mailing list.

The site is written in Python using the django framework, with MySQL as the database. I chose MySQL because my tests showed that it was much faster than PostgreSQL at text searching.

Lately, however, the searches have been taking a huge amount of time. Sometimes they would even time out. It makes sense, since I've got more than 216,000 emails in there now, and body__icontains isn't exactly a speed demon.

But it was also taking forever just to get the posts for a given day. That was pretty easy to solve, though: duh, create an index on the date_sent field. So simple I never thought about it until the system was bogging down like a Golden Week traffic jam.

That solved the date problem, but my text search problem remained. In the end, I had to create a full-text index for the simple search. This solved the speed problem — queries take a second or two now — but the problem with MySQL's full-text index is that it has lousy support for Japanese text (which isn't delimited by spaces). For that reason, I kept the old, slow search method for the advanced search. If you use that, I recommend narrowing the search rather than just entering some body text.

In the end, I'm going to have to bite the bullet and install some kind of n-gram indexing scheme that will support Japanese. Right now, though, I simply don't have the time.

As a stopgap measure, I added a Google search for the Honyaku archive. Google doesn't seem to have indexed the site yet (I just took the main archive out of the robots.txt file), but when it does it'll be a quick way to search with good Japanese support. They even have a gadget that I can put on the Honyaku archive site, but I can't get it to keep the height I set for it.

What price elegance?

Ryan Ginstrom — Fri, 21 Mar 2008 04:03:03 +0000

In a recent post, I gave some code for counting the top n most frequent words in an arbitrary text file using itertools.groupby.

The code is written in a somewhat functional style. It's short and, dare I say, kind of elegant. But it turns out that this code is quite a bit slower than an imperative style using collections.defaultdict.

Here are the two functions:

from itertools import groupby
from collections import defaultdict

def get_top_freqs_gb(filename, num):
"""Get the top num words from filename as a list
of (word, freq) tuples, using itertools.groupby
"""
freqs = [(len(list(g)), k)
for k, g in groupby(sorted(get_words(filename)))]
return get_top(freqs, num)

def get_top_freqs_dd(filename, num):
"""Get the top num words from filename as a list
of (word, freq) tuples, using collections.defaultdict
"""
freq_dict = defaultdict(int)
for word in get_words(filename):
freq_dict[word] += 1
freqs =[(v, k)
for k, v in freq_dict.iteritems()]
return get_top(freqs, num)

Here are the helper functions:

import re

def get_words(filename):
"""Get the words from filename"""
split = re.compile(r"\b\w+\b").findall
return [word
for line in open(filename)
for word in split(line.lower())]

def get_top(freqs, num):
return [(b, a) for a, b in reversed(sorted(freqs)[num*-1:])]

The groupby version is shorter than the defaultdict version, and I'd say that it's simpler and more readable as well. Because it's shorter, the groupby version is less likely to contain bugs. In particular, the defaultdict version has a mutable local variable (used as an accumulator in the for loop), which is a classic source of bugs. The groupby version is also likely to be easier to maintain because it's shorter and simpler.

But the defaultdict version of the function winds up being considerably faster.

The times it took to run these functions 10 times on my computer, retrieving the top 50 most frequent words for "/python25/readme.txt", are as follows (seconds rounded to 4 decimal places).

	Without psyco	With psyco
groupby version	0.3133 s	0.2193 s
defaultdict version	0.2852 s	0.1818 s
groupby / defaultdict	1.41	1.58

The defaultdict version is 1.4x faster than the groupby version. This gap grows even further when psyco is used, making the defaultdict version nearly 1.6x as fast. I'd say that most of the reason for the slowness is that the groupby version of the function performs two sorts, compared to one sort in the defaultdict version.

(The psyco speedup for the defaultdict version comes from the for loop; changing get_words to return a generator expression eliminates the speedup. The speedup for the groupby version comes from the freq list comprehension; changing this to a generator expression eliminates its speedup.)

So which one should I use?

It's pretty common for Python code written in a functional style to be slower than equivalent code written in an imperative style. Nevertheless, I tend to prefer the more functional style of programming, switching to a more imperative style (or other forms of optimization) if performance isn't satisfactory.

It is easier to optimize correct code, than correct optimized code.

–Yves Deville

A big question here is how to tell if the functional version is fast enough. My general rule of thumb is that the user would be prepared to wait up to two seconds for a typical "grovel through these files and tell me something interesting" command that's performed infrequently (how frequently do you need to get word frequencies from files?). For a more common action, the wait time should be under a second, with < .5 seconds being optimal (this includes GUI responsiveness but not Web page loading).

Given the times above, and assuming that the user will search no more than 50 files of sizes comparable to Python's README file, then either version of the function is sufficient. If we assume that the user will search up to 100 files, or files substantially larger than the README file, then only the imperative version is acceptable (and we may need to optimize this further if our demands are higher than this).

That's why it's so important to profile and test Python programs from the very beginning. I keep a suite of test cases that I profile with every build (performed at least daily), noting trends in performance and optimizing when the code has gelled and bottlenecks remain.

Here is the test code:

from time import clock

def time_func(func, iterations, *args, **kwargs):
"""Return the time it takes to execute func
itertations times."""
start = clock()
for x in xrange(iterations):
func(*args, **kwargs)
return clock() - start

def main():
filename = "/python25/readme.txt"
top_gb = get_top_freqs_gb(filename, 100)
top_dd = get_top_freqs_dd(filename, 100)
assert top_gb == top_dd

for func in [get_top_freqs_gb, get_top_freqs_dd]:
name = func.__name__
seconds = time_func(func, 10, filename, 50)
print "%s: %s" % (name, seconds)

print "With psyco"

import psyco
psyco.full()

for func in [get_top_freqs_gb, get_top_freqs_dd]:
name = func.__name__
seconds = time_func(func, 10, filename, 50)
print "%s: %s" % (name, seconds)

if __name__ == "__main__":
main()

The whole shebang:

#coding: UTF8
"""
Testing functional programming stuff
"""

from itertools import groupby
from collections import defaultdict
import re
from time import clock

def get_words(filename):
"""Get the words from filename"""
split = re.compile(r"\b\w+\b").findall
return [word
for line in open(filename)
for word in split(line.lower())]

def get_top(freqs, num):
return [(b, a) for a, b in reversed(sorted(freqs)[num*-1:])]

for func in [get_top_freqs_gb, get_top_freqs_dd]:
name = func.__name__
seconds = time_func(func, 10, filename, 50)
print "%s: %s" % (name, seconds)

if __name__ == "__main__":
main()

Counting occurrences in a sequence with itertools.groupby

Ryan Ginstrom — Thu, 13 Mar 2008 05:29:38 +0000

itertools.groupby is a great tool for counting the numbers of occurrences in a sequence.

Here are some examples from the interactive interpreter.

A list of numbers

>>> # Create a random list of numbers
>>> from random import random
>>> numbers = [int(random() * 10) for x in range(20)]
>>> numbers
[8, 0, 3, 2, 3, 9, 8, 2, 8, 3, 0, 2, 3, 8, 6, 5, 3, 6, 1, 8]
>>> # Now create a dictionary of numbers and numbers
>>> # of occurrences. Feed generator expression of
>>> # (number, frequency) pairs to dict().
>>> from itertools import groupby
>>> valdict = dict((k, len(list(g)))
for k, g in groupby(sorted(numbers)))
>>> for key, val in valdict.items():
print key, ":", val

0 : 2
1 : 1
2 : 3
3 : 5
5 : 1
6 : 2
8 : 5
9 : 1

And a function that does this for any iterable:

from itertools import groupby

def count_occurrences(iterable):
"""return a dictionary with items and numbers of occurrences
in iterable"""

return dict((item, len(list(group)))
for item, group
in groupby(sorted(iterable)))

Top 20 most frequent words in a file

>>> # get a wordlist from the Python README
>>> text = open("/python25/readme.txt").read()
>>> words = text.lower().split()
>>> words[:5]
['this', 'is', 'python', 'version', '2.5.2′]
>>> # get the frequency list, using DSU to sort top words
>>> freqs = [(len(list(g)), k)
for k, g in groupby((sorted(words)))]
>>> # sort the freqs, get last 20, and reverse
>>> # to put most frequent first
>>> for a, b in reversed(sorted(freqs)[-20:]):
print "%s %s" % (b.ljust(7), str(a).rjust(3))

the 442
to 227
is 127
and 127
you 118
a 117
of 110
in 107
for 94
python 81
on 79
if 77
this 72
or 62
be 58
with 56
it 53
are 53
that 52
as 47

Here's a function that will do this.

from itertools import groupby

def get_top_freqs(filename, num=20):
"""Get the top num words from filename as a list
of (word, freq) tuples
"""

text = open(filename).read()
words = text.lower().split()

freqs = ((len(list(g)), k) for k, g in groupby(sorted(words)))

return [(b, a) for a, b in reversed(sorted(freqs)[num*-1:])]

Making the robot dance

Ryan Ginstrom — Wed, 12 Mar 2008 14:33:39 +0000

Some time around 1980, my elementary school classroom got a computer. While most of the other kids fooled around playing Hunt the Wumpus, my friend and I found the BASIC manual that came with the computer. We laboriously copied in the code to make a "robot" appear on the screen. After a lot of typos, we finally got the program working: a crudely drawn robot appeared on the side of the screen, and moved to the center.

Then my friend found the section on modifying the code to make the robot "dance" (basically move its arms up and down a few times after it got to the middle of the screen). I thought that was just about the coolest thing I'd ever seen. I started tweaking the program in various ways, turning the program into a mini disco, with flashing colors and sounds.

I was hooked. I started staying in at lunch so I could have time with the computer. I also pestered my mom to buy me a computer. And she did: a Tandy PC with 3KB of memory for programs, and a tape cassette deck for memory storage. It was great. One of my more ambitious projects was tweaking the horse race game that came with the computer to have odds and varying payoffs like at the tracks. I also made a "lemonade stand" clone. I think the only non-game I made was a "conversation" program that would chat with you and say different things depending on what you typed. It was very simple, of course: like if you typed "HELLO," it would respond, "HELLO, HOW ARE YOU?" (For some reason, I think the BASIC environment only had capital letters.)

In college, I would go on to learn more advanced programming languages, and fancy concepts like finite state theory and formal logic. But that same joy was always there: the joy of getting the computer to do my bidding. At one point I wrote a program to parse English text, and just like when I was 11 and wrote that "conversation" program, it was just so neat to have the computer respond to my input.

Now, I'm a technical translator, but I also program. Although I do program professionally to some extent, most of my programming is just for the pure enjoyment of it. I actually think that's better than being paid to program, because I have a lot more freedom about what I do.

I've often tried to pin down just what is so interesting about controlling a computer. My gut feeling is that it's just intrinsically fun — who wouldn't want to program computers? But of course that's not true: most people think that programming computers is really boring. What makes programming interesting to some and boring to others? Maybe it's a certain way of thinking. Or maybe it's ability: some people "get" programming, and since we tend to like what we're good at, we just gravitate to it.

Of course, there was always the tedious side of programming. Back when I was 11, it was copying programs out of books using hunt and peck, and loading in my programs from cassette tapes. Now, it's fiddling with user interfaces and installers and interoperability. But it's all worth it when I run the program and get to see the robot dance.

Using chardet to convert arbitrary byte strings to Unicode

Ryan Ginstrom — Sat, 08 Mar 2008 02:24:36 +0000

chardet is a fantastic module for finding the encoding of arbitrary byte strings. You can combine this with a check for a BOM to pretty reliably turn them into Unicode.

Edit: Thanks to Kirit's comment below, I added code to check for UTF-32.

import chardet

def bytes2unicode(bytes, errors='replace'):
"""Convert a byte string into Unicode.
First checks for a BOM, and if one is found returns
the Unicode text minus the BOM. If there is no BOM,
falls back to chardet."""

encoding_map = ('\xef\xbb\xbf', 'utf-8′),
　　　　('\xff\xfe\0\0', 'utf-32′),
　　　　('\0\0\xfe\xff', 'UTF-32BE'),
　　　　('\xff\xfe', 'utf-16′),
　　　　('\xfe\xff', 'UTF-16BE'))

for bom, encoding in encoding_map:
if bytes.startswith(bom):
return unicode(bytes[len(bom):],
encoding,
errors=errors)

# No BOM found, so use chardet
detection = chardet.detect(bytes)
encoding = detection.get('encoding') or 'utf-16′
return unicode(bytes, encoding, errors=errors)

Usage:

text = bytes2unicode(open(filename).read(), 'replace')

Discussion: Why check for a BOM?

You might ask, why check for a BOM if chardet already does this? This is because although chardet will correctly detect the BOM, it won't tell you that it found it, so you won't know to chop it off before processing the text. Which means that you'd have to check for a BOM anyway in most cases.

Python GUI programming platforms for Windows

Ryan Ginstrom — Tue, 26 Feb 2008 06:00:57 +0000

[Edit]
By popular demand, I've added a section on PyGTK. See bottom of post.

There are several platforms for programming Windows GUI applications in Python. Below I outline a few of them, with a simple "hello world" example for each. Where I've lifted the example from another site, there's a link to the source.

Tkinter

Tkinter is the ubiquitous GUI toolkit for Python. It's cross platform and easy to use, but it looks non-native on just about every platform. There are various add-ons and improvements you can find to improve the look and feel, but the basic problem is that the toolkit implements its own widgets, rather than using the native ones provided on the platform.

Pros

Most portable GUI toolkit for Python
Very easy to use, with pythonic API

Cons

Non-native look and feel out of the box

Hello world example (code source):

import Tkinter as Tk
la = Tk.Label(None, text='Hello World!', font=('Times', '18′))
la.pack()
la.mainloop()

wxPython

wxPython is probably the most popular GUI toolkit for Python. It's a wrapper for the wxWidgets C++ toolkit, and as such it betrays a few unpythonic edges (like lumpy case, getters and setters, and funky C++ errors creeping up occasionally). There are a few pythonification efforts on top of wxPython, such as dabo and (the now apparently moribund) wax.

Pros

Highly cross platform
Relatively mature and robust
Uses native Windows widgets for authentic look and feel

Cons

Must include large wx runtime when packaging with py2exe (adds ~7 MB)
Cross platform nature makes accessing some native platform features (like ActiveX) difficult to impossible

Hello world example (code source):

import wx

class Application(wx.Frame):
def __init__(self, parent):
wx.Frame.__init__(self, parent, -1, 'My GUI', size=(300, 200))
panel = wx.Panel(self)
sizer = wx.BoxSizer(wx.VERTICAL)
panel.SetSizer(sizer)
txt = wx.StaticText(panel, -1, 'Hello World!')
sizer.Add(txt, 0, wx.TOP|wx.LEFT, 20)
self.Centre()
self.Show(True)

app = wx.App(0)
Application(None)
app.MainLoop()

.NET with IronPython

IronPython is a .NET implementation of Python. As of 1.0 it has full support for Python 2.4 features, and the 2.0 version will duplicate the Python 2.5 feature set. Although there are many CPython libraries/modules that won't run under IronPython (namely, the ones relying on compiled extensions that have not yet been ported), this lack is partially made up by the huge .NET library.

One cool thing about IronPython is that you can easily create lightweight .exe files that you can ship off to your friends — although you pay for this with a dependency on the .NET runtime, which you can't count on random Windows users to have installed.

Of course, when you go the IronPython route, you take all that comes with it: the good things, like access to .NET libraries and possibly the easiest/cleanest optimization path of any Python implementation (C#); and the bad things, like dependence on the .NET runtime and danger of getting caught on the MS upgrade treadmill.

Another way of getting at the .NET libraries is Python.NET, which adds two files to your Python directory to enable you to call the CLR from CPython.

Pros

Leverage .NET libraries
Easily create .exe files

Cons

Depends on .NET runtime

Hello world example (code source):

import sys
sys.path.append(r'C:\Python24\Lib')

import clr
clr.AddReference("System.Windows.Forms")

from System.Windows.Forms import Application, Form

class HelloWorldForm(Form):

def __init__(self):
self.Text = 'Hello World'
self.Name = 'Hello World'

form = HelloWorldForm()
Application.Run(form)

PyQT

PyQT is probably the third most widely used GUI toolkit, after wxPython and Tkinter. It has a dual commercial/GPL license (Edit: but it does let you use other open-source licenses; see comments below). I have to admit that this made it a non-starter for me: I don't want to pay for my toolkit when there are others just as good or better that are free; ~~and when I do release open-source software, I want to choose my own license~~. For others, the GPL might be a non-issue or a plus, so I've left it off my pro/con list.

Pros

Highly cross platform
Very easy to use
Highly mature
Decent looking widgets

Cons

Somewhat non-native look and feel (though much better than Tkinter)
Must include large runtime when packaging with py2exe

Hello world example (from PyQT docs):

import sys
from PyQt4 import QtGui

app = QtGui.QApplication(sys.argv)

hello = QtGui.QPushButton("Hello world!")
hello.resize(100, 30)

hello.show()

sys.exit(app.exec_())

Pyglet

Pyglet is kind of the new kid on the block in terms of GUI toolkits, but it sure made a splash. It implements its own windowing system, but with no dependencies other than Python (for Python 2.5 users). You will need OpenGL to do decent 3D graphics, but that's hardly a black mark for pyglet — other libraries would love to make it this easy.

Pros

High degree of freedom for GUI creation
Only depends on Python
Large number of widgets

Cons

Purposely doesn't duplicate the native platform look and feel
Although there are a lot of widgets, you'll have to roll your own for many things the platform gives you for free.

Hello world example (slightly modified from code source):

from pyglet import font
from pyglet import window

win = window.Window(width=300, height=150, caption="Hello World")

ft = font.load('Arial', 36)
text = font.Text(ft, 'Hello, World!')

while not win.has_exit:
win.dispatch_events()
win.clear()
text.draw()
win.flip()

Win32 with ctypes

Of course, all you really need to write GUI applications on Windows with Python is your trusty ctypes module and a well worn copy of Petzold. The benefit of this style is that you're working right down at the system API level, with nothing to get in your way. The disadvantage is that you're working right down at the system API level, with nothing to relieve you from all that boilerplate (unless you write your own abstraction layer on top; see Venster, below…).

Pros

Enables high level of control
Straightforward if familiar with Win32 API
No added complexity or buried functionality due to need to be cross-platform
Lightest of all Windows GUI programming methods using Python

Cons

All the complexity and inconsistency of Win32 API in gory detail
Lack of high-level libraries (have to write more code)

Hello world example (long, ain't it?):

from ctypes import *
import win32con

WNDPROC = WINFUNCTYPE(c_long, c_int, c_uint, c_int, c_int)

NULL = c_int(win32con.NULL)
_user32 = windll.user32

def ErrorIfZero(handle):
if handle == 0:
raise WinError()
else:
return handle

CreateWindowEx = _user32.CreateWindowExW
CreateWindowEx.argtypes = [c_int,
c_wchar_p,
c_wchar_p,
c_int,
c_int,
c_int,
c_int,
c_int,
c_int,
c_int,
c_int,
c_int]
CreateWindowEx.restype = ErrorIfZero

class WNDCLASS(Structure):
_fields_ = [('style', c_uint),
('lpfnWndProc', WNDPROC),
('cbClsExtra', c_int),
('cbWndExtra', c_int),
('hInstance', c_int),
('hIcon', c_int),
('hCursor', c_int),
('hbrBackground', c_int),
('lpszMenuName', c_wchar_p),
('lpszClassName', c_wchar_p)]

def __init__(self,
wndProc,
style=win32con.CS_HREDRAW | win32con.CS_VREDRAW,
clsExtra=0,
wndExtra=0,
menuName=None,
className=u"PythonWin32",
instance=None,
icon=None,
cursor=None,
background=None,
):

if not instance:
instance = windll.kernel32.GetModuleHandleW(c_int(win32con.NULL))
if not icon:
icon = _user32.LoadIconW(c_int(win32con.NULL),
c_int(win32con.IDI_APPLICATION))
if not cursor:
cursor = _user32.LoadCursorW(c_int(win32con.NULL),
c_int(win32con.IDC_ARROW))
if not background:
background = windll.gdi32.GetStockObject(c_int(win32con.WHITE_BRUSH))

self.lpfnWndProc=wndProc
self.style=style
self.cbClsExtra=clsExtra
self.cbWndExtra=wndExtra
self.hInstance=instance
self.hIcon=icon
self.hCursor=cursor
self.hbrBackground=background
self.lpszMenuName=menuName
self.lpszClassName=className

class RECT(Structure):
_fields_ = [('left', c_long),
('top', c_long),
('right', c_long),
('bottom', c_long)]
def __init__(self, left=0, top=0, right=0, bottom=0 ):
self.left = left
self.top = top
self.right = right
self.bottom = bottom

class PAINTSTRUCT(Structure):
_fields_ = [('hdc', c_int),
('fErase', c_int),
('rcPaint', RECT),
('fRestore', c_int),
('fIncUpdate', c_int),
('rgbReserved', c_wchar * 32)]

class POINT(Structure):
_fields_ = [('x', c_long),
('y', c_long)]
def __init__( self, x=0, y=0 ):
self.x = x
self.y = y

class MSG(Structure):
_fields_ = [('hwnd', c_int),
('message', c_uint),
('wParam', c_int),
('lParam', c_int),
('time', c_int),
('pt', POINT)]

def pump_messages():
"""Calls message loop"""
msg = MSG()
pMsg = pointer(msg)

while _user32.GetMessageW(pMsg, NULL, 0, 0):
_user32.TranslateMessage(pMsg)
_user32.DispatchMessageW(pMsg)

return msg.wParam

class Window(object):
"""Wraps an HWND handle"""

def __init__(self, hwnd=NULL):
self.hwnd = hwnd

self._event_handlers = {}

# Register event handlers
for key in dir(self):
method = getattr(self, key)
if hasattr(method, "win32message") and callable(method):
self._event_handlers[method.win32message] = method

def GetClientRect(self):
rect = RECT()
_user32.GetClientRect(self.hwnd, byref(rect))
return rect

def Create(self,
exStyle=0 , # DWORD dwExStyle
className=u"WndClass",
windowName=u"Window",
style=win32con.WS_OVERLAPPEDWINDOW,
x=win32con.CW_USEDEFAULT,
y=win32con.CW_USEDEFAULT,
width=win32con.CW_USEDEFAULT,
height=win32con.CW_USEDEFAULT,
parent=NULL,
menu=NULL,
instance=NULL,
lparam=NULL,
):

self.hwnd = CreateWindowEx(exStyle,
className,
windowName,
style,
x,
y,
width,
height,
parent,
menu,
instance,
lparam)
return self.hwnd

def Show(self, flag):
return _user32.ShowWindow(self.hwnd, flag)

def Update(self):
if not _user32.UpdateWindow(self.hwnd):
raise WinError()

def WndProc(self, hwnd, message, wParam, lParam):

event_handler = self._event_handlers.get(message, None)
if event_handler:
return event_handler(message, wParam, lParam)
return _user32.DefWindowProcW(c_int(hwnd),
c_int(message),
c_int(wParam),
c_int(lParam))

## Lifted shamelessly from WCK (effbot)'s wckTkinter.bind
def EventHandler(message):
"""Decorator for event handlers"""
def decorator(func):
func.win32message = message
return func
return decorator

class HelloWindow(Window):
"""The application window"""

@EventHandler(win32con.WM_PAINT)
def OnPaint(self, message, wParam, lParam):
"""Draw 'Hello World' in center of window"""
ps = PAINTSTRUCT()
rect = self.GetClientRect()
hdc = _user32.BeginPaint(c_int(self.hwnd), byref(ps))
rect = self.GetClientRect()
flags = win32con.DT_SINGLELINE|win32con.DT_CENTER|win32con.DT_VCENTER
_user32.DrawTextW(c_int(hdc),
u"Hello, world!",
c_int(-1),
byref(rect),
flags)
_user32.EndPaint(c_int(self.hwnd), byref(ps))
return 0

@EventHandler(win32con.WM_DESTROY)
def OnDestroy(self, message, wParam, lParam):
"""Quit app when window is destroyed"""
_user32.PostQuitMessage(0)
return 0

def RunHello():
"""Create window and start message loop"""

# two-stage creation for Win32 windows
hello = HelloWindow()

# register window class…
wndclass = WNDCLASS(WNDPROC(hello.WndProc))
wndclass.lpszClassName = u"HelloWindow"

if not _user32.RegisterClassW(byref(wndclass)):
raise WinError()

# …then create Window
hello.Create( className=wndclass.lpszClassName,
instance=wndclass.hInstance,
windowName=u"Hello World")

# Show Window
hello.Show(win32con.SW_SHOWNORMAL)
hello.Update()

pump_messages()

RunHello()

Venster

Venster was a very promising wrapper over the Win32 API, borrowing heavily from WTL and ATL windowing techniques. Unfortunately, the project hasn't been updated in several years, and doesn't support the latest versions of Python (especially after ctypes.com was dropped).

Pros

Rational abstraction layer on top of Win32
Use to write native, lightweight (relatively speaking) GUI applications
Has most of the cool Win32 tricks like hosting ActiveX and Coolbars

Cons

Out of date; not updated in several years

Hello world example (code source):

from venster.windows import *
from venster.wtl import *

from venster import gdi

class MyWindow(Window):
_window_title_ = "Hello World"
_window_background_ = gdi.GetStockObject(WHITE_BRUSH)
_window_class_style_ = CS_HREDRAW | CS_VREDRAW

def OnPaint(self, event):
ps = PAINTSTRUCT()
hdc = self.BeginPaint(ps)
rc = self.GetClientRect()

msg = "Hello World"
gdi.TextOut(hdc, rc.width / 2, rc.height / 2, msg, len(msg))

self.EndPaint(ps)
msg_handler(WM_PAINT)(OnPaint)

def OnDestroy(self, event):
PostQuitMessage(NULL)
msg_handler(WM_DESTROY)(OnDestroy)

myWindow = MyWindow()
application = Application()
application.Run()

PyGTK

PyGTK seems to have a lot going for it as a cross-platform toolkit. It's also licensed under the LGPL, which I like a lot more than the GPL of PyQT. Unfortunately, it doesn't use native Windows widgets; it does a pretty good job of faking it, but it stands out like a Win32, .NET, or wxPython app wouldn't.

Pros

Cross platform
Lots of widgets
Voluminous (if somewhat disorganized) documenation

Cons

Native Win32 widgets not used (looks good, but not quite all the way there)
Must include large runtime when packaging with py2exe

Hello world example (code source):

import pygtk
pygtk.require('2.0′)
import gtk

class HelloWorld:

def hello(self, widget, data=None):
print "Hello World"

def delete_event(self, widget, event, data=None):
print "delete event occurred"
return False

def destroy(self, widget, data=None):
print "destroy signal occurred"
gtk.main_quit()

def __init__(self):
self.window = gtk.Window(gtk.WINDOW_TOPLEVEL)

self.window.connect("delete_event", self.delete_event)
self.window.connect("destroy", self.destroy)
self.window.set_border_width(10)

self.button = gtk.Button("Hello World")
self.button.connect("clicked", self.hello, None)
self.button.connect_object("clicked", gtk.Widget.destroy, self.window)

self.window.add(self.button)

self.button.show()
self.window.show()

def main(self):
gtk.main()

if __name__ == "__main__":
hello = HelloWorld()
hello.main()