Mercurial > hg > rlgallery-misc
changeset 0:5ba2123d2c20
Put this project under version control, finally.
Scripts for rlgallery.org, using a PostgreSQL backend. The recorder
system is in py/, CGI scripts are in web/.
| author | John "Elwin" Edwards <elwin@sdf.org> |
|---|---|
| date | Wed, 25 Jul 2012 21:59:42 -0700 |
| parents | |
| children | def7fecbd437 |
| files | py/cleandb.py py/recorder.py py/rlgalldb.py py/setupdb.py py/stats.py web/archive.cgi |
| diffstat | 6 files changed, 937 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/py/cleandb.py Wed Jul 25 21:59:42 2012 -0700 @@ -0,0 +1,20 @@ +#!/usr/bin/python +# cleandb.py: empty the database in an orderly fashion + +import rlgalldb as rlgall +import psycopg2 + +dbconn = psycopg2.connect("dbname=rlg") +dbcur = dbconn.cursor() + +dbcur.execute("UPDATE games SET offbytes = %s", [0]) + +for game in rlgall.gamelist: + dbcur.execute("DELETE FROM " + game.uname + ";") + +dbcur.execute("DELETE FROM players;") +dbconn.commit() + +dbcur.close() +dbconn.close() +exit()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/py/recorder.py Wed Jul 25 21:59:42 2012 -0700 @@ -0,0 +1,46 @@ +#!/usr/bin/python + +import os +import time +import calendar +import psycopg2 +import rlgalldb as rlgall + +# Contains a dir for everyone who registered +everydir = "/var/dgl/dgldir/ttyrec/" +# Contains a page for everyone we know about +#knowndir = rlgall.dbdir + "players/" + +# Contact the database +conn = psycopg2.connect("dbname=rlg") +cur = conn.cursor() + +# newnames is the list of newly registered players who are not yet in the +# database. updatenames is the set of players whose pages need updating. +cur.execute("SELECT pname FROM players;") +playersInDB = [ row[0] for row in cur.fetchall() ] +playersAll = os.listdir(everydir) +newnames = [ name for name in playersAll if name not in playersInDB ] +updatenames = set(newnames) + +# Add the new names to the database +for newplayer in newnames: + cur.execute("INSERT INTO players VALUES (%s);", [newplayer]) +conn.commit() +cur.close() +conn.close() + +# Update the database for each game. +for game in rlgall.gamelist: + updatenames.update(game.loadnew()) + +# All the databases have been updated. Now make the pages. + +# Currently the high scores for all the games are on the same page. If +# they split up, this will have to change to a Game method. +rlgall.highpage() + +for name in updatenames: + rlgall.playerpage(name) + +exit()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/py/rlgalldb.py Wed Jul 25 21:59:42 2012 -0700 @@ -0,0 +1,425 @@ +# rlgalldb.py +# Module for the Roguelike Gallery, using a postgres database + +import os +import sys +import time +import calendar +import re +import psycopg2 +from datetime import datetime, tzinfo, timedelta + +# Configuration +logdir = "/var/dgl/var/games/roguelike/" +webdir = "/var/www/lighttpd/scoring/" +ppagename = webdir + "players/{0}.html" +hpagename = webdir + "highscores.html" + +# HTML fragments for templating +phead = """<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"> +<html><head> +<title>{0}</title> +<link rel="stylesheet" href="/scoring/scores.css" type="text/css"> +</head> +""" + +ptop = """<body> +<h1>Yendor Guild</h1> +""" + +navtop = '<div class="nav"><a href="/">rlgallery.org</a> -> {0}</div>\n' +navscore = '<div class="nav"><a href="/">rlgallery.org</a> -> \ +<a href="/scoring/">Scores</a> -> {0}</div>\n' +navplayer = '<div class="nav"><a href="/">rlgallery.org</a> -> \ +<a href="/scoring/">Scores</a> -> <a href="/scoring/players/">Players</a> \ +-> {0}</div>' + +pti = '<h2>{0}</h2>\n' + +secthead = '<h3>{0}</h3>\n' +tblhead = '<div class="stable">\n' +rowstart = '<div class="sentry">\n' +rowend = '</div>\n' +cell = ' <span class="sdata">{0}</span>\n' +hcell = ' <span class="shdata">{0}</span>\n' +tblend = '</div>\n' +pend = "</body></html>\n" + +# This would be more useful if we had to do translation +headerbook = {"etime":"End time", "score":"Score", "name":"Name", "xl":"XL", + "fate":"Fate", "rank":"Rank", "game":"Game"} +# Queries for the games table +offselstr = "SELECT offbytes FROM games WHERE gname = %s;" +newoffstr = "UPDATE games SET offbytes = %s WHERE gname = %s;" + +# A representation of the UTC time zone. They say Py3k can better handle +# this madness. +class UTC(tzinfo): + def utcoffset(self, dt): + return timedelta(0) + def dst(self, dt): + return timedelta(0) + def tzname(self, dt): + return "UTC" +utc = UTC() + +def getconn(): + "Returns a database connection, or None if the connection fails." + try: + conn = psycopg2.connect("dbname=rlg") + except psycopg2.OperationalError: + return None + return conn + +def recnameToInt(filename): + recre = r"(\d{4})-(\d{2})-(\d{2})\.(\d{2}):(\d{2}):(\d{2})\.ttyrec$" + match = re.match(recre, filename) + if not match: + return None + return calendar.timegm([int(val) for val in match.groups()]) + +def recnameToTS(filename): + pattern = "%Y-%m-%d.%H:%M:%S.ttyrec" + try: + dt = datetime.strptime(filename, pattern) + dt.replace(tzinfo=utc) + return dt + except ValueError: + return None + +def ttyreclink(text, name, game, gtime): + "Returns a link to the ttyrec archivist" + lstr = '<a href="/archive.cgi?name={0};game={1};time={2}">{3}</a>' + return lstr.format(name, game, gtime, text) + +def playerlink(name): + "Returns a link to a player's page" + lstr = '<a href="/scoring/players/' + name + '.html">' + name + '</a>' + return lstr + +def linktoArchive(entry): + "Takes an entry dict and returns a link to the ttyrec archivist." + lstr = '<a href="/archive.cgi?name={0};game={1};time={2}">{3}</a>' + #linktext = time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(entry["etime"])) + linktext = entry["etstamp"].strftime("%Y/%m/%d %H:%M:%S") + return lstr.format(entry["name"], entry["game"].uname, entry["etime"], + linktext) + +def maketablerow(cells, isheader=None): + "Takes a list of strings and returns a HTML table row with each string \ + in its own cell. isheader will make them header cells, obviously." + if isheader: + celler = hcell + else: + celler = cell + rowstr = rowstart + for entry in cells: + rowstr = rowstr + celler.format(entry) + rowstr = rowstr + rowend + return rowstr + +def printTable(entries, fields, of): + "Takes a list of entry dicts and a list of field strings and writes a \ + HTML table to of." + of.write(tblhead) + clist = [] + for field in fields: + if field in headerbook: + clist.append(headerbook[field]) + else: + clist.append("{0}".format(field)) + of.write(maketablerow(clist, True)) + rnum = 0 + for i, entry in enumerate(entries): + clist = [] + for field in fields: + if field == "rank": + clist.append("{0}".format(i + 1)) + elif field in entry: + thing = entry[field] + if field == "game": + clist.append(thing.name) + elif field == "xl" or field == "score": # Numerics + clist.append(str(thing)) + elif field == "name": + clist.append(playerlink(thing)) + elif field == "fate": + clist.append(thing) + elif field == "etime": + clist.append(linktoArchive(entry)) + else: + clist.append("{0}".format(thing)) + else: + clist.append("N/A") + of.write(maketablerow(clist)) + of.write(tblend) + return + +def readentries(entfile, entlist): + "Reads a list of entries from a file object" + while True: + nextentry = entfile.readline() + if not nextentry: + break + if nextentry[-1] != '\n': # The line is incomplete + break + entlist.append(nextentry.split(None, 4)) + return + +class Game: + pass + +class RogueGame(Game): + def __init__(self, name, uname, suffix): + self.name = name + self.uname = uname + self.scores = logdir + uname + ".log" + self.logspec = ["etime", "score", "name", "xl", "fate"] + self.sqltypes = {"etime": "int", "score": "int", "name": "varchar(20)", + "xl": "int", "fate": "text", "stime": "int", + "ttyrecs": "text ARRAY", "ststamp": "timestamptz", + "etstamp": "timestamptz"} + self.logdelim = " " + # Class variables, used by some methods + fields = ["name", "score", "xl", "fate", "etime"] + rankfields = ["rank", "score", "name", "xl", "fate", "etime"] + pfields = ["score", "xl", "fate", "etime"] + def logtoDict(self, entry): + "Processes a log entry string, returning a dict." + ndict = {"game": self} + entrylist = entry.strip().split(self.logdelim, len(self.logspec) - 1) + for item, value in zip(self.logspec, entrylist): + if self.sqltypes[item] == "int": + ndict[item] = int(value) + else: + ndict[item] = value + return ndict + def getEntryDicts(self, entfile, entlist): + "Reads logfile entries from entfile, interprets them according to the \ + instructions in self.logspec/logdelim, and adds them as dicts to entlist." + while True: + nextentry = entfile.readline() + if not nextentry: + break + if nextentry[-1] != '\n': + break + entlist.append(self.logtoDict(nextentry)) + return + def getRecent(self, n=20): + "Gets the n most recent games out of the database, returning a list \ + of dicts." + try: + n = int(n) + except (ValueError, TypeError): + return [] + if n <= 0: + return [] + dictlist = [] + conn = psycopg2.connect("dbname=rlg") + cur = conn.cursor() + qstr = "SELECT etime, score, name, xl, fate, ststamp, etstamp FROM {0} ORDER BY etstamp DESC \ + LIMIT %s;".format(self.uname) + cur.execute(qstr, [n]) + for record in cur: + # This should be less hardcodish + ndict = {"game": self} + ndict["etime"] = record[0] + ndict["score"] = record[1] + ndict["name"] = record[2] + ndict["xl"] = record[3] + ndict["fate"] = record[4] + ndict["ststamp"] = record[5] + ndict["etstamp"] = record[6] + dictlist.append(ndict) + cur.close() + conn.close() + return dictlist + def getHigh(self, n=10, offset=0): + "Gets the n highest scores (starting at offset) from the database, \ + returning a list of dicts." + qstr = "SELECT etime, score, name, xl, fate, ststamp, etstamp FROM {0} ORDER BY score DESC\ + ".format(self.uname) + qvals = [] + try: + n = int(n) + except (ValueError, TypeError): + return [] + if n <= 0: + return [] + qstr += " LIMIT %s" + qvals.append(n) + try: + offset = int(offset) + except (ValueError, TypeError): + return [] + if n > 0: + qstr += " OFFSET %s" + qvals.append(offset) + qstr += ";" + conn = psycopg2.connect("dbname=rlg") + cur = conn.cursor() + cur.execute(qstr, qvals) + dictlist = [] + for record in cur: + ndict = {"game": self} + ndict["etime"] = record[0] + ndict["score"] = record[1] + ndict["name"] = record[2] + ndict["xl"] = record[3] + ndict["fate"] = record[4] + ndict["ststamp"] = record[5] + ndict["etstamp"] = record[6] + dictlist.append(ndict) + cur.close() + conn.close() + return dictlist + def getPlayer(self, player): + "Gets all player's games from the database." + qstr = "SELECT etime, score, name, xl, fate, ststamp, etstamp FROM " + self.uname + " WHERE \ + name = %s;" + conn = getconn() + if conn == None: + return [] + cur = conn.cursor() + entries = [] + cur.execute(qstr, [player]) + for record in cur: + ndict = {"game": self} + ndict["etime"] = record[0] + ndict["score"] = record[1] + ndict["name"] = record[2] + ndict["xl"] = record[3] + ndict["fate"] = record[4] + ndict["ststamp"] = record[5] + ndict["etstamp"] = record[6] + entries.append(ndict) + cur.close() + conn.close() + return entries + def putIntoDB(self, dictlist, conn): + "Add the entries in dictlist to the database through connection conn, \ + which needs the INSERT privilege." + # FIXME this monster needs to be procedurally generated + qstr = "INSERT INTO " + self.uname + " (etime, score, name, xl, fate, stime, ttyrecs, ststamp, etstamp) \ + VALUES (%(etime)s, %(score)s, %(name)s, %(xl)s, %(fate)s, %(stime)s, %(ttyrecs)s, %(ststamp)s, %(etstamp)s);" + cur = conn.cursor() + cur.executemany(qstr, [ d for d in dictlist if d["game"] == self ]) + conn.commit() + cur.close() + return + def postprocess(self, gamelist): + names = set([ e["name"] for e in gamelist ]) + conn = getconn() + if conn == None: + return [] + cur = conn.cursor() + for nameF in names: + # Get all this player's games ordered by time + itsEntries = [ entry for entry in gamelist if entry["name"] == nameF ] + itsEntries.sort(key=lambda e: e["etime"]) + # Find the end time of the latest game already in the db + tquery = "SELECT etstamp FROM {0} WHERE name = %s ORDER BY etstamp DESC LIMIT 1;".format(self.uname) + cur.execute(tquery, [nameF]) + result = cur.fetchone() + if result: + prev = calendar.timegm(result[0].utctimetuple()) + else: + prev = 0 + ttyrecdir = "/var/dgl/dgldir/ttyrec/{0}/{1}/".format(nameF, self.uname) + allfilekeys = [ (recnameToInt(f), f) for f in os.listdir(ttyrecdir) ] + vfilekeys = [ e for e in allfilekeys if e[0] > prev ] + vfilekeys.sort(key=lambda e: e[0]) + # Now determine stime and ttyrecs for each game + for i in range(0, len(itsEntries)): + if i == 0: + lowlim = prev + else: + lowlim = itsEntries[i-1]["etime"] + hilim = itsEntries[i]["etime"] + recs = [ k[1] for k in vfilekeys if lowlim <= k[0] < hilim ] + itsEntries[i]["stime"] = recnameToInt(recs[0]) + itsEntries[i]["ttyrecs"] = recs + # While we're at it, replace numeric timestamps with SQL timestamps. + itsEntries[i]["ststamp"] = datetime.fromtimestamp(itsEntries[i]["stime"], utc) + itsEntries[i]["etstamp"] = datetime.fromtimestamp(itsEntries[i]["etime"], utc) + cur.close() + conn.close() + def loadnew(self): + conn = getconn() + if conn == None: + return [] + cur = conn.cursor() + # Get the previous offset + cur.execute(offselstr, [self.uname]) + offset = cur.fetchone()[0] + newlist = [] + try: + scr = open(self.scores) + scr.seek(offset) + self.getEntryDicts(scr, newlist) + except IOError: + noffset = offset # Can't read anything, assume no new games + else: + noffset = scr.tell() + scr.close() + cur.execute(newoffstr, [noffset, self.uname]) + # The new players must already be added to the players table. + updatenames = set([ e["name"] for e in newlist ]) + self.postprocess(newlist) + self.putIntoDB(newlist, conn) + cur.close() + conn.close() + return updatenames +# End RogueGame class definition + +rogue3 = RogueGame("Rogue V3", "rogue3", "r3") +rogue4 = RogueGame("Rogue V4", "rogue4", "r4") +rogue5 = RogueGame("Rogue V5", "rogue5", "r5") +srogue = RogueGame("Super-Rogue", "srogue", "sr") + +gamelist = [rogue3, rogue4, rogue5, srogue] + +def playerpage(pname): + "Generate a player's HTML page" + # Write the beginning of the page + ppagefi = open(ppagename.format(pname), "w") + ppagefi.write(phead.format(pname)) + ppagefi.write(ptop) + ppagefi.write(navplayer.format(pname)) + ppagefi.write(pti.format("Results for " + pname)) + for game in gamelist: + ppagefi.write(secthead.format(game.name)) + entries = game.getPlayer(pname) + if not entries: + ppagefi.write("<div>" + pname + " has not yet completed an expedition\ + in this dungeon.</div>\n") + else: + printTable(entries, RogueGame.pfields, ppagefi) + scoresum = 0 + for entry in entries: + scoresum += int(entry["score"]) + avgscr = scoresum / len(entries) + ppagefi.write("<p>Average score: {0}</p>\n".format(avgscr)) + # That should settle it. Print the end; then stop. + ppagefi.write(pend) + ppagefi.close() + return + +def highpage(): + # open the page and print the beginning + highpfi = open(hpagename, "w") + highpfi.write(phead.format("High Scores")) + highpfi.write(ptop) + highpfi.write(navscore.format("High Scores")) + highpfi.write(pti.format("List of High Scores")) + for game in gamelist: + highpfi.write(secthead.format(game.name)) + entries = game.getHigh(40) + if not entries: + highpfi.write("<div>No one has braved this dungeon yet.</div>\n") + else: + printTable(entries, game.rankfields, highpfi) + # That should finish it. + highpfi.write(pend) + highpfi.close() + return
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/py/setupdb.py Wed Jul 25 21:59:42 2012 -0700 @@ -0,0 +1,37 @@ +#!/usr/bin/python +# setupdb.py: initializes the database tables used by the rlg system. + +import rlgalldb as rlgall +import psycopg2 + +webuser = "webserver" +allowquery = "GRANT SELECT ON {0} TO " + webuser + ";" + +dbconn = psycopg2.connect("dbname=rlg") +dbcur = dbconn.cursor() + +dbcur.execute("CREATE TABLE games ( gname varchar(20), offbytes int );") +dbcur.execute("CREATE TABLE players (pname varchar(20) PRIMARY KEY);") +dbconn.commit() + +for game in rlgall.gamelist: + dbcur.execute("INSERT INTO games VALUES (%s, %s);", (game.uname, 0)) + + createquery = "CREATE TABLE " + game.uname + " ( " + for i, field in enumerate(game.sqltypes.keys()): + createquery += "{0} {1}".format(field, game.sqltypes[field]) + if field == "name": + createquery += " REFERENCES players(pname)" + if i == len(game.sqltypes) - 1: + createquery += " )" + else: + createquery += ", " + createquery += ";" + #print createquery + dbcur.execute(createquery) + dbcur.execute(allowquery.format(game.uname)) + dbconn.commit() + +dbcur.close() +dbconn.close() +exit()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/py/stats.py Wed Jul 25 21:59:42 2012 -0700 @@ -0,0 +1,35 @@ +import math +import psycopg2 +import rlgalldb as rlgall + +def makeExpPDF(lbd): + def lpdf(x): + return lbd * math.exp(-lbd * x) + return lpdf + +def makeExpCDF(lbd): + def lcdf(x): + return 1 - math.exp(-lbd * x) + return lcdf + +conn = psycopg2.connect("dbname=rlg") +cur = conn.cursor() +for game in rlgall.gamelist: + query = "SELECT score FROM {0};".format(game.uname) + cur.execute(query) + scores = [ r[0] for r in cur.fetchall() ] + count = len(scores) + total = sum(scores) + lbd = float(count) / total + cdf = makeExpCDF(lbd) + print "{0}: {1} games, average {2}".format(game.name, count, int(1/lbd)) + for i in range(0, 10000, 1000): + actual = len([ s for s in scores if i <= s < i + 1000 ]) + predicted = (cdf(i + 1000) - cdf(i)) * count + print "{0:5}: {1:4} {2}".format(i, actual, predicted) + high = max(scores) + print "Max: {0} {1}\n".format(high, 1 - cdf(high)) + +cur.close() +conn.close() +exit()