Commit 5730b7fb authored by Wolfgang Rohdewald's avatar Wolfgang Rohdewald
Browse files

new classes Csv and CSV, improve kajonggtest

parent dfc08077
......@@ -7,4 +7,4 @@ SPDX-License-Identifier: GPL-2.0
"""
# from intelligence import AIDefault
# from intelligence import AIDefaultAI
......@@ -279,7 +279,7 @@ class Options:
port = None
playOpen = False
gui = False
AI = 'Default'
AI = 'DefaultAI'
csv = None
continueServer = False
fixed = False
......
......@@ -10,11 +10,13 @@ SPDX-License-Identifier: GPL-2.0
import datetime
import weakref
import os
import sys
from collections import defaultdict
from functools import total_ordering
from twisted.internet.defer import succeed
from util import gitHead, CsvWriter
from util import gitHead
from kajcsv import CsvRow
from rand import CountingRandom
from log import logError, logWarning, logException, logDebug, i18n
from common import Internal, IntDict, Debug, Options
......@@ -121,7 +123,7 @@ class HandId(StrMixin):
@param withSeed: If set, include the seed used for the
random generator.
@type withSeed: C{Boolean}
@param withAI: If set and AI != Default: include AI name for
@param withAI: If set and AI != DefaultAI: include AI name for
human players.
@type withAI: C{Boolean}
@param withMoveCount: If set, include the current count of moves.
......@@ -134,8 +136,8 @@ class HandId(StrMixin):
if self.game.myself:
aiName = self.game.myself.intelligence.name()
else:
aiName = 'Default'
if aiName != 'Default':
aiName = 'DefaultAI'
if aiName != 'DefaultAI':
aiVariant = aiName + '/'
num = self.notRotated
assert isinstance(num, int), num
......@@ -808,22 +810,25 @@ class PlayingGame(Game):
"""write game summary to Options.csv"""
if self.finished() and Options.csv:
gameWinner = max(self.players, key=lambda x: x.balance)
writer = CsvWriter(Options.csv, mode='a')
if Debug.process and os.name != 'nt':
self.csvTags.append('MEM:%s' % resource.getrusage(
resource.RUSAGE_SELF).ru_maxrss)
if Options.rounds:
self.csvTags.append('ROUNDS:%s' % Options.rounds)
row = [self.ruleset.name, Options.AI,
gitHead(), '3',
str(self.seed), ','.join(self.csvTags)]
_ = CsvRow.fields
row = [''] * CsvRow.fields.PLAYERS
row[_.GAME] = str(self.seed)
row[_.RULESET] = self.ruleset.name
row[_.AI] = Options.AI
row[_.COMMIT] = gitHead()
row[_.PY_VERSION] = '{}.{}'.format(*sys.version_info[:2])
row[_.TAGS] = ','.join(self.csvTags)
for player in sorted(self.players, key=lambda x: x.name):
row.append(player.name)
row.append(player.balance)
row.append(player.wonCount)
row.append(1 if player == gameWinner else 0)
writer.writerow(row)
del writer
CsvRow(row).write()
def close(self):
"""log off from the server and return a Deferred"""
......
......@@ -20,7 +20,7 @@ from tilesource import TileSource
from meld import Meld, MeldList
from rule import Score, UsedRule
from common import Debug, StrMixin
from intelligence import AIDefault
from intelligence import AIDefaultAI
from util import callers
from message import Message
......@@ -85,7 +85,7 @@ class Hand(StrMixin):
# shortcuts for speed:
self._player = weakref.ref(player)
self.ruleset = player.game.ruleset
self.intelligence = player.intelligence if player else AIDefault()
self.intelligence = player.intelligence if player else AIDefaultAI()
self.string = string
self.__robbedTile = Tile.unknown
self.prevHand = prevHand
......
......@@ -14,7 +14,7 @@ from common import IntDict, Debug, StrMixin
from tile import Tile
class AIDefault:
class AIDefaultAI:
"""all AI code should go in here"""
......
# -*- coding: utf-8 -*-
"""
Copyright (C) 2008-2016 Wolfgang Rohdewald <wolfgang@rohdewald.de>
SPDX-License-Identifier: GPL-2.0
"""
import csv
import subprocess
import datetime
from enum import IntEnum
from functools import total_ordering
from common import Options, StrMixin
from player import Player, Players
class CsvWriter:
"""how we want it"""
def __init__(self, filename, mode='w'):
self.outfile = open(filename, mode)
self.__writer = csv.writer(self.outfile, delimiter=Csv.delimiter)
def writerow(self, row):
"""write one row"""
self.__writer.writerow([str(cell) for cell in row])
def __del__(self):
"""clean up"""
self.outfile.close()
class Csv:
"""how we want it"""
delimiter = ';'
@staticmethod
def reader(filename):
"""return a generator for decoded strings"""
return csv.reader(open(filename, 'r', encoding='utf-8'), delimiter=Csv.delimiter)
@total_ordering
class CsvRow(StrMixin):
"""represent a row in kajongg.csv"""
fields = IntEnum('Field', 'RULESET AI COMMIT PY_VERSION GAME TAGS PLAYERS', start=0)
commitDates = dict()
def __init__(self, row):
self.row = row
self.ruleset, self.aiVariant, self.commit, self.py_version, self.game, self.tags = row[:6]
self.winner = None
rest = row[6:]
players = []
while rest:
name, balance, wonCount, winner = rest[:4]
player = Player(None, name)
player.balance = balance
player.wonCount = wonCount
players.append(player)
if winner:
self.winner = player
rest = rest[4:]
self.players = Players(players)
@property
def commitDate(self):
"""return datetime"""
if self.commit not in self.commitDates:
try:
self.commitDates[self.commit] = datetime.datetime.fromtimestamp(
int(subprocess.check_output(
'git show -s --format=%ct {}'.format(self.commit).split(), stderr=subprocess.DEVNULL)))
except subprocess.CalledProcessError:
self.commitDates[self.commit] = datetime.datetime.fromtimestamp(0)
return self.commitDates[self.commit]
@property
def game(self):
"""return the game"""
return self.row[self.fields.GAME]
@game.setter
def game(self, value):
self.row[self.fields.GAME] = value
@property
def ruleset(self):
"""return the ruleset"""
return self.row[self.fields.RULESET]
@ruleset.setter
def ruleset(self, value):
self.row[self.fields.RULESET] = value
@property
def aiVariant(self):
"""return the AI used"""
return self.row[self.fields.AI]
@aiVariant.setter
def aiVariant(self, value):
self.row[self.fields.AI] = value
@property
def commit(self):
"""return the git commit"""
return self.row[self.fields.COMMIT]
@commit.setter
def commit(self, value):
self.row[self.fields.COMMIT] = value
@property
def py_version(self):
"""return the python version"""
return self.row[self.fields.PY_VERSION]
@py_version.setter
def py_version(self, value):
self.row[self.fields.PY_VERSION] = value
@property
def tags(self):
"""return the tags"""
return self.row[self.fields.TAGS]
@tags.setter
def tags(self, value):
self.row[self.fields.TAGS] = value
def result(self):
"""return a tuple with the fields holding the result"""
return tuple(self.row[self.fields.PLAYERS:])
def write(self):
"""write to Options.csv"""
writer = CsvWriter(Options.csv, mode='a')
writer.writerow(self.row)
del writer
def __eq__(self, other):
return self.row == other.row
def sortkey(self):
"""return string for comparisons"""
result = [self.game, self.ruleset, self.aiVariant,
self.commitDate or datetime.datetime.fromtimestamp(0), self.py_version]
result.extend(self.row[self.fields.TAGS:])
return result
def __lt__(self, other):
return self.sortkey() < other.sortkey()
def __getitem__(self, field):
"""direct access to row"""
return self.row[field]
def __hash__(self):
return hash(tuple(self.row))
def data(self, field):
"""return a string representing this field for messages"""
result = self.row[field]
if field == self.fields.COMMIT:
result = '{}({})'.format(result, self.commitDate)
return result
def differs_for(self, other):
"""return the field names for the source attributes causing a difference.
Possible values are commit and py_version. If both rows are identical, return None."""
if self.row[self.fields.PLAYERS:] != other.row[self.fields.PLAYERS:]:
differing = []
same = []
for cause in (self.fields.COMMIT, self.fields.PY_VERSION):
if self.row[cause] != other.row[cause]:
_ = '{} {} != {}'.format(cause.name, self.data(cause), other.data(cause))
differing.append(_)
else:
_ = '{} {}'.format(cause.name, self.data(cause))
same.append(_)
return ', '.join(differing), ', '.join(same)
def neutralize(self):
"""for comparisons"""
for idx, field in enumerate(self.row):
field = field.replace(' ', '')
if field.startswith('Tester ') or field.startswith('Tüster'):
field = 'Tester'
if 'MEM' in field:
parts = field.split(',')
for part in parts[:]:
if part.startswith('MEM'):
parts.remove(part)
field = ','.join(parts)
self.row[idx] = field
def __str__(self):
return 'Game {} {} AI={} commit={}({}) py={} {}'.format(
self.game, self.ruleset, self.aiVariant, self.commit, self.commitDate, self.py_version, self.tags)
......@@ -23,19 +23,11 @@ from optparse import OptionParser
from locale import getdefaultlocale
from common import Debug, StrMixin, cacheDir
from util import removeIfExists, gitHead, checkMemory
from util import Csv, CsvWriter, popenReadlines
from util import removeIfExists, gitHead, checkMemory, popenReadlines
from kajcsv import Csv, CsvRow, CsvWriter
signal.signal(signal.SIGINT, signal.SIG_DFL)
# fields in row:
RULESETFIELD = 0
AIFIELD = 1
COMMITFIELD = 2
PYTHON23FIELD = 3
GAMEFIELD = 4
TAGSFIELD = 5
PLAYERSFIELD = 6
OPTIONS = None
......@@ -265,7 +257,7 @@ class Job(StrMixin):
cmd.insert(0, 'python{}'.format(self.pythonVersion))
if OPTIONS.rounds:
cmd.append('--rounds={rounds}'.format(rounds=OPTIONS.rounds))
if self.aiVariant != 'Default':
if self.aiVariant != 'DefaultAI':
cmd.append('--ai={ai}'.format(ai=self.aiVariant))
if OPTIONS.csv:
cmd.append('--csv={csv}'.format(csv=OPTIONS.csv))
......@@ -318,184 +310,143 @@ class Job(StrMixin):
game = 'game={}'.format(self.game)
ruleset = self.shortRulesetName()
aiName = 'AI={}'.format(
self.aiVariant) if self.aiVariant != 'Default' else ''
self.aiVariant) if self.aiVariant != 'DefaultAI' else ''
return ' '.join([
self.commitId, 'Python{}'.format(self.pythonVersion), pid, game, ruleset, aiName]).replace(' ', ' ')
def neutralize(rows):
"""remove things we do not want to compare"""
for row in rows:
for idx, field in enumerate(row):
field = field.replace(' ', '')
if field.startswith('Tester ') or field.startswith('Tüster'):
field = 'Tester'
if 'MEM' in field:
parts = field.split(',')
for part in parts[:]:
if part.startswith('MEM'):
parts.remove(part)
field = ','.join(parts)
row[idx] = field
yield row
def onlyExistingCommits(commits):
"""filter out non-existing commits"""
global KNOWNCOMMITS # pylint: disable=global-statement
if not KNOWNCOMMITS:
for branch in subprocess.check_output(b'git branch'.split()).decode().split('\n'):
if 'detached' not in branch and 'no branch' not in branch:
KNOWNCOMMITS |= set(subprocess.check_output(
'git log --max-count=200 --pretty=%H {branch}'.format(
branch=branch[2:]).split()).decode().split('\n'))
result = list()
for commit in commits:
if any(x.startswith(commit) for x in KNOWNCOMMITS):
result.append(commit)
return result
def removeInvalidCommits(csvFile):
"""remove rows with invalid git commit ids"""
if not os.path.exists(csvFile):
return
rows = list(Csv.reader(csvFile))
_ = {x[COMMITFIELD] for x in rows}
csvCommits = {
x for x in _ if set(
x) <= set(
'0123456789abcdef') and len(
x) >= 7}
nonExisting = set(csvCommits) - set(onlyExistingCommits(csvCommits))
if nonExisting:
print(
'removing rows from kajongg.csv for commits %s' %
','.join(nonExisting))
writer = CsvWriter(csvFile)
for row in rows:
if row[COMMITFIELD] not in nonExisting:
writer.writerow(row)
# remove all logs referencing obsolete commits
def cleanup_data(csv):
"""remove all data referencing obsolete commits"""
logDir = os.path.expanduser(os.path.join('~', '.kajongg', 'log'))
knownCommits = csv.commits()
for dirName, _, fileNames in os.walk(logDir):
for fileName in fileNames:
if fileName not in KNOWNCOMMITS and fileName != 'current':
if fileName not in knownCommits and fileName != 'current':
os.remove(os.path.join(dirName, fileName))
try:
os.removedirs(dirName)
except OSError:
pass # not yet empty
Clone.removeObsolete()
def readGames(csvFile):
"""return a dict holding a frozenset of games for each variant"""
if not os.path.exists(csvFile):
return
allRowsGenerator = neutralize(Csv.reader(csvFile))
if not allRowsGenerator:
return
# we want unique tuples so we can work with sets
allRows = {tuple(x) for x in allRowsGenerator}
games = dict()
# build set of rows for every ai
for variant in {tuple(x[:COMMITFIELD]) for x in allRows}:
games[variant] = frozenset(
x for x in allRows if tuple(x[:COMMITFIELD]) == variant)
return games
def hasDifferences(rows):
"""True if rows have unwanted differences"""
return (len({tuple(list(x)[GAMEFIELD:]) for x in rows})
> len({tuple(list(x)[:COMMITFIELD]) for x in rows}))
def firstDifference(rows):
"""reduce to two rows showing a difference"""
result = rows
last = rows[-1]
while hasDifferences(result):
last = result[-1]
result = result[:-1]
return list([result[-1], last])
def closerLook(gameId, gameIdRows):
"""print detailled info about one difference"""
for ruleset in OPTIONS.rulesets:
for intelligence in OPTIONS.allAis:
shouldBeIdentical = [x for x in gameIdRows if x[RULESETFIELD] == ruleset and x[AIFIELD] == intelligence]
for commit in (x[COMMITFIELD] for x in shouldBeIdentical):
rows2 = [x for x in shouldBeIdentical if x[COMMITFIELD] == commit]
if hasDifferences(rows2):
first = firstDifference(rows2)
print('Game {} {} {} {} has differences between Python2 and Python3'.format(
gameId, ruleset, intelligence, commit))
for py23 in '23':
rows2 = [x for x in shouldBeIdentical if x[PYTHON23FIELD] == py23]
if hasDifferences(rows2):
first = firstDifference(rows2)
print('Game {} {} {} Python{} has differences between commits {} and {}'.format(
gameId, ruleset, intelligence, py23, first[0][COMMITFIELD], first[1][COMMITFIELD]))
def printDifferingResults(rowLists):
"""if most games get the same result with all tried variants,
dump those games that do not"""
allGameIds = {}
for rows in rowLists:
for row in rows:
rowId = row[GAMEFIELD]
if rowId not in allGameIds:
allGameIds[rowId] = []
allGameIds[rowId].append(row)
differing = []
for key, value in allGameIds.items():
if hasDifferences(value):
differing.append(key)
if not differing:
print('no games differ')
else:
print(
'differing games (%d out of %d): %s' % (
len(differing), len(allGameIds),
' '.join(sorted(differing, key=int))))
# now look closer at one example. Differences may be caused by git commits or by py2/p3
for gameId in sorted(differing):
closerLook(gameId, allGameIds[gameId])
def evaluate(games):
"""evaluate games"""
if not games:
return
for variant, rows in games.items():
gameIds = {x[GAMEFIELD] for x in rows}
if len(gameIds) != len({tuple(list(x)[GAMEFIELD:]) for x in rows}):
print(
'ruleset "%s" AI "%s" has different rows for games' %
(variant[0], variant[1]), end=' ')
for game in sorted(gameIds, key=int):
if len({tuple(x[GAMEFIELD:] for x in rows if x[GAMEFIELD] == game)}) > 1:
print(game, end=' ')
print()
break
printDifferingResults(games.values())
print()
print('the 3 robot players always use the Default AI')
print()
print('{ruleset:<25} {ai:<20} {games:>5} {points:>4} human'.format(
ruleset='Ruleset', ai='AI variant', games='games', points='points'))
for variant, rows in games.items():
ruleset, aiVariant = variant
print('{ruleset:<25} {ai:<20} {rows:>5} '.format(
ruleset=ruleset[:25], ai=aiVariant[:20],
rows=len(rows)), end=' ')
for playerIdx in range(4):
def pairs(data):
"""return all consecutive pairs"""
prev = None
for _ in data:
if prev:
yield prev, _
prev = _
class CSV(StrMixin):
"""represent kajongg.csv"""
knownCommits = None
def __init__(self):
self.findKnownCommits()
self.rows = []
if os.path.exists(OPTIONS.csv):
self.rows = list(sorted({CsvRow(x) for x in Csv.reader(OPTIONS.csv)}))
self.removeInvalidCommits()
def neutralize(self):
"""remove things we do not want to compare"""
for row in self.rows:
row.neutralize()
def commits(self):
"""return set of all our commit ids"""
# TODO: sorted by date
return {x.commit for x in self.rows}
def games(self):
"""return a sorted unique list of all games"""
return sorted({x.game for x in self.rows})
@classmethod
def findKnownCommits(cls):
"""find known commits"""
if cls.knownCommits is None:
cls.knownCommits = set()
for branch in subprocess.check_output(b'git branch'.split()).decode().split('\n'):
if 'detached' not in branch and 'no branch' not in branch:
cls.knownCommits |= set(subprocess.check_output(
'git log --max-count=400 --pretty=%H {branch}'.format(
branch=branch[2:]).split()).decode().split('\n'))
@classmethod
def onlyExistingCommits(cls, commits):
"""return a set with only existing commits"""
result = set()
for commit in commits:
if any(x.startswith(commit) for x in cls.knownCommits):
result.add(commit)
return result
def removeInvalidCommits(self):
"""remove rows with invalid git commit ids"""
csvCommits = {x.commit for x in self.rows}
csvCommits = {
x for x in csvCommits if set(
x) <= set(
'0123456789abcdef') and len(
x) >= 7}
nonExisting = csvCommits - self.onlyExistingCommits(set(x.commit for x in self.rows))
if nonExisting:
print(
'{p:>8}'.format(
p=sum(
int(x[
PLAYERSFIELD + 1 + playerIdx * 4]) for x in rows)),
end=' ')
print()
'removing rows from kajongg.csv for commits %s' %
','.join(nonExisting))
self.rows = [x for x in self.rows if x.commit not in nonExisting]
self.write()
def write(self):
"""write new csv file"""
writer = CsvWriter(OPTIONS.csv)
for row in self.rows:
writer.writerow(row