new classes Csv and CSV, improve kajonggtest

src/altint.py

@@ -7,4 +7,4 @@ SPDX-License-Identifier: GPL-2.0
 
 """
 
-# from intelligence import AIDefault
+# from intelligence import AIDefaultAI

src/common.py

@@ -279,7 +279,7 @@ class Options:
     port = None
     playOpen = False
     gui = False
-    AI = 'Default'
+    AI = 'DefaultAI'
     csv = None
     continueServer = False
     fixed = False

src/game.py

@@ -10,11 +10,13 @@ SPDX-License-Identifier: GPL-2.0
 import datetime
 import weakref
 import os
+import sys
 from collections import defaultdict
 from functools import total_ordering
 
 from twisted.internet.defer import succeed
-from util import gitHead, CsvWriter
+from util import gitHead
+from kajcsv import CsvRow
 from rand import CountingRandom
 from log import logError, logWarning, logException, logDebug, i18n
 from common import Internal, IntDict, Debug, Options
@@ -121,7 +123,7 @@ class HandId(StrMixin):
         @param withSeed: If set, include the seed used for the
         random generator.
         @type  withSeed: C{Boolean}
-        @param withAI:   If set and AI != Default: include AI name for
+        @param withAI:   If set and AI != DefaultAI: include AI name for
         human players.
         @type  withAI:   C{Boolean}
         @param withMoveCount:   If set, include the current count of moves.
@@ -134,8 +136,8 @@ class HandId(StrMixin):
             if self.game.myself:
                 aiName = self.game.myself.intelligence.name()
             else:
-                aiName = 'Default'
-            if aiName != 'Default':
+                aiName = 'DefaultAI'
+            if aiName != 'DefaultAI':
                 aiVariant = aiName + '/'
         num = self.notRotated
         assert isinstance(num, int), num
@@ -808,22 +810,25 @@ class PlayingGame(Game):
         """write game summary to Options.csv"""
         if self.finished() and Options.csv:
             gameWinner = max(self.players, key=lambda x: x.balance)
-            writer = CsvWriter(Options.csv, mode='a')
             if Debug.process and os.name != 'nt':
                 self.csvTags.append('MEM:%s' % resource.getrusage(
                     resource.RUSAGE_SELF).ru_maxrss)
             if Options.rounds:
                 self.csvTags.append('ROUNDS:%s' % Options.rounds)
-            row = [self.ruleset.name, Options.AI,
-                   gitHead(), '3',
-                   str(self.seed), ','.join(self.csvTags)]
+            _ = CsvRow.fields
+            row = [''] * CsvRow.fields.PLAYERS
+            row[_.GAME] = str(self.seed)
+            row[_.RULESET] = self.ruleset.name
+            row[_.AI] = Options.AI
+            row[_.COMMIT] = gitHead()
+            row[_.PY_VERSION] = '{}.{}'.format(*sys.version_info[:2])
+            row[_.TAGS] = ','.join(self.csvTags)
             for player in sorted(self.players, key=lambda x: x.name):
                 row.append(player.name)
                 row.append(player.balance)
                 row.append(player.wonCount)
                 row.append(1 if player == gameWinner else 0)
-            writer.writerow(row)
-            del writer
+            CsvRow(row).write()
 
     def close(self):
         """log off from the server and return a Deferred"""

src/hand.py

@@ -20,7 +20,7 @@ from tilesource import TileSource
 from meld import Meld, MeldList
 from rule import Score, UsedRule
 from common import Debug, StrMixin
-from intelligence import AIDefault
+from intelligence import AIDefaultAI
 from util import callers
 from message import Message
 
@@ -85,7 +85,7 @@ class Hand(StrMixin):
         # shortcuts for speed:
         self._player = weakref.ref(player)
         self.ruleset = player.game.ruleset
-        self.intelligence = player.intelligence if player else AIDefault()
+        self.intelligence = player.intelligence if player else AIDefaultAI()
         self.string = string
         self.__robbedTile = Tile.unknown
         self.prevHand = prevHand

src/intelligence.py

@@ -14,7 +14,7 @@ from common import IntDict, Debug, StrMixin
 from tile import Tile
 
 
-class AIDefault:
+class AIDefaultAI:
 
     """all AI code should go in here"""
 

src/kajcsv.py

+# -*- coding: utf-8 -*-
+
+"""
+Copyright (C) 2008-2016 Wolfgang Rohdewald <wolfgang@rohdewald.de>
+
+SPDX-License-Identifier: GPL-2.0
+
+"""
+
+import csv
+import subprocess
+import datetime
+
+from enum import IntEnum
+from functools import total_ordering
+
+from common import Options, StrMixin
+from player import Player, Players
+
+class CsvWriter:
+    """how we want it"""
+    def __init__(self, filename, mode='w'):
+        self.outfile = open(filename, mode)
+        self.__writer = csv.writer(self.outfile, delimiter=Csv.delimiter)
+
+    def writerow(self, row):
+        """write one row"""
+        self.__writer.writerow([str(cell) for cell in row])
+
+    def __del__(self):
+        """clean up"""
+        self.outfile.close()
+
+
+class Csv:
+    """how we want it"""
+
+    delimiter = ';'
+
+    @staticmethod
+    def reader(filename):
+        """return a generator for decoded strings"""
+        return csv.reader(open(filename, 'r', encoding='utf-8'), delimiter=Csv.delimiter)
+
+@total_ordering
+class CsvRow(StrMixin):
+    """represent a row in kajongg.csv"""
+
+    fields = IntEnum('Field', 'RULESET AI COMMIT PY_VERSION GAME TAGS PLAYERS', start=0)
+
+    commitDates = dict()
+
+    def __init__(self, row):
+        self.row = row
+        self.ruleset, self.aiVariant, self.commit, self.py_version, self.game, self.tags = row[:6]
+        self.winner = None
+        rest = row[6:]
+        players = []
+        while rest:
+            name, balance, wonCount, winner = rest[:4]
+            player = Player(None, name)
+            player.balance = balance
+            player.wonCount = wonCount
+            players.append(player)
+            if winner:
+                self.winner = player
+            rest = rest[4:]
+        self.players = Players(players)
+
+    @property
+    def commitDate(self):
+        """return datetime"""
+        if self.commit not in self.commitDates:
+            try:
+                self.commitDates[self.commit] = datetime.datetime.fromtimestamp(
+                    int(subprocess.check_output(
+                        'git show -s --format=%ct {}'.format(self.commit).split(), stderr=subprocess.DEVNULL)))
+            except subprocess.CalledProcessError:
+                self.commitDates[self.commit] = datetime.datetime.fromtimestamp(0)
+        return self.commitDates[self.commit]
+
+    @property
+    def game(self):
+        """return the game"""
+        return self.row[self.fields.GAME]
+
+    @game.setter
+    def game(self, value):
+        self.row[self.fields.GAME] = value
+
+    @property
+    def ruleset(self):
+        """return the ruleset"""
+        return self.row[self.fields.RULESET]
+
+    @ruleset.setter
+    def ruleset(self, value):
+        self.row[self.fields.RULESET] = value
+
+    @property
+    def aiVariant(self):
+        """return the AI used"""
+        return self.row[self.fields.AI]
+
+    @aiVariant.setter
+    def aiVariant(self, value):
+        self.row[self.fields.AI] = value
+
+    @property
+    def commit(self):
+        """return the git commit"""
+        return self.row[self.fields.COMMIT]
+
+    @commit.setter
+    def commit(self, value):
+        self.row[self.fields.COMMIT] = value
+
+    @property
+    def py_version(self):
+        """return the python version"""
+        return self.row[self.fields.PY_VERSION]
+
+    @py_version.setter
+    def py_version(self, value):
+        self.row[self.fields.PY_VERSION] = value
+
+    @property
+    def tags(self):
+        """return the tags"""
+        return self.row[self.fields.TAGS]
+
+    @tags.setter
+    def tags(self, value):
+        self.row[self.fields.TAGS] = value
+
+    def result(self):
+        """return a tuple with the fields holding the result"""
+        return tuple(self.row[self.fields.PLAYERS:])
+
+    def write(self):
+        """write to Options.csv"""
+        writer = CsvWriter(Options.csv, mode='a')
+        writer.writerow(self.row)
+        del writer
+
+    def __eq__(self, other):
+        return self.row == other.row
+
+    def sortkey(self):
+        """return string for comparisons"""
+        result = [self.game, self.ruleset, self.aiVariant,
+                  self.commitDate or datetime.datetime.fromtimestamp(0), self.py_version]
+        result.extend(self.row[self.fields.TAGS:])
+        return result
+
+    def __lt__(self, other):
+        return self.sortkey() < other.sortkey()
+
+    def __getitem__(self, field):
+        """direct access to row"""
+        return self.row[field]
+
+    def __hash__(self):
+        return hash(tuple(self.row))
+
+    def data(self, field):
+        """return a string representing this field for messages"""
+        result = self.row[field]
+        if field == self.fields.COMMIT:
+            result = '{}({})'.format(result, self.commitDate)
+        return result
+
+    def differs_for(self, other):
+        """return the field names for the source attributes causing a difference.
+        Possible values are commit and py_version. If both rows are identical, return None."""
+        if self.row[self.fields.PLAYERS:] != other.row[self.fields.PLAYERS:]:
+            differing = []
+            same = []
+            for cause in (self.fields.COMMIT, self.fields.PY_VERSION):
+                if self.row[cause] != other.row[cause]:
+                    _ = '{} {} != {}'.format(cause.name, self.data(cause), other.data(cause))
+                    differing.append(_)
+                else:
+                    _ = '{} {}'.format(cause.name, self.data(cause))
+                    same.append(_)
+            return ', '.join(differing), ', '.join(same)
+
+    def neutralize(self):
+        """for comparisons"""
+        for idx, field in enumerate(self.row):
+            field = field.replace(' ', '')
+            if field.startswith('Tester ') or field.startswith('Tüster'):
+                field = 'Tester'
+            if 'MEM' in field:
+                parts = field.split(',')
+                for part in parts[:]:
+                    if part.startswith('MEM'):
+                        parts.remove(part)
+                field = ','.join(parts)
+            self.row[idx] = field
+
+    def __str__(self):
+        return 'Game {} {} AI={} commit={}({}) py={} {}'.format(
+            self.game, self.ruleset, self.aiVariant, self.commit, self.commitDate, self.py_version, self.tags)

src/kajonggtest.py

@@ -23,19 +23,11 @@ from optparse import OptionParser
 from locale import getdefaultlocale
 
 from common import Debug, StrMixin, cacheDir
-from util import removeIfExists, gitHead, checkMemory
-from util import Csv, CsvWriter, popenReadlines
+from util import removeIfExists, gitHead, checkMemory, popenReadlines
+from kajcsv import Csv, CsvRow, CsvWriter
 
 signal.signal(signal.SIGINT, signal.SIG_DFL)
 
-# fields in row:
-RULESETFIELD = 0
-AIFIELD = 1
-COMMITFIELD = 2
-PYTHON23FIELD = 3
-GAMEFIELD = 4
-TAGSFIELD = 5
-PLAYERSFIELD = 6
 
 OPTIONS = None
 
@@ -265,7 +257,7 @@ class Job(StrMixin):
         cmd.insert(0, 'python{}'.format(self.pythonVersion))
         if OPTIONS.rounds:
             cmd.append('--rounds={rounds}'.format(rounds=OPTIONS.rounds))
-        if self.aiVariant != 'Default':
+        if self.aiVariant != 'DefaultAI':
             cmd.append('--ai={ai}'.format(ai=self.aiVariant))
         if OPTIONS.csv:
             cmd.append('--csv={csv}'.format(csv=OPTIONS.csv))
@@ -318,184 +310,143 @@ class Job(StrMixin):
         game = 'game={}'.format(self.game)
         ruleset = self.shortRulesetName()
         aiName = 'AI={}'.format(
-            self.aiVariant) if self.aiVariant != 'Default' else ''
+            self.aiVariant) if self.aiVariant != 'DefaultAI' else ''
         return ' '.join([
             self.commitId, 'Python{}'.format(self.pythonVersion), pid, game, ruleset, aiName]).replace('  ', ' ')
 
 
-def neutralize(rows):
-    """remove things we do not want to compare"""
-    for row in rows:
-        for idx, field in enumerate(row):
-            field = field.replace(' ', '')
-            if field.startswith('Tester ') or field.startswith('Tüster'):
-                field = 'Tester'
-            if 'MEM' in field:
-                parts = field.split(',')
-                for part in parts[:]:
-                    if part.startswith('MEM'):
-                        parts.remove(part)
-                field = ','.join(parts)
-            row[idx] = field
-        yield row
-
-
-def onlyExistingCommits(commits):
-    """filter out non-existing commits"""
-    global KNOWNCOMMITS  # pylint: disable=global-statement
-    if not KNOWNCOMMITS:
-        for branch in subprocess.check_output(b'git branch'.split()).decode().split('\n'):
-            if 'detached' not in branch and 'no branch' not in branch:
-                KNOWNCOMMITS |= set(subprocess.check_output(
-                    'git log --max-count=200 --pretty=%H {branch}'.format(
-                        branch=branch[2:]).split()).decode().split('\n'))
-    result = list()
-    for commit in commits:
-        if any(x.startswith(commit) for x in KNOWNCOMMITS):
-            result.append(commit)
-    return result
-
-
-def removeInvalidCommits(csvFile):
-    """remove rows with invalid git commit ids"""
-    if not os.path.exists(csvFile):
-        return
-    rows = list(Csv.reader(csvFile))
-    _ = {x[COMMITFIELD] for x in rows}
-    csvCommits = {
-        x for x in _ if set(
-            x) <= set(
-                '0123456789abcdef') and len(
-                    x) >= 7}
-    nonExisting = set(csvCommits) - set(onlyExistingCommits(csvCommits))
-    if nonExisting:
-        print(
-            'removing rows from kajongg.csv for commits %s' %
-            ','.join(nonExisting))
-        writer = CsvWriter(csvFile)
-        for row in rows:
-            if row[COMMITFIELD] not in nonExisting:
-                writer.writerow(row)
-    # remove all logs referencing obsolete commits
+
+
+def cleanup_data(csv):
+    """remove all data referencing obsolete commits"""
     logDir = os.path.expanduser(os.path.join('~', '.kajongg', 'log'))
+    knownCommits = csv.commits()
     for dirName, _, fileNames in os.walk(logDir):
         for fileName in fileNames:
-            if fileName not in KNOWNCOMMITS and fileName != 'current':
+            if fileName not in knownCommits and fileName != 'current':
                 os.remove(os.path.join(dirName, fileName))
         try:
             os.removedirs(dirName)
         except OSError:
             pass  # not yet empty
-
     Clone.removeObsolete()
 
-def readGames(csvFile):
-    """return a dict holding a frozenset of games for each variant"""
-    if not os.path.exists(csvFile):
-        return
-    allRowsGenerator = neutralize(Csv.reader(csvFile))
-    if not allRowsGenerator:
-        return
-    # we want unique tuples so we can work with sets
-    allRows = {tuple(x) for x in allRowsGenerator}
-    games = dict()
-    # build set of rows for every ai
-    for variant in {tuple(x[:COMMITFIELD]) for x in allRows}:
-        games[variant] = frozenset(
-            x for x in allRows if tuple(x[:COMMITFIELD]) == variant)
-    return games
-
-def hasDifferences(rows):
-    """True if rows have unwanted differences"""
-    return (len({tuple(list(x)[GAMEFIELD:]) for x in rows})
-            > len({tuple(list(x)[:COMMITFIELD]) for x in rows}))
-
-def firstDifference(rows):
-    """reduce to two rows showing a difference"""
-    result = rows
-    last = rows[-1]
-    while hasDifferences(result):
-        last = result[-1]
-        result = result[:-1]
-    return list([result[-1], last])
-
-def closerLook(gameId, gameIdRows):
-    """print detailled info about one difference"""
-    for ruleset in OPTIONS.rulesets:
-        for intelligence in OPTIONS.allAis:
-            shouldBeIdentical = [x for x in gameIdRows if x[RULESETFIELD] == ruleset and x[AIFIELD] == intelligence]
-            for commit in (x[COMMITFIELD] for x in shouldBeIdentical):
-                rows2 = [x for x in shouldBeIdentical if x[COMMITFIELD] == commit]
-                if hasDifferences(rows2):
-                    first = firstDifference(rows2)
-                    print('Game {} {} {} {} has differences between Python2 and Python3'.format(
-                        gameId, ruleset, intelligence, commit))
-            for py23 in '23':
-                rows2 = [x for x in shouldBeIdentical if x[PYTHON23FIELD] == py23]
-                if hasDifferences(rows2):
-                    first = firstDifference(rows2)
-                    print('Game {} {} {} Python{} has differences between commits {} and {}'.format(
-                        gameId, ruleset, intelligence, py23, first[0][COMMITFIELD], first[1][COMMITFIELD]))
-
-def printDifferingResults(rowLists):
-    """if most games get the same result with all tried variants,
-    dump those games that do not"""
-    allGameIds = {}
-    for rows in rowLists:
-        for row in rows:
-            rowId = row[GAMEFIELD]
-            if rowId not in allGameIds:
-                allGameIds[rowId] = []
-            allGameIds[rowId].append(row)
-    differing = []
-    for key, value in allGameIds.items():
-        if hasDifferences(value):
-            differing.append(key)
-    if not differing:
-        print('no games differ')
-    else:
-        print(
-            'differing games (%d out of %d): %s' % (
-                len(differing), len(allGameIds),
-                ' '.join(sorted(differing, key=int))))
-        # now look closer at one example. Differences may be caused by git commits or by py2/p3
-        for gameId in sorted(differing):
-            closerLook(gameId, allGameIds[gameId])
-
-def evaluate(games):
-    """evaluate games"""
-    if not games:
-        return
-    for variant, rows in games.items():
-        gameIds = {x[GAMEFIELD] for x in rows}
-        if len(gameIds) != len({tuple(list(x)[GAMEFIELD:]) for x in rows}):
-            print(
-                'ruleset "%s" AI "%s" has different rows for games' %
-                (variant[0], variant[1]), end=' ')
-            for game in sorted(gameIds, key=int):
-                if len({tuple(x[GAMEFIELD:] for x in rows if x[GAMEFIELD] == game)}) > 1:
-                    print(game, end=' ')
-            print()
-            break
-    printDifferingResults(games.values())
-    print()
-    print('the 3 robot players always use the Default AI')
-    print()
-    print('{ruleset:<25} {ai:<20} {games:>5}     {points:>4}                      human'.format(
-        ruleset='Ruleset', ai='AI variant', games='games', points='points'))
-    for variant, rows in games.items():
-        ruleset, aiVariant = variant
-        print('{ruleset:<25} {ai:<20} {rows:>5}  '.format(
-            ruleset=ruleset[:25], ai=aiVariant[:20],
-            rows=len(rows)), end=' ')
-        for playerIdx in range(4):
+def pairs(data):
+    """return all consecutive pairs"""
+    prev = None
+    for _ in data:
+        if prev:
+            yield prev, _
+        prev = _
+
+
+class CSV(StrMixin):
+    """represent kajongg.csv"""
+
+    knownCommits = None
+
+    def __init__(self):
+        self.findKnownCommits()
+        self.rows = []
+        if os.path.exists(OPTIONS.csv):
+            self.rows = list(sorted({CsvRow(x) for x in Csv.reader(OPTIONS.csv)}))
+        self.removeInvalidCommits()
+
+    def neutralize(self):
+        """remove things we do not want to compare"""
+        for row in self.rows:
+            row.neutralize()
+
+    def commits(self):
+        """return set of all our commit ids"""
+        # TODO: sorted by date
+        return {x.commit for x in self.rows}
+
+    def games(self):
+        """return a sorted unique list of all games"""
+        return sorted({x.game for x in self.rows})
+
+    @classmethod
+    def findKnownCommits(cls):
+        """find known commits"""
+        if cls.knownCommits is None:
+            cls.knownCommits = set()
+            for branch in subprocess.check_output(b'git branch'.split()).decode().split('\n'):
+                if 'detached' not in branch and 'no branch' not in branch:
+                    cls.knownCommits |= set(subprocess.check_output(
+                        'git log --max-count=400 --pretty=%H {branch}'.format(
+                            branch=branch[2:]).split()).decode().split('\n'))
+
+    @classmethod
+    def onlyExistingCommits(cls, commits):
+        """return a set with only  existing commits"""
+        result = set()
+        for commit in commits:
+            if any(x.startswith(commit) for x in cls.knownCommits):
+                result.add(commit)
+        return result
+
+    def removeInvalidCommits(self):
+        """remove rows with invalid git commit ids"""
+        csvCommits = {x.commit for x in self.rows}
+        csvCommits = {
+            x for x in csvCommits if set(
+                x) <= set(
+                    '0123456789abcdef') and len(
+                        x) >= 7}
+        nonExisting = csvCommits - self.onlyExistingCommits(set(x.commit for x in self.rows))
+        if nonExisting:
             print(
-                '{p:>8}'.format(
-                    p=sum(
-                        int(x[
-                            PLAYERSFIELD + 1 + playerIdx * 4]) for x in rows)),
-                end=' ')
-        print()
+                'removing rows from kajongg.csv for commits %s' %
+                ','.join(nonExisting))
+            self.rows = [x for x in self.rows if x.commit not in nonExisting]
+            self.write()
+
+    def write(self):
+        """write new csv file"""
+        writer = CsvWriter(OPTIONS.csv)
+        for row in self.rows: