# -*- coding: utf-8 -*-
"""
Created on Fri Apr  8 16:36:26 2011

@author: ProfMobius, Searge, Fesh0r
@version: v1.2
"""

import sys
import fnmatch
import logging
import os
import shutil
import zipfile
import csv
import re
import subprocess
import ConfigParser
import urllib
import time
import stat
import errno
import shlex
import platform
from hashlib import md5  # pylint: disable-msg=E0611
from contextlib import closing
from textwrap import TextWrapper

from filehandling.srgsexport import writesrgsfromcsvs
from filehandling.srgshandler import parse_srg
from pylibs.annotate_gl_constants import annotate_file
from pylibs.whereis import whereis
from pylibs.jadfix import jadfix
from pylibs.fffix import fffix
from pylibs.cleanup_src import strip_comments, src_cleanup
from pylibs.normlines import normaliselines
from pylibs.normpatch import normalisepatch


CLIENT = 0
SERVER = 1
SIDE_NAME = {CLIENT: 'client', SERVER: 'server'}


class Error(Exception):
    pass


class CalledProcessError(Error):
    def __init__(self, returncode, cmd, output=None):
        super(CalledProcessError, self).__init__()
        self.returncode = returncode
        self.cmd = cmd
        self.output = output

    def __str__(self):
        return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)


def reallyrmtree(path):
    if not sys.platform.startswith('win'):
        if os.path.exists(path):
            shutil.rmtree(path)
    else:
        i = 0
        try:
            while os.stat(path) and i < 20:
                shutil.rmtree(path, onerror=rmtree_onerror)
                i += 1
        except OSError:
            pass

        # raise OSError if the path still exists even after trying really hard
        try:
            os.stat(path)
        except OSError:
            pass
        else:
            raise OSError(errno.EPERM, "Failed to remove: '" + path + "'", path)


def rmtree_onerror(func, path, _):
    if not os.access(path, os.W_OK):
        os.chmod(path, stat.S_IWUSR)
    time.sleep(0.5)
    try:
        func(path)
    except OSError:
        pass


def filterdirs(src_dir, pattern=None, ignore_dirs=None, append_pattern=False, all_files=False):
    """Return list of subdirs containing given file pattern filtering out ignored dirs"""
    # avoid problems with mutable default parameters
    if ignore_dirs is None:
        ignore_dirs = []
    dirs = []
    for path, dirlist, filelist in os.walk(src_dir, followlinks=True):
        sub_dir = os.path.relpath(path, src_dir)
        test_dirlist = dirlist[:]
        for cur_dir in test_dirlist:
            if os.path.normpath(os.path.join(sub_dir, cur_dir)) in ignore_dirs:
                # if the full subdir is in the ignored package list delete it so that we don't descend into it
                dirlist.remove(cur_dir)
        if pattern is None:
            if all_files:
                dirs.extend([os.path.join(path, f) for f in filelist])
            else:
                dirs.append(path)
        else:
            files = fnmatch.filter(filelist, pattern)
            if files:
                if all_files:
                    dirs.extend([os.path.join(path, f) for f in files])
                elif append_pattern:
                    dirs.append(os.path.join(path, pattern))
                else:
                    dirs.append(path)
    return dirs


def cmdsplit(args):
    if os.sep == '\\':
        args = args.replace('\\', '\\\\')
    return shlex.split(args)


def truncate(text, length):
    if len(text) > length:
        return text[:length] + '...'
    return text


def csv_header(csvfile):
    fieldnames = []
    if os.path.isfile(csvfile):
        with open(csvfile, 'rb') as fh:
            csvreader = csv.DictReader(fh)
            fieldnames = csvreader.fieldnames
    return set(fieldnames)


class Commands(object):
    """Contains the commands and initialisation for a full mcp run"""

    MCPVersion = '7.51'
    _default_config = 'conf/mcp.cfg'
    _version_config = 'conf/version.cfg'

    @classmethod
    def fullversion(cls):
        """Read the version configuration file and return a full version string"""
        full_version = None
        try:
            config = ConfigParser.SafeConfigParser()
            with open(os.path.normpath(cls._version_config)) as fh:
                config.readfp(fh)
            data_version = config.get('VERSION', 'MCPVersion')
            client_version = config.get('VERSION', 'ClientVersion')
            server_version = config.get('VERSION', 'ServerVersion')
            full_version = ' (data: %s, client: %s, server: %s)' % (data_version, client_version, server_version)
        except IOError:
            pass
        except ConfigParser.Error:
            pass

        if full_version is None:
            return cls.MCPVersion
        else:
            return cls.MCPVersion + full_version

    def __init__(self, conffile=None, verify=False, no_patch=False):
        self.conffile = conffile
        self.readconf()
        self.checkfolders()
        self.startlogger()
        self.logger.info('== MCP %s ==', Commands.fullversion())

        if sys.platform.startswith('linux'):
            self.osname = 'linux'
        elif sys.platform.startswith('darwin'):
            self.osname = 'osx'
        elif sys.platform.startswith('win'):
            self.osname = 'win'
        else:
            self.logger.error('OS not supported : %s', sys.platform)
            sys.exit(1)
        self.logger.debug('OS : %s', sys.platform)

        # tell off people running as root as it screws up wine
        if self.osname in ['linux', 'osx']:
            if not os.getuid():  # pylint: disable-msg=E1101
                self.logger.error("!! Please don't run MCP as root !!")
                sys.exit(1)

        self.checkjava()
        self.readcommands(verify, no_patch=no_patch)

    def checkcommand(self, name, command, java=False, single_line=False, check_return=True, error=True):
        self.logger.debug("# %s: '%s'", name, command)
        try:
            if java:
                command = '%s -jar %s' % (self.cmdjava, command)
            output = self.runcmd(command, quiet=True, check_return=check_return)
            if single_line:
                output = output.splitlines()[0]
            output = output.strip()
            self.logger.debug(output)
            return True
        except OSError as ex:
            if error:
                self.logger.error('!! %s check FAILED !!', name)
                self.logger.error(ex)
                sys.exit(1)
        except CalledProcessError as ex:
            output = ex.output
            output = output.strip()
            self.logger.debug(output)
            if error:
                self.logger.error('!! %s check FAILED !!', name)
                self.logger.error(ex)
                sys.exit(1)
        return False

    def readcommands(self, verify=False, no_patch=False):
        if verify:
            self.logger.debug('# VERSION INFO')
            self.logger.debug('python: %s', sys.version)
            self.logger.debug('platform: %s', platform.platform())
            self.checkcommand('java', '%s -version' % self.cmdjava)
            self.checkcommand('javac', '%s -version' % self.cmdjavac)
            self.checkcommand('javac runtime', '%s -J-version' % self.cmdjavac)
            self.checkcommand('retroguard', '%s --version' % self.retroguard, java=True)

        self.exceptor = os.path.normpath(self.config.get('COMMANDS', 'Exceptor'))
        if verify:
            self.checkcommand('mcinjector', '%s --version' % self.exceptor, java=True)

        # verify below along with jad if required
        self.jadretro = os.path.normpath(self.config.get('COMMANDS', 'JadRetro'))

        self.patcher = os.path.normpath(self.config.get('COMMANDS', 'Patcher_%s' % self.osname))
        if verify:
            self.checkcommand('patch', '%s --version' % self.patcher, single_line=True)

        self.has_wine = False
        self.has_jad = False
        self.has_ff = False
        self.has_astyle = False
        if self.osname in ['linux']:
            self.wine = os.path.normpath(self.config.get('COMMANDS', 'Wine'))
            if verify:
                self.has_wine = self.checkcommand('wine', '%s --version' % self.wine, error=False)
            self.astyle = os.path.normpath(self.config.get('COMMANDS', 'AStyle_linux'))
            if verify:
                self.has_astyle = self.checkcommand('astyle', '%s --version' % self.astyle, error=False)
            if self.has_wine:
                self.jad = self.wine + ' ' + os.path.normpath(self.config.get('COMMANDS', 'Jad_win'))
                if not self.has_astyle:
                    self.astyle = self.wine + ' ' + os.path.normpath(self.config.get('COMMANDS', 'AStyle_win'))
                    if verify:
                        self.has_astyle = self.checkcommand('astyle', '%s --version' % self.astyle, error=False)
            else:
                # need to set to string so the below CmdJad stuff doesn't error out
                self.jad = ''
        else:
            self.jad = os.path.normpath(self.config.get('COMMANDS', 'Jad_%s' % self.osname))
            self.astyle = os.path.normpath(self.config.get('COMMANDS', 'AStyle_%s' % self.osname))
            if verify:
                self.has_astyle = self.checkcommand('astyle', '%s --version' % self.astyle, error=False)

        # only check jad and jadretro if we can use it
        if self.jad:
            if verify:
                has_jadretro = self.checkcommand('jadretro', '%s' % self.jadretro, java=True, single_line=True,
                                                 error=False)
                has_jad = self.checkcommand('jad', '%s' % self.jad, single_line=True, check_return=False,
                                            error=False)
                self.has_jad = has_jad and has_jadretro

        self.fernflower = os.path.normpath(self.config.get('COMMANDS', 'Fernflower'))
        if verify:
            self.has_ff = self.checkcommand('fernflower', '%s' % self.fernflower, java=True, single_line=True,
                                            error=False)

        # check requirements
        # windows: all requirements supplied
        # osx: require python and patch, will error out before now anyway so don't need to check further
        # linux: require python, patch and either wine for jad or fernflower, and optionally astyle if wine not present
        if verify:
            reqs = []
            if self.osname in ['linux']:
                if not self.has_wine:
                    if not self.has_ff:
                        self.logger.error('!! Please install either wine or fernflower for decompilation !!')
                        sys.exit(1)
                    if not self.has_astyle:
                        self.logger.error('!! Please install either wine or astyle for source cleanup !!')
                else:
                    reqs.append('wine')
            if self.has_jad:
                reqs.append('jad')
            if self.has_ff:
                reqs.append('ff')
            if self.has_jad_patch:
                reqs.append('jad patches')
            if self.has_ff_patch:
                reqs.append('ff patches')
            if self.has_osx_patch:
                reqs.append('osx patches')
            if self.has_srg:
                reqs.append('srgs')
            if self.has_map_csv:
                reqs.append('map csvs')
            if self.has_name_csv:
                reqs.append('name csvs')
            if self.has_doc_csv:
                reqs.append('doc csvs')
            if self.has_param_csv:
                reqs.append('param csvs')
            if self.has_renumber_csv:
                reqs.append('renumber csv')
            if self.has_astyle:
                reqs.append('astyle')
            if self.has_astyle_cfg:
                reqs.append('astyle config')
            self.logger.info('# found %s', ', '.join(reqs))
            if not self.has_jad_patch and not no_patch:
                self.has_jad = False
            if not self.has_ff_patch and not no_patch:
                self.has_ff = False
            if not self.has_astyle_cfg:
                self.has_astyle = False
            if not self.has_map_csv and not self.has_srg:
                self.logger.error('!! need either srgs or csvs !!')
                sys.exit(1)
            if not self.has_exc:
                self.logger.error('!! need mcinjector configs !!')
                sys.exit(1)
            if not self.has_jad and not self.has_ff:
                self.logger.error('!! need either jad or fernflower available with patches !!')
                sys.exit(1)

        self.cmdpatch = self.config.get('COMMANDS', 'CmdPatch', raw=1) % self.patcher
        self.cmdjad = self.config.get('COMMANDS', 'CmdJad', raw=1) % self.jad
        self.cmdastyle = self.config.get('COMMANDS', 'CmdAStyle', raw=1) % self.astyle
        self.cmdrg = self.config.get('COMMANDS', 'CmdRG', raw=1) % self.cmdjava
        self.cmdrgreobf = self.config.get('COMMANDS', 'CmdRGReobf', raw=1) % self.cmdjava
        self.cmdjadretro = self.config.get('COMMANDS', 'CmdJadretro', raw=1) % (self.cmdjava, self.jadretro)
        self.cmdfernflower = self.config.get('COMMANDS', 'CmdFernflower', raw=1) % (self.cmdjava, self.fernflower)
        self.cmdexceptor = self.config.get('COMMANDS', 'CmdExceptor', raw=1) % (self.cmdjava, self.exceptor)
        self.cmdrecomp = self.config.get('COMMANDS', 'CmdRecomp', raw=1) % self.cmdjavac
        self.cmdstartsrv = self.config.get('COMMANDS', 'CmdStartSrv', raw=1) % self.cmdjava
        self.cmdstartclt = self.config.get('COMMANDS', 'CmdStartClt', raw=1) % self.cmdjava

    def startlogger(self):
        self.logger = logging.getLogger()
        self.logger.setLevel(logging.DEBUG)
        # create file handler which logs even debug messages
        filehandler = logging.FileHandler(filename=self.mcplogfile)
        filehandler.setLevel(logging.DEBUG)
        # create console handler with a higher log level
        consolehandler = logging.StreamHandler()
        consolehandler.setLevel(logging.INFO)
        # File output of everything Warning or above
        errorhandler = logging.FileHandler(filename=self.mcperrlogfile)
        errorhandler.setLevel(logging.WARNING)
        # create formatter and add it to the handlers
        formatterconsole = logging.Formatter('%(message)s')
        consolehandler.setFormatter(formatterconsole)
        formatterfile = logging.Formatter('%(asctime)s - %(message)s', datefmt='%H:%M:%S')
        filehandler.setFormatter(formatterfile)
        errorhandler.setFormatter(formatterfile)
        # add the handlers to logger
        self.logger.addHandler(consolehandler)
        self.logger.addHandler(filehandler)
        self.logger.addHandler(errorhandler)

        # HINT: SECONDARY LOGGER FOR CLIENT & SERVER
        self.loggermc = logging.getLogger('MCRunLog')
        self.loggermc.setLevel(logging.DEBUG)
        chmc = logging.StreamHandler()
        chmc.setLevel(logging.DEBUG)
        formatterconsolemc = logging.Formatter('[%(asctime)s] %(message)s', datefmt='%H:%M:%S')
        chmc.setFormatter(formatterconsolemc)
        # add the handlers to logger
        self.loggermc.addHandler(chmc)

    def readconf(self):
        """Read the configuration file to setup some basic paths"""
        config = ConfigParser.SafeConfigParser()
        try:
            with open(os.path.normpath(self._default_config)) as fh:
                config.readfp(fh)
            if self.conffile is not None:
                config.read(os.path.normpath(self.conffile))
        except IOError:
            logging.error('!! Missing mcp.cfg !!')
            sys.exit(1)
        self.config = config

        # HINT: We read the directories for cleanup
        self.dirtemp = os.path.normpath(config.get('DEFAULT', 'DirTemp'))
        self.dirsrc = os.path.normpath(config.get('DEFAULT', 'DirSrc'))
        self.dirlogs = os.path.normpath(config.get('DEFAULT', 'DirLogs'))
        self.dirbin = os.path.normpath(config.get('DEFAULT', 'DirBin'))
        self.dirjars = os.path.normpath(config.get('DEFAULT', 'DirJars'))
        self.dirreobf = os.path.normpath(config.get('DEFAULT', 'DirReobf'))
        self.dirlib = os.path.normpath(config.get('DEFAULT', 'DirLib'))
        self.dirmodsrc = os.path.normpath(config.get('DEFAULT', 'DirModSrc'))

        # HINT: We read the position of the CSV files
        self.csvclasses = os.path.normpath(config.get('CSV', 'Classes'))
        self.csvmethods = os.path.normpath(config.get('CSV', 'Methods'))
        self.csvfields = os.path.normpath(config.get('CSV', 'Fields'))
        self.csvparams = os.path.normpath(config.get('CSV', 'Params'))
        self.csvnewids = os.path.normpath(config.get('CSV', 'NewIds'))

        # check what csvs we have
        self.has_map_csv = False
        self.has_name_csv = False
        self.has_doc_csv = False
        self.has_param_csv = False
        self.has_renumber_csv = False
        header_classes = csv_header(self.csvclasses)
        header_methods = csv_header(self.csvmethods)
        header_fields = csv_header(self.csvfields)
        header_params = csv_header(self.csvparams)
        header_newids = csv_header(self.csvnewids)
        if set(['notch', 'name', 'package', 'side']) <= header_classes and \
           set(['notch', 'searge', 'notchsig', 'sig', 'classnotch', 'classname', 'package', 'side']) <= header_methods and \
           set(['notch', 'searge', 'classnotch', 'classname', 'package', 'side']) <= header_fields:
            self.has_map_csv = True
        if header_methods >= set(['searge', 'name', 'side']) <= header_fields:
            self.has_name_csv = True
        if header_methods >= set(['searge', 'name', 'desc', 'side']) <= header_fields:
            self.has_doc_csv = True
        if set(['param', 'name', 'side']) <= header_params:
            self.has_param_csv = True
        if set(['client', 'server', 'newid']) <= header_newids:
            self.has_renumber_csv = True

        # HINT: We read the names of the SRG output
        self.srgsconfclient = os.path.normpath(config.get('SRGS', 'ConfClient'))
        self.srgsconfserver = os.path.normpath(config.get('SRGS', 'ConfServer'))
        self.srgsclient = os.path.normpath(config.get('SRGS', 'Client'))
        self.srgsserver = os.path.normpath(config.get('SRGS', 'Server'))
        self.deobsrgclient = os.path.normpath(config.get('SRGS', 'DeobfClient'))
        self.deobsrgserver = os.path.normpath(config.get('SRGS', 'DeobfServer'))
        self.reobsrgclient = os.path.normpath(config.get('SRGS', 'ReobfClient'))
        self.reobsrgserver = os.path.normpath(config.get('SRGS', 'ReobfServer'))
        self.reobsrgclientsrg = os.path.normpath(config.get('SRGS', 'ReobfClientSrg'))
        self.reobsrgserversrg = os.path.normpath(config.get('SRGS', 'ReobfServerSrg'))

        # do we have full srg files
        self.has_srg = False
        if os.path.isfile(self.srgsconfclient) and os.path.isfile(self.srgsconfserver):
            self.has_srg = True

        # HINT: We read the position of the jar files
        self.dirnatives = os.path.normpath(config.get('JAR', 'DirNatives'))
        self.jarclient = os.path.normpath(config.get('JAR', 'Client'))
        self.jarserver = os.path.normpath(config.get('JAR', 'Server'))
        self.md5jarclt = config.get('JAR', 'MD5Client').lower()
        self.md5jarsrv = config.get('JAR', 'MD5Server').lower()
        jarslwjgl = config.get('JAR', 'LWJGL').split(',')
        self.jarslwjgl = [os.path.normpath(p) for p in jarslwjgl]

        # HINT: We read keys relevant to retroguard
        self.retroguard = os.path.normpath(config.get('RETROGUARD', 'Location'))
        self.rgconfig = os.path.normpath(config.get('RETROGUARD', 'RetroConf'))
        self.rgclientconf = os.path.normpath(config.get('RETROGUARD', 'ClientConf'))
        self.rgserverconf = os.path.normpath(config.get('RETROGUARD', 'ServerConf'))
        self.rgclientout = os.path.normpath(config.get('RETROGUARD', 'ClientOut'))
        self.rgserverout = os.path.normpath(config.get('RETROGUARD', 'ServerOut'))
        self.rgclientlog = os.path.normpath(config.get('RETROGUARD', 'ClientLog'))
        self.rgserverlog = os.path.normpath(config.get('RETROGUARD', 'ServerLog'))
        self.rgclientdeoblog = os.path.normpath(config.get('RETROGUARD', 'ClientDeobLog'))
        self.rgserverdeoblog = os.path.normpath(config.get('RETROGUARD', 'ServerDeobLog'))
        self.rgreobconfig = os.path.normpath(config.get('RETROGUARD', 'RetroReobConf'))
        self.rgclientreobconf = os.path.normpath(config.get('RETROGUARD', 'ClientReobConf'))
        self.rgserverreobconf = os.path.normpath(config.get('RETROGUARD', 'ServerReobConf'))
        self.nullpkg = config.get('RETROGUARD', 'NullPkg')

        # HINT: We read keys relevant to exceptor
        self.xclientconf = os.path.normpath(config.get('EXCEPTOR', 'XClientCfg'))
        self.xserverconf = os.path.normpath(config.get('EXCEPTOR', 'XServerCfg'))
        self.xclientout = os.path.normpath(config.get('EXCEPTOR', 'XClientOut'))
        self.xserverout = os.path.normpath(config.get('EXCEPTOR', 'XServerOut'))
        self.xclientlog = os.path.normpath(config.get('EXCEPTOR', 'XClientLog'))
        self.xserverlog = os.path.normpath(config.get('EXCEPTOR', 'XServerLog'))

        # do we have the exc files
        self.has_exc = False
        if os.path.isfile(self.xclientconf) and os.path.isfile(self.xserverconf):
            self.has_exc = True

        # HINT: We read keys relevant to decompilation
        self.srcclienttmp = os.path.normpath(config.get('DECOMPILE', 'SrcClientTemp'))
        self.srcservertmp = os.path.normpath(config.get('DECOMPILE', 'SrcServerTemp'))
        self.clsclienttmp = os.path.normpath(config.get('DECOMPILE', 'ClsClientTemp'))
        self.clsservertmp = os.path.normpath(config.get('DECOMPILE', 'ClsServerTemp'))
        self.ffsource = config.get('DECOMPILE', 'FFSource')

        # HINT: We read the output directories
        self.binclienttmp = os.path.normpath(config.get('OUTPUT', 'BinClientTemp'))
        self.binservertmp = os.path.normpath(config.get('OUTPUT', 'BinServerTemp'))
        self.srcclient = os.path.normpath(config.get('OUTPUT', 'SrcClient'))
        self.srcserver = os.path.normpath(config.get('OUTPUT', 'SrcServer'))
        self.testclient = config.get('OUTPUT', 'TestClient')
        self.testserver = config.get('OUTPUT', 'TestServer')

        # HINT: Patcher related configs
        self.patchclient = os.path.normpath(config.get('PATCHES', 'PatchClient'))
        self.patchserver = os.path.normpath(config.get('PATCHES', 'PatchServer'))
        self.patchtemp = os.path.normpath(config.get('PATCHES', 'PatchTemp'))
        self.ffpatchclient = os.path.normpath(config.get('PATCHES', 'FFPatchClient'))
        self.ffpatchserver = os.path.normpath(config.get('PATCHES', 'FFPatchServer'))
        self.patchclient_osx = os.path.normpath(config.get('PATCHES', 'PatchClient_osx'))
        self.patchserver_osx = os.path.normpath(config.get('PATCHES', 'PatchServer_osx'))

        # check what patches we have
        self.has_jad_patch = False
        if os.path.isfile(self.patchclient) and os.path.isfile(self.patchserver):
            self.has_jad_patch = True
        self.has_ff_patch = False
        if os.path.isfile(self.ffpatchclient) and os.path.isfile(self.ffpatchserver):
            self.has_ff_patch = True
        self.has_osx_patch = False
        if os.path.isfile(self.patchclient_osx) and os.path.isfile(self.patchserver_osx):
            self.has_osx_patch = True

        # HINT: Recompilation related configs
        self.binclient = os.path.normpath(config.get('RECOMPILE', 'BinClient'))
        self.binserver = os.path.normpath(config.get('RECOMPILE', 'BinServer'))
        self.clientrecomplog = os.path.normpath(config.get('RECOMPILE', 'LogClient'))
        self.serverrecomplog = os.path.normpath(config.get('RECOMPILE', 'LogServer'))
        cpathclient = config.get('RECOMPILE', 'ClassPathClient').split(',')
        self.cpathclient = [os.path.normpath(p) for p in cpathclient]
        self.fixesclient = os.path.normpath(config.get('RECOMPILE', 'ClientFixes'))
        cpathserver = config.get('RECOMPILE', 'ClassPathServer').split(',')
        self.cpathserver = [os.path.normpath(p) for p in cpathserver]
        if config.has_option('RECOMPILE', 'FixSound'):
            self.fixsound = config.get('RECOMPILE', 'FixSound')
        else:
            self.fixsound = None
        self.fixstart = config.get('RECOMPILE', 'FixStart')
        self.ignorepkg = config.get('RECOMPILE', 'IgnorePkg').split(',')

        # HINT: Reobf related configs
        self.md5client = os.path.normpath(config.get('REOBF', 'MD5Client'))
        self.md5server = os.path.normpath(config.get('REOBF', 'MD5Server'))
        self.md5reobfclient = os.path.normpath(config.get('REOBF', 'MD5PreReobfClient'))
        self.md5reobfserver = os.path.normpath(config.get('REOBF', 'MD5PreReobfServer'))
        self.rgclientrolog = os.path.normpath(config.get('REOBF', 'ClientRoLog'))
        self.rgserverrolog = os.path.normpath(config.get('REOBF', 'ServerRoLog'))
        self.cmpjarclient = os.path.normpath(config.get('REOBF', 'RecompJarClient'))
        self.cmpjarserver = os.path.normpath(config.get('REOBF', 'RecompJarServer'))
        self.reobfjarclient = os.path.normpath(config.get('REOBF', 'ObfJarClient'))
        self.reobfjarserver = os.path.normpath(config.get('REOBF', 'ObfJarServer'))
        self.dirreobfclt = os.path.normpath(config.get('REOBF', 'ReobfDirClient'))
        self.dirreobfsrv = os.path.normpath(config.get('REOBF', 'ReobfDirServer'))
        self.clientreoblog = os.path.normpath(config.get('REOBF', 'ReobfClientLog'))
        self.serverreoblog = os.path.normpath(config.get('REOBF', 'ReobfServerLog'))

        self.mcplogfile = os.path.normpath(config.get('MCP', 'LogFile'))
        self.mcperrlogfile = os.path.normpath(config.get('MCP', 'LogFileErr'))
        if config.has_option('MCP', 'UpdateUrl'):
            updateurl = config.get('MCP', 'UpdateUrl')
            self.updateurl = updateurl.format(version=Commands.MCPVersion)
        else:
            self.updateurl = None
        ignoreupdate = config.get('MCP', 'IgnoreUpdate').split(',')
        self.ignoreupdate = [os.path.normpath(p) for p in ignoreupdate]
        self.mcprgindex = os.path.normpath(config.get('MCP', 'RGIndex'))
        self.mcpparamindex = os.path.normpath(config.get('MCP', 'ParamIndex'))

        # Get changed source
        self.srcmodclient = os.path.normpath(config.get('GETMODSOURCE', 'OutSRCClient'))
        self.srcmodserver = os.path.normpath(config.get('GETMODSOURCE', 'OutSRCServer'))

        # Source formatter
        self.astyleconf = os.path.normpath(config.get('ASTYLE', 'AstyleConfig'))

        # do we have a config for astyle
        self.has_astyle_cfg = False
        if os.path.isfile(self.astyleconf):
            self.has_astyle_cfg = True

    def creatergcfg(self, reobf=False, keep_lvt=False, keep_generics=False, rg_update=False, srg_names=False):
        """Create the files necessary for RetroGuard"""
        if reobf:
            rgconfig_file = self.rgreobconfig
            rgclientconf_file = self.rgclientreobconf
            rgserverconf_file = self.rgserverreobconf
        else:
            rgconfig_file = self.rgconfig
            rgclientconf_file = self.rgclientconf
            rgserverconf_file = self.rgserverconf

        with open(rgconfig_file, 'w') as rgout:
            rgout.write('.option Application\n')
            rgout.write('.option Applet\n')
            rgout.write('.option Repackage\n')
            rgout.write('.option Annotations\n')
            rgout.write('.option MapClassString\n')
            rgout.write('.attribute LineNumberTable\n')
            rgout.write('.attribute EnclosingMethod\n')
            rgout.write('.attribute Deprecated\n')
            if keep_lvt:
                # may cause issues trying to patch/recompile when decompiling mods
                rgout.write('.attribute LocalVariableTable\n')
            if keep_generics:
                # still not very reliable even with rg fixed
                rgout.write('.option Generic\n')
                rgout.write('.attribute LocalVariableTypeTable\n')
            if reobf:
                # this is obfuscated in vanilla and breaks the patches
                rgout.write('.attribute SourceFile\n')

        with open(rgclientconf_file, 'w') as rgout:
            rgout.write('%s = %s\n' % ('input', self.jarclient))
            rgout.write('%s = %s\n' % ('output', self.rgclientout))
            rgout.write('%s = %s\n' % ('reobinput', self.cmpjarclient))
            rgout.write('%s = %s\n' % ('reoboutput', self.reobfjarclient))
            if reobf:
                rgout.write('%s = %s\n' % ('script', self.rgreobconfig))
                rgout.write('%s = %s\n' % ('log', self.rgclientrolog))
            else:
                rgout.write('%s = %s\n' % ('script', self.rgconfig))
                rgout.write('%s = %s\n' % ('log', self.rgclientlog))
            rgout.write('%s = %s\n' % ('deob', self.srgsclient))
            if srg_names:
                rgout.write('%s = %s\n' % ('identifier', 'RGMCPSRG'))
                rgout.write('%s = %s\n' % ('reob', self.reobsrgclientsrg))
            else:
                rgout.write('%s = %s\n' % ('reob', self.reobsrgclient))
            rgout.write('%s = %s\n' % ('nplog', self.rgclientdeoblog))
            rgout.write('%s = %s\n' % ('rolog', self.clientreoblog))
            rgout.write('%s = %s\n' % ('verbose', '0'))
            rgout.write('%s = %s\n' % ('quiet', '1'))
            if rg_update:
                rgout.write('%s = %s\n' % ('fullmap', '1'))
                rgout.write('%s = %s\n' % ('startindex', self.mcprgindex))
            else:
                rgout.write('%s = %s\n' % ('fullmap', '0'))
                rgout.write('%s = %s\n' % ('startindex', '0'))
            for pkg in self.ignorepkg:
                rgout.write('%s = %s\n' % ('protectedpackage', pkg))

        with open(rgserverconf_file, 'w') as rgout:
            rgout.write('%s = %s\n' % ('startindex', '0'))
            rgout.write('%s = %s\n' % ('input', self.jarserver))
            rgout.write('%s = %s\n' % ('output', self.rgserverout))
            rgout.write('%s = %s\n' % ('reobinput', self.cmpjarserver))
            rgout.write('%s = %s\n' % ('reoboutput', self.reobfjarserver))
            if reobf:
                rgout.write('%s = %s\n' % ('script', self.rgreobconfig))
                rgout.write('%s = %s\n' % ('log', self.rgserverrolog))
            else:
                rgout.write('%s = %s\n' % ('script', self.rgconfig))
                rgout.write('%s = %s\n' % ('log', self.rgserverlog))
            rgout.write('%s = %s\n' % ('deob', self.srgsserver))
            if srg_names:
                rgout.write('%s = %s\n' % ('reob', self.reobsrgserversrg))
                rgout.write('%s = %s\n' % ('identifier', 'RGMCPSRG'))
            else:
                rgout.write('%s = %s\n' % ('reob', self.reobsrgserver))
            rgout.write('%s = %s\n' % ('nplog', self.rgserverdeoblog))
            rgout.write('%s = %s\n' % ('rolog', self.serverreoblog))
            rgout.write('%s = %s\n' % ('verbose', '0'))
            rgout.write('%s = %s\n' % ('quiet', '1'))
            if rg_update:
                rgout.write('%s = %s\n' % ('fullmap', '1'))
                rgout.write('%s = %s\n' % ('startindex', self.mcprgindex))
            else:
                rgout.write('%s = %s\n' % ('fullmap', '0'))
                rgout.write('%s = %s\n' % ('startindex', '0'))
            for pkg in self.ignorepkg:
                rgout.write('%s = %s\n' % ('protectedpackage', pkg))

    def createsrgs(self, side, use_srg=False):
        """Write the srgs files."""
        sidelk = {CLIENT: self.srgsclient, SERVER: self.srgsserver}
        srglk = {CLIENT: self.srgsconfclient, SERVER: self.srgsconfserver}

        if use_srg:
            if not self.has_srg:
                self.logger.error('!! srgs not found !!')
                sys.exit(1)
            shutil.copyfile(srglk[side], sidelk[side])
        else:
            if not self.has_map_csv:
                self.logger.error('!! csvs not found !!')
                sys.exit(1)
            fixes = [self.fixstart]
            if self.fixsound:
                fixes.append(self.fixsound)
            writesrgsfromcsvs(self.csvclasses, self.csvmethods, self.csvfields, sidelk[side],
                              side, fixes)

    def checkjava(self):
        """Check for java and setup the proper directory if needed"""
        results = []
        if self.osname == 'win':
            if not results:
                import _winreg

                for flag in [_winreg.KEY_WOW64_64KEY, _winreg.KEY_WOW64_32KEY]:
                    try:
                        k = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, r'Software\JavaSoft\Java Development Kit', 0,
                                            _winreg.KEY_READ | flag)
                        version, _ = _winreg.QueryValueEx(k, 'CurrentVersion')
                        k.Close()
                        k = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE,
                                            r'Software\JavaSoft\Java Development Kit\%s' % version, 0,
                                            _winreg.KEY_READ | flag)
                        path, _ = _winreg.QueryValueEx(k, 'JavaHome')
                        k.Close()
                        path = os.path.join(str(path), 'bin')
                        self.runcmd('"%s" -version' % os.path.join(path, 'javac'), quiet=True)
                        results.append(path)
                    except (CalledProcessError, OSError):
                        pass
            if not results:
                try:
                    self.runcmd('javac -version', quiet=True)
                    results.append('')
                except (CalledProcessError, OSError):
                    pass
            if not results and 'ProgramW6432' in os.environ:
                results.extend(whereis('javac.exe', os.environ['ProgramW6432']))
            if not results and 'ProgramFiles' in os.environ:
                results.extend(whereis('javac.exe', os.environ['ProgramFiles']))
            if not results and 'ProgramFiles(x86)' in os.environ:
                results.extend(whereis('javac.exe', os.environ['ProgramFiles(x86)']))
        elif self.osname in ['linux', 'osx']:
            if not results:
                try:
                    self.runcmd('javac -version', quiet=True)
                    results.append('')
                except (CalledProcessError, OSError):
                    pass
            if not results:
                results.extend(whereis('javac', '/usr/bin'))
            if not results:
                results.extend(whereis('javac', '/usr/local/bin'))
            if not results:
                results.extend(whereis('javac', '/opt'))
        if not results:
            self.logger.error('Java JDK is not installed ! Please install java JDK from http://java.oracle.com')
            sys.exit(1)
        self.cmdjavac = '"%s"' % os.path.join(results[0], 'javac')
        self.cmdjava = '"%s"' % os.path.join(results[0], 'java')

    def checkjars(self, side):
        jarlk = {CLIENT: self.jarclient, SERVER: self.jarserver}
        md5jarlk = {CLIENT: self.md5jarclt, SERVER: self.md5jarsrv}

        if not os.path.exists(jarlk[side]):
            return False
        with open(jarlk[side], 'rb') as fh:
            md5jar = md5(fh.read()).hexdigest()
        self.logger.debug('%s md5: %s', SIDE_NAME[side], md5jar)
        if md5jar != md5jarlk[side]:
            self.logger.warning('!! Modified jar detected. Unpredictable results !!')
        if side == CLIENT:
            fail = False
            for jar in self.jarslwjgl:
                if not os.path.exists(jar):
                    self.logger.error('!! %s not found !!' % jar)
                    fail = True
            if not os.path.exists(self.dirnatives):
                self.logger.error('!! %s not found !!' % self.dirnatives)
                fail = True
            if fail:
                self.logger.error('!! LWJGL check FAILED. Make sure to copy the entire .minecraft/bin folder into jars !!')
                sys.exit(1)
        return True

    def checksourcedir(self, side):
        srclk = {CLIENT: self.srcclient, SERVER: self.srcserver}
        srcdir = os.path.join(srclk[side], os.path.normpath(self.ffsource))

        if not os.path.exists(srcdir):
            return False
        return True

    def checksources(self, side):
        srclk = {CLIENT: self.srcclient, SERVER: self.srcserver}
        testlk = {CLIENT: self.testclient, SERVER: self.testserver}

        if not os.path.exists(os.path.join(srclk[side], os.path.normpath(testlk[side] + '.java'))):
            return False
        return True

    def checkbins(self, side):
        binlk = {CLIENT: self.binclient, SERVER: self.binserver}
        testlk = {CLIENT: self.testclient, SERVER: self.testserver}

        if not os.path.exists(os.path.join(binlk[side], os.path.normpath(testlk[side] + '.class'))):
            return False
        return True

    def checkmd5s(self, side, reobf=False):
        if not reobf:
            md5lk = {CLIENT: self.md5client, SERVER: self.md5server}
        else:
            md5lk = {CLIENT: self.md5reobfclient, SERVER: self.md5reobfserver}

        if not os.path.isfile(md5lk[side]):
            return False
        return True

    def checkfolders(self):
        if not os.path.exists(self.dirtemp):
            os.makedirs(self.dirtemp)
        if not os.path.exists(self.dirsrc):
            os.makedirs(self.dirsrc)
        if not os.path.exists(self.dirlogs):
            os.makedirs(self.dirlogs)
        if not os.path.exists(self.dirbin):
            os.makedirs(self.dirbin)
        if not os.path.exists(self.dirreobf):
            os.makedirs(self.dirreobf)
        if not os.path.exists(self.dirlib):
            os.makedirs(self.dirlib)

    def checkupdates(self, silent=False):
        # updates disabled?
        if not self.updateurl:
            if not silent:
                self.logger.debug('Updates disabled')
            return []

        # HINT: Each local entry is of the form dict[filename]=(md5,modificationtime)
        md5lcldict = {}
        files = filterdirs('.', ignore_dirs=self.ignoreupdate, all_files=True)
        for trgfile in files:
            cur_file = os.path.normpath(trgfile)
            with open(cur_file, 'rb') as fh:
                md5_file = md5(fh.read()).hexdigest()
            md5lcldict[cur_file] = (md5_file, os.stat(cur_file).st_mtime)
        try:
            update_url = self.updateurl + 'mcp.md5'
            listfh = urllib.urlopen(update_url)
            if listfh.getcode() != 200:
                return []
            md5srvlist = listfh.readlines()
            md5srvdict = {}
        except IOError:
            return []

        # HINT: Each remote entry is of the form dict[filename]=(md5,modificationtime,action)
        for entry in md5srvlist:
            md5srvdict[entry.split()[0]] = (entry.split()[1], float(entry.split()[2]), entry.split()[3])

        results = []
        for key, value in md5srvdict.items():
            cur_file = os.path.normpath(key)
            # HINT: If the remote entry is not in the local table, append
            if cur_file not in md5lcldict:
                results.append([key, value[0], value[1], value[2]])
                continue

            # HINT: If the remote entry has a different MD5 checksum and modtime is > local entry modtime
            if md5lcldict[cur_file][0] != value[0] and md5lcldict[cur_file][1] < value[1]:
                results.append([key, value[0], value[1], value[2]])

        if results and not silent:
            self.logger.warning('!! Updates available. Please run updatemcp to get them. !!')

        return results

    def cleanbindirs(self, side):
        pathbinlk = {CLIENT: self.binclient, SERVER: self.binserver}

        for path, _, filelist in os.walk(pathbinlk[side]):
            for bin_file in fnmatch.filter(filelist, '*.class'):
                os.remove(os.path.normpath(os.path.join(path, bin_file)))

    def cleanreobfdir(self, side):
        outpathlk = {CLIENT: self.dirreobfclt, SERVER: self.dirreobfsrv}

        reallyrmtree(outpathlk[side])
        os.makedirs(outpathlk[side])

    def applyrg(self, side, reobf=False):
        """Apply rg to the given side"""
        rgcplk = {CLIENT: self.cpathclient, SERVER: self.cpathserver}
        if reobf:
            rgcmd = self.cmdrgreobf
            rgconflk = {CLIENT: self.rgclientreobconf, SERVER: self.rgserverreobconf}
            rgdeoblog = None
            deobsrg = None
            reobsrg = None
        else:
            rgcmd = self.cmdrg
            rgconflk = {CLIENT: self.rgclientconf, SERVER: self.rgserverconf}
            rgdeoblog = {CLIENT: self.rgclientdeoblog, SERVER: self.rgserverdeoblog}
            deobsrg = {CLIENT: self.deobsrgclient, SERVER: self.deobsrgserver}
            reobsrg = {CLIENT: self.reobsrgclient, SERVER: self.reobsrgserver}

        # add retroguard.jar to copy of client or server classpath
        rgcp = [self.retroguard] + rgcplk[side]
        rgcp = os.pathsep.join(rgcp)
        forkcmd = rgcmd.format(classpath=rgcp, conffile=rgconflk[side])
        try:
            self.runcmd(forkcmd)
            if not reobf:
                shutil.copyfile(rgdeoblog[side], deobsrg[side])
                shutil.copyfile(deobsrg[side], reobsrg[side])
        except CalledProcessError as ex:
            self.logger.error('')
            self.logger.error('== ERRORS FOUND ==')
            self.logger.error('')
            for line in ex.output.splitlines():
                if line.strip():
                    if line[0] != '#':
                        self.logger.error(line)
            self.logger.error('==================')
            self.logger.error('')
            raise

    def applyff(self, side):
        """Apply fernflower to the given side"""
        pathclslk = {CLIENT: self.clsclienttmp, SERVER: self.clsservertmp}
        pathsrclk = {CLIENT: self.srcclienttmp, SERVER: self.srcservertmp}

        # HINT: We delete the old temp source folder and recreate it
        reallyrmtree(pathsrclk[side])
        os.makedirs(pathsrclk[side])

        forkcmd = self.cmdfernflower.format(indir=pathclslk[side], outdir=pathsrclk[side])
        self.runcmd(forkcmd)

    def applyexceptor(self, side, exc_update=False):
        """Apply exceptor to the given side"""
        excinput = {CLIENT: self.rgclientout, SERVER: self.rgserverout}
        excoutput = {CLIENT: self.xclientout, SERVER: self.xserverout}
        excconf = {CLIENT: self.xclientconf, SERVER: self.xserverconf}
        exclog = {CLIENT: self.xclientlog, SERVER: self.xserverlog}

        forkcmd = self.cmdexceptor.format(input=excinput[side], output=excoutput[side], conf=excconf[side],
                                          log=exclog[side])
        if exc_update:
            forkcmd += ' %s.exc %s' % (exclog[side], self.mcpparamindex)
        self.runcmd(forkcmd)

    def applyjadretro(self, side):
        """Apply jadretro to the class output directory"""
        pathclslk = {CLIENT: self.clsclienttmp, SERVER: self.clsservertmp}

        ignoredirs = [os.path.normpath(p) for p in self.ignorepkg]
        pkglist = filterdirs(pathclslk[side], '*.class', ignore_dirs=ignoredirs)
        dirs = ' '.join(pkglist)
        forkcmd = self.cmdjadretro.format(targetdir=dirs)
        self.runcmd(forkcmd)

    def applyjad(self, side):
        """Decompile the code using jad"""
        pathclslk = {CLIENT: self.clsclienttmp, SERVER: self.clsservertmp}
        pathsrclk = {CLIENT: self.srcclienttmp, SERVER: self.srcservertmp}

        # HINT: We delete the old temp source folder and recreate it
        reallyrmtree(pathsrclk[side])
        os.makedirs(pathsrclk[side])

        ignoredirs = [os.path.normpath(p) for p in self.ignorepkg]
        pkglist = filterdirs(pathclslk[side], '*.class', ignore_dirs=ignoredirs, append_pattern=True)
        outdir = pathsrclk[side]
        # on linux with wine we need to use \\ as a directory seperator
        if self.cmdjad[:4] == 'wine':
            pkglist = [p.replace(os.sep, '\\\\') for p in pkglist]
            outdir = outdir.replace(os.sep, '\\\\')
        dirs = ' '.join(pkglist)
        forkcmd = self.cmdjad.format(outdir=outdir, classes=dirs)
        self.runcmd(forkcmd)

    def process_jadfixes(self, side):
        """Fix up some JAD miscompiles"""
        pathsrclk = {CLIENT: self.srcclient, SERVER: self.srcserver}

        jadfix(pathsrclk[side])

    def process_fffixes(self, side):
        """Clean up fernflower output"""
        pathsrclk = {CLIENT: self.srcclient, SERVER: self.srcserver}

        fffix(pathsrclk[side])

    def applypatches(self, side, use_ff=False, use_osx=False):
        """Applies the patches to the src directory"""
        pathsrclk = {CLIENT: self.srcclient, SERVER: self.srcserver}
        if use_ff:
            patchlk = {CLIENT: self.ffpatchclient, SERVER: self.ffpatchserver}
        elif use_osx:
            patchlk = {CLIENT: self.patchclient_osx, SERVER: self.patchserver_osx}
        else:
            patchlk = {CLIENT: self.patchclient, SERVER: self.patchserver}

        if use_ff:
            if not self.has_ff_patch:
                self.logger.error('!! Missing ff patches. Aborting !!')
                sys.exit(1)
        elif use_osx:
            if not self.has_osx_patch:
                self.logger.warning('!! Missing osx patches. Aborting !!')
                return False
        else:
            if not self.has_jad_patch:
                self.logger.error('!! Missing jad patches. Aborting !!')
                sys.exit(1)

        # HINT: Here we transform the patches to match the directory separator of the specific platform
        # also normalise lineendings to platform default to keep patch happy
        normalisepatch(patchlk[side], self.patchtemp)
        patchfile = os.path.relpath(self.patchtemp, pathsrclk[side])
        forkcmd = self.cmdpatch.format(srcdir=pathsrclk[side], patchfile=patchfile)
        try:
            self.runcmd(forkcmd)
        except CalledProcessError as ex:
            self.logger.warning('')
            self.logger.warning('== ERRORS FOUND ==')
            if side == CLIENT and not use_ff:
                self.logger.warning('When decompiling with ModLoader a single hunk failure in RenderBlocks is expected '
                                    'and is not a problem')
            self.logger.warning('')
            for line in ex.output.splitlines():
                if 'saving rejects' in line:
                    self.logger.warning(line)
            self.logger.warning('==================')
            self.logger.warning('')

    def recompile(self, side):
        """Recompile the sources and produce the final bins"""
        cplk = {CLIENT: self.cpathclient, SERVER: self.cpathserver}
        pathbinlk = {CLIENT: self.binclient, SERVER: self.binserver}
        pathsrclk = {CLIENT: self.srcclient, SERVER: self.srcserver}
        pathlog = {CLIENT: self.clientrecomplog, SERVER: self.serverrecomplog}

        if not os.path.exists(pathbinlk[side]):
            os.makedirs(pathbinlk[side])

        # HINT: We create the list of source directories based on the list of packages
        # on windows we just pass wildcards, otherwise we pass the full file list
        if self.osname == 'win':
            all_files = False
            append_pattern = True
        else:
            all_files = True
            append_pattern = False
        pkglist = filterdirs(pathsrclk[side], '*.java', append_pattern=append_pattern, all_files=all_files)
        dirs = ' '.join(pkglist)
        classpath = os.pathsep.join(cplk[side])
        forkcmd = self.cmdrecomp.format(classpath=classpath, sourcepath=pathsrclk[side], outpath=pathbinlk[side],
                                        pkgs=dirs)
        try:
            self.runcmd(forkcmd, log_file=pathlog[side])
        except CalledProcessError as ex:
            self.logger.error('')
            self.logger.error('== ERRORS FOUND ==')
            self.logger.error('')
            for line in ex.output.splitlines():
                if line.strip():
                    if line[0] != '[' and line[0:4] != 'Note':
                        self.logger.error(line)
                        if '^' in line:
                            self.logger.error('')
            self.logger.error('==================')
            self.logger.error('')
            raise

    def startserver(self):
        classpath = [self.binserver] + self.cpathserver
        classpath = [os.path.join('..', p) for p in classpath]
        classpath = os.pathsep.join(classpath)
        os.chdir(self.dirjars)
        forkcmd = self.cmdstartsrv.format(classpath=classpath)
        self.runmc(forkcmd)

    def startclient(self):
        classpath = [self.binclient] + self.cpathclient
        classpath = [os.path.join('..', p) for p in classpath]
        classpath = os.pathsep.join(classpath)
        natives = os.path.join('..', self.dirnatives)
        os.chdir(self.dirjars)
        forkcmd = self.cmdstartclt.format(classpath=classpath, natives=natives)
        self.runmc(forkcmd)

    def runcmd(self, forkcmd, quiet=False, check_return=True, log_file=None):
        forklist = cmdsplit(forkcmd)
        if not quiet:
            self.logger.debug("runcmd: '%s'", truncate(forkcmd, 500))
            self.logger.debug("shlex: %s", truncate(str(forklist), 500))
        process = subprocess.Popen(forklist, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=-1)
        output, _ = process.communicate()
        if log_file is not None:
            with open(log_file, 'w') as log:
                log.write(output)
        if not quiet:
            for line in output.splitlines():
                self.logger.debug(line)
        if process.returncode:
            if not quiet:
                self.logger.error("'%s' failed : %d", truncate(forkcmd, 100), process.returncode)
            if check_return:
                raise CalledProcessError(process.returncode, forkcmd, output)
        return output

    def runmc(self, forkcmd, quiet=False, check_return=True):
        forklist = cmdsplit(forkcmd)
        if not quiet:
            self.logger.debug("runmc: '%s'", truncate(forkcmd, 500))
            self.logger.debug("shlex: %s", truncate(str(forklist), 500))
        output = ''
        process = subprocess.Popen(forklist, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1)
        while process.poll() is None:
            line = process.stdout.readline()
            if line:
                line = line.rstrip()
                output += line
                if not quiet:
                    self.loggermc.debug(line)
        if process.returncode:
            if not quiet:
                self.logger.error("'%s' failed : %d", truncate(forkcmd, 100), process.returncode)
            if check_return:
                raise CalledProcessError(process.returncode, forkcmd, output)
        return output

    def extractjar(self, side):
        """Unzip the jar file to the bin directory defined in the config file"""
        pathbinlk = {CLIENT: self.binclienttmp, SERVER: self.binservertmp}
        jarlk = {CLIENT: self.xclientout, SERVER: self.xserverout}

        # HINT: We delete the specific side directory and recreate it
        reallyrmtree(pathbinlk[side])
        os.makedirs(pathbinlk[side])

        # HINT: We extract the jar to the right location
        with closing(zipfile.ZipFile(jarlk[side])) as zipjar:
            zipjar.extractall(pathbinlk[side])

    def copycls(self, side):
        """Copy the class files to the temp directory defined in the config file"""
        pathbinlk = {CLIENT: self.binclienttmp, SERVER: self.binservertmp}
        pathclslk = {CLIENT: self.clsclienttmp, SERVER: self.clsservertmp}

        # HINT: We delete the specific side directory and recreate it
        reallyrmtree(pathclslk[side])
        os.makedirs(pathclslk[side])

        ignore_dirs = [os.path.normpath(p) for p in self.ignorepkg]
        files = filterdirs(pathbinlk[side], '*.class', ignore_dirs=ignore_dirs, all_files=True)
        for src_file in files:
            sub_dir = os.path.relpath(os.path.dirname(src_file), pathbinlk[side])
            dest_file = os.path.join(pathclslk[side], sub_dir, os.path.basename(src_file))
            if not os.path.exists(os.path.dirname(dest_file)):
                os.makedirs(os.path.dirname(dest_file))
            shutil.copy(src_file, dest_file)

    def copysrc(self, side):
        """Copy the source files to the src directory defined in the config file"""
        pathsrctmplk = {CLIENT: self.srcclienttmp, SERVER: self.srcservertmp}
        pathsrclk = {CLIENT: self.srcclient, SERVER: self.srcserver}

        # HINT: We check if the top output directory exists. If not, we create it
        if not os.path.exists(pathsrclk[side]):
            os.makedirs(pathsrclk[side])

        # HINT: copy source to final dir, fixing line endings
        self.copyandfixsrc(pathsrctmplk[side], pathsrclk[side])

        # HINT: copy Start and soundfix to source dir
        if side == CLIENT:
            normaliselines(os.path.join(self.fixesclient, self.fixstart + '.java'),
                           os.path.join(pathsrclk[side], self.fixstart + '.java'))
            if self.fixsound:
                normaliselines(os.path.join(self.fixesclient, self.fixsound + '.java'),
                               os.path.join(pathsrclk[side], self.fixsound + '.java'))

    def copyandfixsrc(self, src_dir, dest_dir):
        src_dir = os.path.normpath(src_dir)
        dest_dir = os.path.normpath(dest_dir)
        ignore_dirs = [os.path.normpath(p) for p in self.ignorepkg]
        files = filterdirs(src_dir, '*.java', ignore_dirs=ignore_dirs, all_files=True)
        for src_file in files:
            sub_dir = os.path.relpath(os.path.dirname(src_file), src_dir)
            dest_file = os.path.join(dest_dir, sub_dir, os.path.basename(src_file))
            if not os.path.exists(os.path.dirname(dest_file)):
                os.makedirs(os.path.dirname(dest_file))
            # normalise lineendings to platform default, to keep patch happy
            normaliselines(src_file, dest_file)

    def process_rename(self, side):
        """Rename the sources using the CSV data"""
        pathsrclk = {CLIENT: self.srcclient, SERVER: self.srcserver}
        reoblk = {CLIENT: self.reobsrgclient, SERVER: self.reobsrgserver}

        if not self.has_name_csv:
            self.logger.warning('!! renaming disabled due to no csvs !!')
            return False

        # HINT: We read the relevant CSVs
        names = {'methods': {}, 'fields': {}, 'params': {}}
        with open(self.csvmethods, 'rb') as fh:
            methodsreader = csv.DictReader(fh)
            for row in methodsreader:
                if int(row['side']) == side:
                    if row['name'] != row['searge']:
                        names['methods'][row['searge']] = row['name']
        with open(self.csvfields, 'rb') as fh:
            fieldsreader = csv.DictReader(fh)
            for row in fieldsreader:
                if int(row['side']) == side:
                    if row['name'] != row['searge']:
                        names['fields'][row['searge']] = row['name']
        if self.has_param_csv:
            with open(self.csvparams, 'rb') as fh:
                paramsreader = csv.DictReader(fh)
                for row in paramsreader:
                    if int(row['side']) == side:
                        names['params'][row['param']] = row['name']

        regexps = {
            'methods': re.compile(r'func_[0-9]+_[a-zA-Z_]+'),
            'fields': re.compile(r'field_[0-9]+_[a-zA-Z_]+'),
            'params': re.compile(r'p_[\w]+_\d+_'),
        }

        def updatefile(src_file):
            tmp_file = src_file + '.tmp'
            with open(src_file, 'r') as fh:
                buf = fh.read()
            for group in ['methods', 'fields', 'params']:
                def mapname(match):
                    try:
                        return names[group][match.group(0)]
                    except KeyError:
                        pass
                    return match.group(0)
                buf = regexps[group].sub(mapname, buf)
            with open(tmp_file, 'w') as fh:
                fh.write(buf)
            shutil.move(tmp_file, src_file)

        # HINT: update reobf srg
        updatefile(reoblk[side])

        # HINT: We pathwalk the sources
        for path, _, filelist in os.walk(pathsrclk[side], followlinks=True):
            for cur_file in fnmatch.filter(filelist, '*.java'):
                updatefile(os.path.normpath(os.path.join(path, cur_file)))
        return True

    def process_renumber(self, side):
        """Renumber the sources using the CSV data"""
        pathsrclk = {CLIENT: self.srcclient, SERVER: self.srcserver}

        if not self.has_renumber_csv:
            self.logger.warning('!! renumbering disabled due to no csv !!')
            return False

        regexps = {
            'methods': re.compile(r'func_([0-9]+)_[a-zA-Z_]+'),
            'fields': re.compile(r'field_([0-9]+)_[a-zA-Z_]+'),
            'params': re.compile(r'p_[\d]+_'),
        }

        # HINT: We read the relevant CSVs
        mapping = {'methods': {}, 'fields': {}, 'params': {}}
        with open(self.csvnewids, 'rb') as fh:
            in_csv = csv.DictReader(fh)
            for line in in_csv:
                in_id = None
                if side == CLIENT:
                    if line['client'] != '*':
                        in_id = line['client']
                else:
                    if line['server'] != '*':
                        in_id = line['server']
                if in_id:
                    method_match = regexps['methods'].match(in_id)
                    if method_match is not None:
                        if in_id not in mapping['methods']:
                            mapping['methods'][in_id] = line['newid']
                            in_param = 'p_' + method_match.group(1) + '_'
                            method_match = regexps['methods'].match(line['newid'])
                            if method_match is not None:
                                out_param = 'p_' + method_match.group(1) + '_'
                                mapping['params'][in_param] = out_param
                    field_match = regexps['fields'].match(in_id)
                    if field_match is not None:
                        if in_id not in mapping['fields']:
                            mapping['fields'][in_id] = line['newid']

        def updatefile(src_file):
            tmp_file = src_file + '.tmp'
            with open(src_file, 'r') as fh:
                buf = fh.read()
            for group in mapping.keys():
                def mapname(match):
                    try:
                        return mapping[group][match.group(0)]
                    except KeyError:
                        pass
                    return match.group(0)
                buf = regexps[group].sub(mapname, buf)
            with open(tmp_file, 'w') as fh:
                fh.write(buf)
            shutil.move(tmp_file, src_file)

        # HINT: We pathwalk the sources
        for path, _, filelist in os.walk(pathsrclk[side], followlinks=True):
            for cur_file in fnmatch.filter(filelist, '*.java'):
                updatefile(os.path.normpath(os.path.join(path, cur_file)))
        return True

    def process_annotate(self, side):
        """Annotate OpenGL constants"""
        pathsrclk = {CLIENT: self.srcclient, SERVER: self.srcserver}

        # HINT: We pathwalk the sources
        for path, _, filelist in os.walk(pathsrclk[side], followlinks=True):
            for cur_file in fnmatch.filter(filelist, '*.java'):
                src_file = os.path.normpath(os.path.join(path, cur_file))
                annotate_file(src_file)

    def process_comments(self, side):
        """Removes all C/C++/Java-style comments from files"""
        pathsrclk = {CLIENT: self.srcclient, SERVER: self.srcserver}

        strip_comments(pathsrclk[side])

    def process_cleanup(self, side):
        """Do lots of random cleanups including stripping comments, trailing whitespace and extra blank lines"""
        pathsrclk = {CLIENT: self.srcclient, SERVER: self.srcserver}

        src_cleanup(pathsrclk[side], fix_imports=True, fix_unicode=True, fix_charval=True, fix_pi=True, fix_round=False)

    def process_javadoc(self, side):
        """Add CSV descriptions to methods and fields as javadoc"""
        pathsrclk = {CLIENT: self.srcclient, SERVER: self.srcserver}

        if not self.has_doc_csv:
            self.logger.warning('!! javadoc disabled due to no csvs !!')
            return False

        #HINT: We read the relevant CSVs
        methodsreader = csv.DictReader(open(self.csvmethods, 'r'))
        fieldsreader = csv.DictReader(open(self.csvfields, 'r'))

        methods = {}
        for row in methodsreader:
            #HINT: Only include methods that have a non-empty description
            if int(row['side']) == side and row['desc']:
                methods[row['searge']] = row['desc'].replace('*/', '* /')

        fields = {}
        for row in fieldsreader:
            #HINT: Only include fields that have a non-empty description
            if int(row['side']) == side and row['desc']:
                fields[row['searge']] = row['desc'].replace('*/', '* /')

        regexps = {
            'field': re.compile(r'^(?P<indent> {4}|\t)(?:[\w$.[\]]+ )*(?P<name>field_[0-9]+_[a-zA-Z_]+) *(?:=|;)'),
            'method': re.compile(r'^(?P<indent> {4}|\t)(?:[\w$.[\]]+ )*(?P<name>func_[0-9]+_[a-zA-Z_]+)\('),
        }
        wrapper = TextWrapper(width=120)

        # HINT: We pathwalk the sources
        for path, _, filelist in os.walk(pathsrclk[side], followlinks=True):
            for cur_file in fnmatch.filter(filelist, '*.java'):
                src_file = os.path.normpath(os.path.join(path, cur_file))
                tmp_file = src_file + '.tmp'
                with open(src_file, 'r') as fh:
                    buf_in = fh.readlines()

                buf_out = []
                #HINT: Look for method/field declarations in this file
                for line in buf_in:
                    fielddecl = regexps['field'].match(line)
                    methoddecl = regexps['method'].match(line)
                    if fielddecl:
                        prev_line = buf_out[-1].strip()
                        indent = fielddecl.group('indent')
                        name = fielddecl.group('name')
                        if name in fields:
                            desc = fields[name]
                            if len(desc) < 70:
                                if prev_line != '' and prev_line != '{':
                                    buf_out.append('\n')
                                buf_out.append(indent + '/** ')
                                buf_out.append(desc)
                                buf_out.append(' */\n')
                            else:
                                wrapper.initial_indent = indent + ' * '
                                wrapper.subsequent_indent = indent + ' * '
                                if prev_line != '' and prev_line != '{':
                                    buf_out.append('\n')
                                buf_out.append(indent + '/**\n')
                                buf_out.append(wrapper.fill(desc) + '\n')
                                buf_out.append(indent + ' */\n')
                    elif methoddecl:
                        prev_line = buf_out[-1].strip()
                        indent = methoddecl.group('indent')
                        name = methoddecl.group('name')
                        if name in methods:
                            desc = methods[name]
                            wrapper.initial_indent = indent + ' * '
                            wrapper.subsequent_indent = indent + ' * '
                            if prev_line != '' and prev_line != '{':
                                buf_out.append('\n')
                            buf_out.append(indent + '/**\n')
                            buf_out.append(wrapper.fill(desc) + '\n')
                            buf_out.append(indent + ' */\n')
                    buf_out.append(line)

                with open(tmp_file, 'w') as fh:
                    fh.writelines(buf_out)
                shutil.move(tmp_file, src_file)
        return True

    def applyastyle(self, side):
        """Recompile the sources and produce the final bins"""
        pathsrclk = {CLIENT: self.srcclient, SERVER: self.srcserver}

        if not self.has_astyle_cfg:
            self.logger.warning('!! reformatting disabled due to no config !!')
            return False

        # HINT: We create the list of source directories based on the list of packages
        pkglist = filterdirs(pathsrclk[side], '*.java', append_pattern=True)
        dirs = ' '.join(pkglist)
        forkcmd = self.cmdastyle.format(classes=dirs, conffile=self.astyleconf)
        self.runcmd(forkcmd)
        return True

    def gathermd5s(self, side, reobf=False):
        if not reobf:
            md5lk = {CLIENT: self.md5client, SERVER: self.md5server}
        else:
            md5lk = {CLIENT: self.md5reobfclient, SERVER: self.md5reobfserver}
        pathbinlk = {CLIENT: self.binclient, SERVER: self.binserver}

        with open(md5lk[side], 'w') as md5file:
            # HINT: We pathwalk the recompiled classes
            for path, _, filelist in os.walk(pathbinlk[side]):
                class_path = os.path.relpath(path, pathbinlk[side]).replace(os.sep, '/')
                if class_path == '.':
                    class_path = ''
                else:
                    class_path += '/'
                for class_file in fnmatch.filter(filelist, '*.class'):
                    class_name = class_path + os.path.splitext(class_file)[0]
                    bin_file = os.path.normpath(os.path.join(path, class_file))
                    with open(bin_file, 'rb') as fh:
                        class_md5 = md5(fh.read()).hexdigest()
                    md5file.write('%s %s\n' % (class_name, class_md5))

    def packbin(self, side):
        jarlk = {CLIENT: self.cmpjarclient, SERVER: self.cmpjarserver}
        pathbinlk = {CLIENT: self.binclient, SERVER: self.binserver}
        pathtmpbinlk = {CLIENT: self.binclienttmp, SERVER: self.binservertmp}

        ignore_files = []
        if side == CLIENT:
            ignore_files.append(self.fixstart + '.class')
            if self.fixsound:
                ignore_files.append(self.fixsound + '.class')

        # HINT: We create the zipfile and add all the files from the bin directory
        with closing(zipfile.ZipFile(jarlk[side], 'w')) as zipjar:
            for path, _, filelist in os.walk(pathbinlk[side]):
                class_path = os.path.relpath(path, pathbinlk[side]).replace(os.sep, '/')
                if class_path == '.':
                    class_path = ''
                else:
                    class_path += '/'
                for class_file in fnmatch.filter(filelist, '*.class'):
                    class_name = class_path + class_file
                    bin_file = os.path.normpath(os.path.join(path, class_file))
                    if class_name not in ignore_files:
                        zipjar.write(bin_file, class_name)
            for pkg in self.ignorepkg:
                curpath = os.path.join(pathtmpbinlk[side], os.path.normpath(pkg))
                for path, _, filelist in os.walk(curpath):
                    class_path = os.path.relpath(path, pathtmpbinlk[side]).replace(os.sep, '/')
                    if class_path == '.':
                        class_path = ''
                    else:
                        class_path += '/'
                    for class_file in fnmatch.filter(filelist, '*.class'):
                        class_name = class_path + class_file
                        bin_file = os.path.join(path, class_file)
                        zipjar.write(bin_file, class_name)

    def unpackreobfclasses(self, side, reobf_all=False, srg_names=False):
        jarlk = {CLIENT: self.reobfjarclient, SERVER: self.reobfjarserver}
        md5lk = {CLIENT: self.md5client, SERVER: self.md5server}
        md5reoblk = {CLIENT: self.md5reobfclient, SERVER: self.md5reobfserver}
        outpathlk = {CLIENT: self.dirreobfclt, SERVER: self.dirreobfsrv}
        if srg_names:
            srglk = {CLIENT: self.reobsrgclientsrg, SERVER: self.reobsrgserversrg}
        else:
            srglk = {CLIENT: self.srgsclient, SERVER: self.srgsserver}

        # HINT: We need a table for the old md5 and the new ones
        md5table = {}
        with open(md5lk[side], 'r') as fh:
            for row in fh:
                row = row.strip().split()
                if len(row) == 2:
                    md5table[row[0]] = row[1]
        md5reobtable = {}
        with open(md5reoblk[side], 'r') as fh:
            for row in fh:
                row = row.strip().split()
                if len(row) == 2:
                    md5reobtable[row[0]] = row[1]
        ignore_classes = []
        if side == CLIENT:
            ignore_classes.append(self.fixstart)
            if self.fixsound:
                ignore_classes.append(self.fixsound)
        trgclasses = []
        for key in md5reobtable.keys():
            if key in ignore_classes:
                continue
            if key not in md5table:
                trgclasses.append(key)
                if '$' in key:
                    self.logger.info('> New inner class found: %s', key)
                else:
                    self.logger.info('> New class found      : %s', key)
            elif md5table[key] != md5reobtable[key]:
                trgclasses.append(key)
                self.logger.info('> Modified class found : %s', key)
            elif reobf_all:
                trgclasses.append(key)
                self.logger.info('> Unchanged class found: %s', key)
        classes = {}
        srg_data = parse_srg(srglk[side])
        for row in srg_data['CL']:
            classes[row['deobf_name']] = row['obf_name']

        if not os.path.exists(outpathlk[side]):
            os.makedirs(outpathlk[side])

        # HINT: We extract the modified class files
        with closing(zipfile.ZipFile(jarlk[side])) as zipjar:
            for in_class in trgclasses:
                parent_class, sep, inner_class = in_class.partition('$')
                if in_class in classes:
                    out_class = classes[in_class] + '.class'
                elif parent_class in classes:
                    out_class = classes[parent_class] + sep + inner_class + '.class'
                else:
                    out_class = in_class + '.class'
                    out_class = out_class.replace(self.nullpkg, '')
                    if out_class[0] == '/':
                        out_class = out_class[1:]
                try:
                    zipjar.extract(out_class, outpathlk[side])
                    self.logger.info('> Outputted %s to %s as %s', in_class.ljust(35), outpathlk[side], out_class)
                except KeyError:
                    self.logger.error('* File %s not found for %s', out_class, in_class)
                except IOError:
                    self.logger.error('* File %s failed extracting for %s', out_class, in_class)

    def downloadupdates(self, force=False):
        if not self.updateurl:
            self.logger.error('Updates disabled.')
            return

        newfiles = self.checkupdates(silent=True)
        if not newfiles:
            self.logger.info('No new updates found.')
            return

        for entry in newfiles:
            if entry[3] == 'U':
                self.logger.info('New version found for : %s', entry[0])
            elif entry[3] == 'D':
                self.logger.info('File tagged for deletion : %s', entry[0])

        if 'CHANGELOG' in [i[0] for i in newfiles]:
            print ''
            self.logger.info('== CHANGELOG ==')
            changelog_url = self.updateurl + 'mcp/CHANGELOG'
            changelog = urllib.urlopen(changelog_url).readlines()
            for line in changelog:
                self.logger.info(line.strip())
                if not line.strip():
                    break
            print ''
            print ''

        if not force:
            print 'WARNING:'
            print 'You are going to update MCP'
            print 'Are you sure you want to continue ?'
            answer = raw_input('If you really want to update, enter "Yes" ')
            if answer.lower() not in ['yes', 'y']:
                print 'You have not entered "Yes", aborting the update process'
                sys.exit(1)

        for entry in newfiles:
            if entry[3] == 'U':
                self.logger.info('Retrieving file from server : %s', entry[0])
                cur_file = os.path.normpath(entry[0])
                path = os.path.dirname(cur_file)
                if not os.path.isdir(path):
                    try:
                        os.makedirs(path)
                    except OSError:
                        pass
                file_url = self.updateurl + 'mcp/' + entry[0]
                urllib.urlretrieve(file_url, cur_file)
            elif entry[3] == 'D':
                self.logger.info('Removing file from local install : %s', entry[0])
                # Remove file here

    def unpackmodifiedclasses(self, side):
        md5lk = {CLIENT: self.md5client, SERVER: self.md5server}
        md5reoblk = {CLIENT: self.md5reobfclient, SERVER: self.md5reobfserver}
        outpathlk = {CLIENT: self.srcmodclient, SERVER: self.srcmodserver}
        src = {CLIENT: self.srcclient, SERVER: self.srcserver}

        # HINT: We need a table for the old md5 and the new ones
        md5table = {}
        with open(md5lk[side], 'r') as fh:
            for row in fh:
                row = row.strip().split()
                if len(row) == 2:
                    md5table[row[0]] = row[1]
        md5reobtable = {}
        with open(md5reoblk[side], 'r') as fh:
            for row in fh:
                row = row.strip().split()
                if len(row) == 2:
                    md5reobtable[row[0]] = row[1]
        trgclasses = []
        for key in md5reobtable.keys():
            if key not in md5table:
                trgclasses.append(key)
                if '$' in key:
                    self.logger.info('> New inner class found: %s', key)
                else:
                    self.logger.info('> New class found      : %s', key)
            elif md5table[key] != md5reobtable[key]:
                trgclasses.append(key)
                self.logger.info('> Modified class found : %s', key)

        if not os.path.exists(outpathlk[side]):
            os.makedirs(outpathlk[side])

        # HINT: We extract the source files for the modified class files
        for in_class in trgclasses:
            src_file = os.path.normpath(os.path.join(src[side], in_class + '.java'))
            dest_file = os.path.normpath(os.path.join(outpathlk[side], in_class + '.java'))
            if os.path.isfile(src_file):
                if not os.path.exists(os.path.dirname(dest_file)):
                    os.makedirs(os.path.dirname(dest_file))
                try:
                    shutil.copyfile(src_file, dest_file)
                    self.logger.info('> Outputted %s to %s', in_class.ljust(35), outpathlk[side])
                except IOError:
                    self.logger.error('* File %s copy failed', in_class)

    def createreobfsrg(self):
        targsrg = {CLIENT: self.reobsrgclientsrg, SERVER: self.reobsrgserversrg}
        deobsrg = {CLIENT: self.deobsrgclient, SERVER: self.deobsrgserver}
        reobsrg = {CLIENT: self.reobsrgclient, SERVER: self.reobsrgserver}
        
        for side in [CLIENT, SERVER]:
            if not os.path.isfile(reobsrg[side]):
                continue
            deob = self.loadsrg(deobsrg[side], reverse=True)
            reob = self.loadsrg(reobsrg[side], reverse=False)
            out = {'CL:': {}, 'MD:': {}, 'FD:': {}, 'PK:': {}}
            
            for type,de in deob.items():
                re = reob[type]
                out[type] = dict([[k,re[v]] for k,v in de.items()])
            
            if os.path.isfile(targsrg[side]):
                os.remove(targsrg[side])
            
            with open(targsrg[side], 'wb') as fh:
                for type in ['PK:', 'CL:', 'FD:', 'MD:']:
                    for k in sorted(out[type], key=out[type].get):
                        fh.write('%s %s %s\n' % (type, k, out[type][k]))
    
    def loadsrg(self, srg_file, reverse=False):
        with open(srg_file, 'r') as fh:
            lines = fh.readlines()
        srg = {'CL:': {}, 'MD:': {}, 'FD:': {}, 'PK:': {}}
        
        for line in lines:
            line = line.strip()
            if len(line) == 0: continue
            if line[0] == '#': continue
            args = line.split(' ')
            type = args[0]
            
            if type == 'PK:' or type == 'CL:' or type == 'FD:':
                srg[type][args[1]] = args[2]
            elif type == 'MD:':
                srg[type][args[1] + ' ' + args[2]] = args[3] + ' ' + args[4]
            else:
                assert 'Unknown type %s' % line

        if reverse:
            for type,map in srg.items():
                srg[type] = dict([[v,k] for k,v in map.items()])
        return srg