xmipp

#!/usr/bin/env python2
# ***************************************************************************
# * Authors:     Carlos Oscar S. Sorzano (coss@cnb.csic.es)
# *              David Maluenda (dmaluenda@cnb.csic.es)
# *
# *
# * This program is free software; you can redistribute it and/or modify
# * it under the terms of the GNU General Public License as published by
# * the Free Software Foundation; either version 2 of the License, or
# * (at your option) any later version.
# *
# * This program is distributed in the hope that it will be useful,
# * but WITHOUT ANY WARRANTY; without even the implied warranty of
# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# * GNU General Public License for more details.
# *
# * You should have received a copy of the GNU General Public License
# * along with this program; if not, write to the Free Software
# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
# * 02111-1307  USA
# *
# *  All comments concerning this program package may be sent to the
# *  e-mail address 'scipion@cnb.csic.es'
# ***************************************************************************/


import distutils.spawn
import glob
import os
import re
import shutil
import sys
import unittest
import subprocess
from datetime import datetime


# --K-E-E-P--U-P-D-A-T-E-D-- #
VERSION_TAG = "Xmipp version"
##############################
XMIPP_VERSION = 'devel'  #
RELEASE_DATE = 'not released yet'  #
##############################

XMIPP = 'xmipp'
XMIPP_CORE = 'xmippCore'
XMIPP_VIZ = 'xmippViz'

XMIPP_SCRIPT_VERSION = ''

REPOSITORIES = {XMIPP: 'https://github.com/Vilax/xmipp-Simple.git'}

CONFIG_FILE_NAME = "xmipp.conf"

def checkGithubConnection():
    from httplib import HTTPConnection
    from socket import gaierror
    conn = HTTPConnection("www.github.com", timeout=3)
    try:
        conn.request("HEAD", "/")
        return True
    except gaierror:
        return False
    finally:
        conn.close()


def stampVersion():
    LAST_COMPILATION = datetime.now().strftime("%d/%m/%Y")

    def getCommit(repo):
        currDir = os.getcwd()
        os.chdir('src/%s'%repo)
        if os.path.exists('.git'):
            branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD'])
            hash = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD'])
            repoInfo = "%s (%s)" % (branch.strip('\n'), hash.strip('\n'))
        else:
            repoInfo = "(no git repo detected)"
        os.chdir(currDir)

        return repoInfo

    versionBinFn = os.path.join('src', 'xmipp', 'applications', 'programs',
                                'version', 'version.cpp')
    createDir(os.path.dirname(versionBinFn))
    with open(versionBinFn, 'w') as f:
        f.write("""// Auto-generated code to get compilation Info
#include <sys/utsname.h>
#include <iostream>
#include <string.h>
using namespace std;

int main(int argc, char** argv){

    if (argc>2)
    {
        std::cout << "Incorrect parameter" << std::endl;
        return 1;
    }
    int shrt = 0;
    if (argc>1)
    {
        if((strcmp(argv[1], "--short") == 0))
        {
            shrt = 1;
        }else{
            std::cout << "Incorrect parameter: " << argv[1] << std::endl;
            return 2;
        }
    }

    if (shrt==1)
    {
        std::cout << "%s: %s" << std::endl;
    }else{
        struct utsname utsname; // stores the data returned by uname()
        struct utsname *utsname_ptr = &utsname; // pointer to the struct holding the data returned by uname()
    
        int ret;
        ret = uname(utsname_ptr);
    
        std::cout << std::endl;
        std::cout << "  \033[4m%s\033[24m: \033[1m%s\033[0m" << std::endl;
        std::cout << std::endl;
        std::cout << "  Release date:     %s" << std::endl;
        std::cout << "  Xmipp branch:     %s" << std::endl;
        std::cout << "  Compilation date: %s" << std::endl;
        std::cout << "  Compiler:         g++ " << __VERSION__ << std::endl;
        std::cout << "  Compiling system: " << utsname.machine << " " << utsname.sysname
                  << " " << utsname.release << std::endl 
                  << "                    " << utsname.version << std::endl;
        std::cout << std::endl;
    }
    return 0;
}
""" % (VERSION_TAG, XMIPP_VERSION, VERSION_TAG, XMIPP_VERSION, RELEASE_DATE,
       getCommit(XMIPP), LAST_COMPILATION))

def whereis(program):
    programPath=distutils.spawn.find_executable(program)
    if programPath:
        return os.path.dirname(programPath)
    else:
        return None

def createDir(dirname):
    if not os.path.exists(dirname):
        os.makedirs(dirname)

def checkProgram(programName, show=True):
    systems = ["Ubuntu/Debian","ManjaroLinux"]
    try:
        osInfo = subprocess.Popen(["lsb_release", "--id"],
                                   stdout=subprocess.PIPE).stdout.read()
        osName = osInfo.split('\t')[1].strip('\n')
        osId = -1  # no default OS
        for idx, system in enumerate(systems):
            if osName in system:
                osId = idx
    except:
        osId = -1

    systemInstructions = {}   #        Ubuntu/Debian          ;      ManjaroLinux
    systemInstructions["git"] = ["sudo apt-get -y install git","sudo pacman -Syu --noconfirm git"]
    systemInstructions["gcc"] = ["sudo apt-get -y install gcc","sudo pacman -Syu --noconfirm gcc"]
    systemInstructions["g++"] = ["sudo apt-get -y install g++","sudo pacman -Syu --noconfirm g++"]
    systemInstructions["mpicc"] = ["sudo apt-get -y install libopenmpi-dev","sudo pacman -Syu --noconfirm openmpi"]
    systemInstructions["mpiCC"] = ["sudo apt-get -y install libopenmpi-dev","sudo pacman -Syu --noconfirm openmpi"]
    systemInstructions["scons"] = ['sudo apt-get -y install scons or make sure that Scons is in the path',"sudo pacman -Syu --noconfirm scons"]
    systemInstructions["javac"] = ['sudo apt-get -y install default-jdk default-jre',"sudo pacman -Syu --noconfirm jre"]
    systemInstructions["rsync"] = ["sudo apt-get -y install rsync" , "sudo pacman -Syu --noconfirm rsync"]
    systemInstructions["pip"] = ["sudo apt-get -y install python-pip" , "sudo pacman -Syu --noconfirm rsync"]
    ok=True
    cont = True
    if not whereis(programName):
        # if programName == "scons":
        #     if checkProgram("pip"):
        #         cont=runJob("pip install scons")
        #     else:
        #         ok = False
        if cont:
            if show:
                print(red("Cannot find %s."%programName))
                idx=0
                if programName in systemInstructions:
                    if osId >= 0:
                        print(red(" - %s OS detected, please try: %s"
                                  % (systems[osId],
                                     systemInstructions[programName][osId])))
                    else:
                        print(red("   Do:"))
                        for instructions in systemInstructions[programName]:
                            print(red("    - In %s: %s"%(systems[idx],instructions)))
                            idx+=1
                    print("\nRemember to re-run './xmipp config' after install new software in order to "
                          "take into account the new system configuration.")
            ok = False
        else:
            ok = False
    return ok

def green(text):
    return "\033[92m "+text+"\033[0m"

def red(text):
    return "\033[91m "+text+"\033[0m"

def blue(text):
    return "\033[34m "+text+"\033[0m"


def runJob(cmd, cwd='./', show_output=True, log=None, show_command=True,
           inParallel=False):
    if show_command:
        print(green(cmd))
    p = subprocess.Popen(cmd, cwd=cwd,
        stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
    while not inParallel:
        output = p.stdout.readline()
        if output == '' and p.poll() is not None:
            break
        if output:
            l = output.rstrip()
            if show_output:
                print(l)
            if log is not None:
                log.append(l)
    if inParallel:
        return p
    else:
        return 0 == p.poll()

def cleanSources():
    runJob("rm -rf %s build src" % CONFIG_FILE_NAME)

def cleanBinaries():
    for ext in ['so', 'os', 'o']:
        runJob('find src/* -name "*.%s" -exec rm -rf {} \;' % ext)
    runJob('find . -iname "*.pyc" -delete')
    runJob("rm -rf %s build" % CONFIG_FILE_NAME)


def checkout(branch):
    r, currentBranch = getCurrentBranch()
    if currentBranch == branch:
        return True
    if isRepositoryClean() and runJob("git checkout %s" % branch):
        return True
    print(red("Cannot checkout branch '%s'. Remaining on the branch '%s'." % (branch, currentBranch)))
    return False


def isRepositoryClean(showError=True):
    log = []
    words = ['working directory clean', 'working tree clean']
    runJob('git status', show_output=False, show_command=False, log=log)
    result = any(w in l for w in words for l in log)  # True for clean repo, False otherwise
    if showError and not result:
        print(red('Repository contains uncommitted changes.'))
    return result


def pull():
    isRemoteBranch = runJob('git rev-parse HEAD@{upstream}', show_command=False, show_output=False)
    if checkGithubConnection() and isRemoteBranch:
        return runJob("git pull", show_command=False)
    return True  # meaning that this is a local branch or we are offline, so pull doesn't make sense


def cloneOrCheckout(repo, branch):
    repo_dir = os.path.join('src', repo)
    branch = branch or getBranch(repo, repo_dir)[1]
    if not os.path.exists(repo_dir):
        # If the repo doesn't exist, just clone the whole repo
        if branch is None:
            # let the git client to decide what is the default branch
            return runJob("git clone %s %s" % (REPOSITORIES[repo], repo_dir))
        else:
            return runJob("git clone -b %s %s %s" % (branch, REPOSITORIES[repo], repo_dir))
    else:
        workDir = os.getcwd()
        os.chdir(repo_dir)
        print(green('Checkouting ' + repo + ' ...'))
        res = checkout(branch) and pull()
        os.chdir(workDir)
        return res


def getCurrentTravisBranch():
    # see https://docs.travis-ci.com/user/environment-variables/
    # On Travis, PR will have the TRAVIS_PULL_REQUEST_BRANCH variable non-empty
    # otherwise the TRAVIS_BRANCH will hold the name of the current branch
    if 'TRAVIS_PULL_REQUEST_BRANCH' in os.environ and 'TRAVIS_BRANCH' in os.environ:
        current_branch = os.environ['TRAVIS_PULL_REQUEST_BRANCH'] or os.environ['TRAVIS_BRANCH']
        print(green("Detected branch: " + current_branch))
        return True, current_branch
    return False, None


def getCurrentBranch(cwd='./'):
    log = []
    commit = []
    if not os.path.exists(cwd):
        return False, None
    runJob('git rev-parse HEAD', cwd=cwd, show_output=False, show_command=False, log=commit)
    runJob('git name-rev ' + commit[0], cwd=cwd, show_output=False, show_command=False, log=log)
    if log:
        return True, log[0].split()[1].strip()  # log contains commit_space_branchName
    print(red('Cannot get current branch'))
    return False, None

def getAllBranches(repo):
    log = []
    prefix = 'refs/heads/'
    result = runJob('git ls-remote -h %s' % REPOSITORIES[repo],
        show_output=False, log=log, show_command=False)
    if result:
        branches = [l.split(prefix)[1] for l in log]
        return (True, branches)
    print(red('Cannot listl branches for ' + repo))
    return (False, None)

def getBranch(repo, repo_dir):
    if 'TRAVIS' in os.environ:
        # we need to get current branch of the xmipp
        r1, branchHint = getCurrentTravisBranch()
        r2, branches = getAllBranches(repo)
        if r1 and r2 and branchHint in branches:
            return True, branchHint
    r, branch = getCurrentBranch(repo_dir)
    if r:
        return r, branch
    return getDefaultBranch(repo)

def getDefaultBranch(repo):
    log = []
    key = 'HEAD branch:'
    # this might not work for git < 1.8.5,
    # see https://stackoverflow.com/a/32503667/5484355
    # and https://stackoverflow.com/questions/2832269/git-remote-head-is-ambiguous
    # In such a case we return None (and during e.g. clone the client should decide what is the default branch)
    result = runJob('git remote show %s' % REPOSITORIES[repo],
        show_output=False, log=log, show_command=False)
    if result:
        for l in log:
            if key in l:
                branch = l.split(key)[1] # HEAD branch: devel
                return (True, branch.strip())
    print(red('Cannot auto-detect default branch for ' + repo + '. Maybe git version < 1.8.5?'))
    return (False, None)


def getSources(branch):
    print("Getting sources -------------------------------------")
    createDir("src")
    repos = []
    if 'TRAVIS' not in os.environ: 
        # on Travis, do not change current commit
        repos.append(XMIPP)
    for r in repos:
        if not cloneOrCheckout(r, branch):
            print(red("Cannot get the sources"))
            return False
    return True


def is_config_true(key):
    return configDict and (key in configDict) and (configDict[key] == 'True')


def getDependencies():
    print("Getting Dependencies -------------------------------------")
    createDir("src")
    result = True
    if not result:
        print(red("Cannot get dependencies"))
    return result

def readConfigFile(fnConfig):
    try:
        from ConfigParser import ConfigParser, ParsingError
    except ImportError:
        from configparser import ConfigParser, ParsingError  # Python 3
    retval = None
    cf = ConfigParser()
    cf.optionxform = str  # keep case (stackoverflow.com/questions/1611799)
    try:
        if os.path.isdir(fnConfig):
            if os.path.exists(os.path.join(fnConfig,CONFIG_FILE_NAME)):
                fnConfig = os.path.join(fnConfig,CONFIG_FILE_NAME)
            else:
                fnConfig = os.path.join(fnConfig, "xmipp.template")
        if os.path.exists(fnConfig):
            cf.read(fnConfig)
            if not 'BUILD' in cf.sections():
                print(red("Cannot find section BUILD in %s"%fnConfig))
                return retval
            return dict(cf.items('BUILD'))
    except:
        sys.exit("%s\nPlease fix the configuration file %s." % (sys.exc_info()[1],fnConfig))
    return retval


def createEmptyConfig():
    labels = ['CC','CXX','LINKERFORPROGRAMS','INCDIRFLAGS','LIBDIRFLAGS','CCFLAGS','CXXFLAGS',
              'LINKFLAGS','PYTHONINCFLAGS','MPI_CC','MPI_CXX','MPI_LINKERFORPROGRAMS','MPI_CXXFLAGS',
              'MPI_LINKFLAGS',
              'DEBUG',
              'JAVA_HOME','JAVA_BINDIR','JAVAC','JAR','JNI_CPPPATH', 'USE_DL', 'VERIFIED', 'CONFIG_VERSION']
    configDict = {}
    for label in labels:
        configDict[label]=""
    return configDict


def findFileInDirList(fnH,dirlist):
    for dir in dirlist:
        if len(glob.glob(os.path.join(dir,fnH)))>0:
           return True

def getDependenciesInclude():
    return ['../']


def configCompiler(configDict):
    if configDict["DEBUG"]=="":
        configDict["DEBUG"]="False"

    if configDict["CC"]=="":
        configDict["CC"]="gcc" if checkProgram("gcc") else ""
    if configDict["CXX"]=="":
        if 'TRAVIS' in os.environ:
            # on TRAVIS, we can use cache to speed up the build
            configDict["CXX"]="ccache g++" if checkProgram("g++") else ""
        else:
            configDict["CXX"]="g++" if checkProgram("g++") else "" 
    if configDict["LINKERFORPROGRAMS"]=="":
        if 'TRAVIS' in os.environ:
            # on TRAVIS, we can use cache to speed up the build
            configDict["LINKERFORPROGRAMS"]="ccache g++" if checkProgram("g++") else ""
        else:
            configDict["LINKERFORPROGRAMS"]="g++" if checkProgram("g++") else ""

    if configDict["CC"]=="gcc":
        if not "-std=c99" in configDict["CCFLAGS"]:
            configDict["CCFLAGS"]+=" -std=c99"
    if 'g++' in configDict["CXX"]:
        configDict["CXXFLAGS"] += " -mtune=native -march=native"  # optimize for current machine
        if "-std=c99" not in configDict["CXXFLAGS"]:
            configDict["CXXFLAGS"] += " -std=c++11"
        if 'TRAVIS' in os.environ:
            configDict["CXXFLAGS"] += " -Werror"  # don't tolerate any warnings on build machine
            configDict["CXXFLAGS"] += " -O0"  # don't optimize on Travis, as it slows down the build
        else:
            configDict["CXXFLAGS"] += " -O3"
        if is_config_true("DEBUG"):
            configDict["CXXFLAGS"] += " -g"
    # Nothing special to add to LINKFLAGS


    if configDict["LIBDIRFLAGS"]=="":
        libDirs=[]
        if not findFileInDirList("libhdf5*",libDirs):
            if findFileInDirList("libhdf5*",["/usr/lib/x86_64-linux-gnu"]):
                configDict["LIBDIRFLAGS"]+=" -L/usr/lib/x86_64-linux-gnu"
                # libDirs+=["/usr/lib/x86_64-linux-gnu"]
    if configDict["INCDIRFLAGS"]=="":
        incDirs=[]
        configDict["INCDIRFLAGS"] += ' '.join(map(lambda x: '-I' + str(x), getDependenciesInclude()))
        if not findFileInDirList("hdf5.h",incDirs):
            if findFileInDirList("hdf5.h",["/usr/include/hdf5/serial"]):
                configDict["INCDIRFLAGS"]+=" -I/usr/include/hdf5/serial"
                incDirs+=["/usr/include/hdf5/serial"]
    if configDict["PYTHONINCFLAGS"]=="":
        incDirs=[]
        if not findFileInDirList("Python.h",incDirs):
            if findFileInDirList("Python.h",["/usr/include/python2.7"]):
                configDict["PYTHONINCFLAGS"]+=" -I/usr/include/python2.7"
                incDirs+=["/usr/include/python2.7"]
            if findFileInDirList("ndarraytypes.h",["/usr/lib/python2.7/site-packages/numpy/core/include/numpy"]):
                configDict["PYTHONINCFLAGS"]+=" -I/usr/lib/python2.7/site-packages/numpy/core/include/"
                incDirs+=["usr/lib/python2.7/site-packages/numpy/core/include/"]
            elif findFileInDirList("ndarraytypes.h",["/usr/lib/python2.7/dist-packages/numpy/core/include/numpy"]):
                configDict["PYTHONINCFLAGS"]+=" -I/usr/lib/python2.7/dist-packages/numpy/core/include/"
                incDirs+=["usr/lib/python2.7/dist-packages/numpy/core/include/"]
            elif findFileInDirList("ndarraytypes.h",["/usr/local/lib/python2.7/dist-packages/numpy/core/include/numpy"]):
                configDict["PYTHONINCFLAGS"]+=" -I/usr/local/lib/python2.7/dist-packages/numpy/core/include/"
                incDirs+=["usr/local/lib/python2.7/dist-packages/numpy/core/include/"]

def getHdf5Name(libdirflags):
    libdirs=libdirflags.split("-L")
    for dir in libdirs:
        if os.path.exists(os.path.join(dir.strip(),"libhdf5.so")):
            return "hdf5"
        elif os.path.exists(os.path.join(dir.strip(),"libhdf5_serial.so")):
            return "hdf5_serial"
    return "hdf5"


def checkCompiler(configDict):
    print("Checking compiler configuration ...")
    ensureCompilerVersion(configDict["CXX"])

    cppProg="""
#include <fftw3.h>
#include <hdf5.h>
#include <tiffio.h>
#include <jpeglib.h>
#include <sqlite3.h>
#include <pthread.h>
#include <Python.h>
#include <numpy/ndarraytypes.h>
    """
    cppProg+="\n int main(){}\n"
    with open("xmipp_test_main.cpp", "w") as cppFile:
        cppFile.write(cppProg)

    if not runJob("%s -c -w %s xmipp_test_main.cpp -o xmipp_test_main.o %s %s"%\
              (configDict["CXX"],configDict["CXXFLAGS"],configDict["INCDIRFLAGS"],configDict["PYTHONINCFLAGS"])):
        print(red("Check the SCIPION_HOME, INCDIRFLAGS, CXX, CXXFLAGS and PYTHONINCFLAGS"))
        print(red("If some of the libraries headers fail, try installing fftw3_dev, tiff_dev, jpeg_dev, sqlite_dev"))
        return False
    libhdf5=getHdf5Name(configDict["LIBDIRFLAGS"])
    if not runJob("%s %s %s xmipp_test_main.o -o xmipp_test_main -lfftw3 -lfftw3_threads -l%s  -lhdf5_cpp -ltiff -ljpeg -lsqlite3 -lpthread" % \
              (configDict["LINKERFORPROGRAMS"], configDict["LINKFLAGS"], configDict["LIBDIRFLAGS"],libhdf5)):
        print(red("Check the LINKERFORPROGRAMS, LINKFLAGS and LIBDIRFLAGS"))
        return False
    runJob("rm xmipp_test_main*")
    return True

def configMPI(configDict):
    if configDict["MPI_CC"]=="":
        configDict["MPI_CC"]="mpicc" if checkProgram("mpicc") else ""
    if configDict["MPI_CXX"]=="":
        configDict["MPI_CXX"]="mpiCC" if checkProgram("mpiCC") else ""
    if configDict["MPI_LINKERFORPROGRAMS"]=="":
        configDict["MPI_LINKERFORPROGRAMS"]="mpiCC" if checkProgram("mpiCC") else ""

    # MPI_CXXFLAGS is normally not needed, but if it is we may use mpicc --showme:compile
    # MPI_LINKFLAGS is normally not needed, but if it is we may use mpicc --showme:link

def checkMPI(configDict):
    print("Checking MPI configuration ...")
    cppProg="""
#include <mpi.h>
int main(){}
"""
    with open("xmipp_mpi_test_main.cpp", "w") as cppFile:
        cppFile.write(cppProg)

    if not runJob("%s -c -w %s xmipp_mpi_test_main.cpp -o xmipp_mpi_test_main.o"%\
              (configDict["MPI_CXX"],configDict["INCDIRFLAGS"])):
        print(red("Check the INCDIRFLAGS, MPI_CXX and CXXFLAGS"))
        return False
    libhdf5=getHdf5Name(configDict["LIBDIRFLAGS"])
    if not runJob("%s %s %s xmipp_mpi_test_main.o -o xmipp_mpi_test_main -lfftw3 -lfftw3_threads -l%s  -lhdf5_cpp -ltiff -ljpeg -lsqlite3 -lpthread" % \
              (configDict["MPI_LINKERFORPROGRAMS"], configDict["LINKFLAGS"], configDict["LIBDIRFLAGS"], libhdf5)):
        print(red("Check the LINKERFORPROGRAMS, LINKFLAGS and LIBDIRFLAGS"))
        return False
    runJob("rm xmipp_mpi_test_main*")

    ok = False
    if checkProgram("mpirun",False):
        echoString = "This sentence should be printed 4 times if mpi runs fine"
        ok=(runJob("mpirun -np 4 echo '%s (by mpirun).'" % echoString) or
            runJob("mpirun -np 4 --allow-run-as-root echo '%s (by mpirun).'" % echoString))
    elif checkProgram("mpiexec",False):
        ok=(runJob("mpiexec -np 4 echo '%s (by mpiexec).'" % echoString) or
            runJob("mpiexec -np 4 --allow-run-as-root echo '%s (by mpiexec).'" % echoString))
    else:
        print(red("mpirun or mpiexec have failed."))

    return ok


def configJava(configDict):
    if configDict["JAVA_HOME"]=="":
        javaProgramPath = distutils.spawn.find_executable("java")
        javaHomeDir = None
        if javaProgramPath:
            javaProgramPath=os.path.dirname(os.path.realpath(javaProgramPath))
            javaHomeDir = javaProgramPath.replace("/jre/bin","")
            javaHomeDir = javaHomeDir.replace("/bin","")
        if javaHomeDir:
            configDict["JAVA_HOME"]=javaHomeDir

    if configDict["JAVA_BINDIR"]=="" and javaHomeDir:
        configDict["JAVA_BINDIR"]="%(JAVA_HOME)s/bin"
    if configDict["JAVAC"]=="" and javaHomeDir:
        configDict["JAVAC"]="%(JAVA_BINDIR)s/javac"
    if configDict["JAR"]=="" and javaHomeDir:
        configDict["JAR"]="%(JAVA_BINDIR)s/jar"
    if configDict["JNI_CPPPATH"]=="" and javaHomeDir:
        configDict["JNI_CPPPATH"]="%(JAVA_HOME)s/include:%(JAVA_HOME)s/include/linux"

def checkJava(configDict):
    if not checkProgram("javac"):
        return False

    javaProg="""
    public class Xmipp {
       public static void main(String[] args) {}
    }
"""
    with open("Xmipp.java", "w") as javaFile:
        javaFile.write(javaProg)
    if not runJob("%s Xmipp.java" % configDict["JAVAC"]):
        print(red("Check the JAVAC"))
        return False
    runJob("rm Xmipp.java Xmipp.class")

    cppProg="""
#include <jni.h>
int dummy(){}
"""
    with open("xmipp_jni_test.cpp", "w") as cppFile:
        cppFile.write(cppProg)

    incs=""
    for x in configDict['JNI_CPPPATH'].split(':'):
        incs+=" -I"+x
    if not runJob("%s -c -w %s %s xmipp_jni_test.cpp -o xmipp_jni_test.o"%\
              (configDict["CXX"],incs,configDict["INCDIRFLAGS"])):
        print(red("Check the JNI_CPPPATH, CXX and INCDIRFLAGS"))
        return False
    runJob("rm xmipp_jni_test*")
    return True

def writeConfig(configDict):
    with open(CONFIG_FILE_NAME, "w") as configFile:
        configFile.write("[BUILD]\n")
        for label in sorted(configDict.keys()):
            configFile.write("%s=%s\n"%(label,configDict[label]))

def updateConfig(updatingDict):
    cmdTemplate = "sed -i -e 's/^%s=.*/%s=%s/' %s"
    for k, v in updatingDict.iteritems():
        print(blue("Setting %s=%s" % (k, v)))
        runJob(cmdTemplate % (k, k, v.replace('/', '\/'), CONFIG_FILE_NAME),
               show_command=False)

def config_DL(configDict):
    k = 'USE_DL'
    if (k in configDict) and (configDict[k] != 'True'):
        configDict[k] = 'False'


def configConfigVersion(configDict):
    key = 'CONFIG_VERSION'
    configDict[key] = getScriptVersion()


def ensureConfigVersion(configDict):
    key = 'CONFIG_VERSION'
    if key not in configDict or configDict[key] != XMIPP_SCRIPT_VERSION:
        print(red('We did some changes which are not compatible with your current config file. '
                  'We recommend you to create a backup before regenerating it (use --help for additional info)'))
        exit(-1)


def config():
    print("Configuring -----------------------------------------")
    new_config_dict = createEmptyConfig()
    useScipion = not ('XMIPP_NOSCIPION' in os.environ)
    if useScipion and not getScipionHome():
        print(red("$SCIPION_HOME is not set and scipion is not in the path. Use 'export XMIPP_NOSCIPION=True; ./xmipp' to configure without scipion"))
        return False

    if new_config_dict['VERIFIED'] == '':
        new_config_dict['VERIFIED'] = 'False'

    configCompiler(new_config_dict)
    configMPI(new_config_dict)
    configJava(new_config_dict)
    configConfigVersion(new_config_dict)

    writeConfig(new_config_dict)

    return new_config_dict


def checkConfig():
    print("Checking configuration ------------------------------")
    ensureConfig()
    if configDict['VERIFIED'] != 'True':
        newConf = {}  # to update the config if something fails
        if not checkCompiler(configDict):
            print(red("Cannot compile"))
            print("Possible solutions")
            print("In Ubuntu: sudo apt-get -y install libsqlite3-dev libfftw3-dev libhdf5-dev libopencv-dev python2.7-dev "\
                  "python-numpy python-scipy python-mpi4py")
            print("In Manjaro: sudo pacman -Syu install hdf5 python2-numpy python2-scipy --noconfirm")
            print("\nRemember to re-run './xmipp config' after installing libraries in order to "
                  "take into account the new system configuration.")
            return False
        if not checkMPI(configDict):
            print(red("Cannot compile with MPI or use it"))
            return False
        if not checkJava(configDict):
            print(red("Cannot compile with Java"))
            return False
        newConf['VERIFIED']="True"
        updateConfig(newConf)
    return True


def compileModule(Nproc,module):
    shutil.copyfile(CONFIG_FILE_NAME,"src/%s/install/%s" % (module, CONFIG_FILE_NAME))
    if module == "xmipp":
        stampVersion()
    log = []
    ok = runJob("scons -j%s"%Nproc, "src/%s"%module, log=log)

    return ok

def compile(Nproc):
    ensureConfig()
    #ensureConfigVersion(configDict)
    if not compileDependencies(Nproc):
        return False
    return compileXmipp(Nproc)


def compileDependencies(Nproc):
    print("Building Dependencies -------------------------------------")
    result = True
    if not result:
        print(red("Cannot build dependencies"))
    return result


def compileXmipp(Nproc):
    print("Compiling -------------------------------------------")
    if not compileModule(Nproc,"xmipp"):
        return False
    return True

def runTests(testNames):
    if len(testNames)==0 or 'help' in testNames or '--help' in testNames:
        print("Usage: xmipp test op\n"
              "\n"
              "         op = --show: Show how to invoke all available tests\n"
              "              --allPrograms: Run all program tests\n"
              "              --allFuncs: Run all function tests\n"
              "              'testName': Run certain test (more than one is available)."
              "\n")
        return

    print("Testing ---------------------------------------------")

    xmippSrc = os.environ.get('XMIPP_SRC', None)
    if xmippSrc and os.path.isdir(xmippSrc):
        os.environ['PYTHONPATH'] = ':'.join([
            os.path.join(os.environ['XMIPP_SRC'], XMIPP),
            os.environ.get('PYTHONPATH', '')])

        testsPath = os.path.join(os.environ['XMIPP_SRC'], XMIPP, 'tests')
    else:
        print(red('XMIPP_SRC is not in the enviroment.') +
              '\nTo run the tests you need to run: ' +
              blue('source build/xmipp.bashrc'))
        sys.exit(1)

    dataSetPath = os.path.join(testsPath, 'data')
    # if not os.path.isdir(dataSetPath):
    #     createDir(dataSetPath)
    os.environ["XMIPP_TEST_DATA"] = dataSetPath

    # downloading/updating the dataset
    url = "http://scipion.cnb.csic.es/downloads/scipion/data/tests"
    dataset = 'xmipp_programs'
    if os.path.isdir(dataSetPath):
        print(blue("Updating the test files"))
        task = "update"
    else:
        print(blue("Downloading the test files"))
        task = "download"
    args = "%s %s %s" % ("tests/data", url, dataset)
    runJob("bin/xmipp_sync_data %s %s" % (task, args), cwd='src/xmipp')

    configDict = readConfigFile(CONFIG_FILE_NAME)
    noCudaStr = '--noCuda' if not is_config_true('CUDA') else ''
    print(" Tests to do: %s" % ', '.join(testNames))
    runJob("(cd src/xmipp/tests; %s test.py %s %s)"
           % (getPython(), ' '.join(testNames), noCudaStr))

def getPython():
    python = 'python'
    return python

def install(dirname):
    print("Installing ------------------------------------------")
    cpCmd = "rsync -LptgoD" if checkProgram("rsync", False) else "cp"
    ok = True

    createDir(dirname)

    createDir(dirname+"/lib")
    ok = ok and runJob(cpCmd+" src/*/lib/lib* "+dirname+"/lib/")

    createDir(dirname+"/bin")
    ok = ok and runJob(cpCmd+" src/*/bin/* "+dirname+"/bin/")
    destPathPyModule = os.path.expanduser(os.path.abspath(os.path.join(dirname, "pylib", "xmippPyModules")))
    createDir(destPathPyModule)
    initFn = destPathPyModule + "/__init__.py"
    if not os.path.isfile(initFn):
        with open(initFn, 'w') as f:
            pass  # just to create a init file to be able to import it as module

    createDir(dirname+"/bindings")

    createDir(dirname+"/bindings/python")
    ok = ok and runJob(cpCmd+" src/xmipp/bindings/python/xmipp_base.py "+dirname+"/bindings/python/")
    ok = ok and runJob(cpCmd+" src/xmipp/bindings/python/xmipp.py " + dirname + "/bindings/python/")
    ok = ok and runJob(cpCmd+" src/xmipp/lib/xmippLib.so "+dirname+"/bindings/python/")
    ok = ok and runJob(cpCmd+" src/xmipp/lib/_swig_frm.so "+dirname+"/bindings/python/")

    createDir(dirname+"/resources")
    ok = ok and runJob(cpCmd+" -r src/*/resources/* "+dirname+"/resources/")

    ok = ok and runJob(cpCmd + " -r src/xmippViz/bindings/chimera " + dirname + "/bindings/")
    createDir(dirname+"/bindings/java")
    ok = ok and runJob(cpCmd+" -Lr src/xmippViz/java/lib "+dirname+"/bindings/java/")
    ok = ok and runJob(cpCmd+" -Lr src/xmippViz/java/build "+dirname+"/bindings/java/")
    ok = ok and runJob(cpCmd+" -Lr src/xmippViz/external/imagej "+dirname+"/bindings/java/")
    ok = ok and runJob(cpCmd+" src/xmippViz/bindings/python/xmippViz.py "+dirname+"/bindings/python/")

    if not ok:
        print(red("\nSome error occurred during the installation.\n"))
        sys.exit(1)

    runJob("touch %s/v%s" % (dirname, XMIPP_VERSION))  # version token
    fhBash = open(dirname+"/xmipp.bashrc","w")
    fhFish = open(dirname+"/xmipp.fish","w")
    fhBash.write("# This script is valid for bash and zsh\n\n")
    fhFish.write("# This script is valid for fish\n\n")

    XMIPP_HOME = os.path.realpath(dirname)
    fhBash.write("export XMIPP_HOME=%s\n"%XMIPP_HOME)
    fhFish.write("set -x XMIPP_HOME %s\n"%XMIPP_HOME)

    XMIPP_SRC = os.path.realpath("src")
    fhBash.write("export XMIPP_SRC=%s\n"%XMIPP_SRC)
    fhFish.write("set -x XMIPP_SRC %s\n"%XMIPP_SRC)

    fhBash.write("export PATH=%s/bin:$PATH\n"%XMIPP_HOME)
    fhBash.write("export LD_LIBRARY_PATH=%s/lib:%s/bindings/python:$LD_LIBRARY_PATH\n"%(XMIPP_HOME,XMIPP_HOME))
    fhBash.write("export PYTHONPATH=%s/bindings/python:%s/pylib:$PYTHONPATH\n"%(XMIPP_HOME,XMIPP_HOME))
    fhFish.write("set -px PATH %s/bin\n"%XMIPP_HOME)
    fhFish.write("set -px LD_LIBRARY_PATH %s/lib %s/bindings/python\n"%(XMIPP_HOME,XMIPP_HOME))
    fhFish.write("set -px PYTHONPATH %s/bindings %s/pylib\n"%(XMIPP_HOME,XMIPP_HOME))   

    fhBash.write('\n')
    fhBash.write("alias x='xmipp'\n")
    fhBash.write("alias xsj='xmipp_showj'\n")
    fhBash.write("alias xio='xmipp_image_operate'\n")
    fhBash.write("alias xis='xmipp_image_statistics'\n")
    fhBash.write("alias xih='xmipp_image_header'\n")
    fhBash.write("alias xmu='xmipp_metadata_utilities'\n")
    fhFish.write('\n')
    fhFish.write("alias x 'xmipp'\n")
    fhFish.write("alias xsj 'xmipp_showj'\n")
    fhFish.write("alias xio 'xmipp_image_operate'\n")
    fhFish.write("alias xis 'xmipp_image_statistics'\n")
    fhFish.write("alias xih 'xmipp_image_header'\n")
    fhFish.write("alias xmu 'xmipp_metadata_utilities'\n")

    fhBash.close()
    fhFish.close()

    print("\n"
          "  *********************************************\n"
          "  *                                           *\n"
          "  *  Xmipp have been successfully installed!  *\n"
          "  *                                           *\n"
          "  *********************************************\n\n")
    return True

def writeDevelPaths(dirname):
    fhBash = open(dirname+"/xmipp.bashrc","w")

    XMIPP_HOME = os.path.realpath(dirname)
    fhBash.write("export XMIPP_HOME=%s\n"%XMIPP_HOME)

    XMIPP_SRC = os.path.realpath("src")
    fhBash.write("export XMIPP_SRC=%s\n"%XMIPP_SRC)

    fhBash.write("export LD_LIBRARY_PATH=%s/xmippCore/lib:$LD_LIBRARY_PATH\n"%XMIPP_HOME)
    fhBash.write("export LD_LIBRARY_PATH=%s/xmippCore/bindings/python:$LD_LIBRARY_PATH\n"%XMIPP_HOME)
    fhBash.write("export LD_LIBRARY_PATH=%s/xmipp/lib:$LD_LIBRARY_PATH\n"%XMIPP_HOME)
    fhBash.write("export LD_LIBRARY_PATH=%s/xmipp/bindings/python:$LD_LIBRARY_PATH\n"%XMIPP_HOME)
    fhBash.write("export PYTHONPATH=%s/xmippCore/bindings/python:$PYTHONPATH\n"%XMIPP_HOME)
    fhBash.write("export PYTHONPATH=%s/xmipp/bindings/python:$PYTHONPATH\n"%XMIPP_HOME)
    fhBash.write("export PYTHONPATH=%s/xmippViz/bindings/python:$PYTHONPATH\n"%XMIPP_HOME)

    fhBash.close()

def usage(msg=''):
    if msg != '':
        print(red(msg))
    print("Usage: xmipp [options]\n"
          "   version                     Returns the version information\n"
          "   all [op1=opt1 op2=opt2...]: (Default) Retrieve [br=branch], configure, check, compile [N=8], install [dir=build]\n"
          "   get_dependencies:           Retrieve dependencies from github\n"
          "   get_devel_sources [branch]: Retrieve development sources from github for a given branch (devel branch by default)\n"
          "   cleanBin:                   Clean all already compiled files (build, .so,.os,.o in src/* and " + CONFIG_FILE_NAME + ")\n"
          "   cleanAll:                   Delete all (sources and build directories)\n"
          "   config:                     Configure compilation variables\n"
          "                               for compiling using system libraries\n"
          "   check_config:               Check that the configuration is correct\n"
          "   compile [N]:                Compile all modules with N processors (8 by default)\n"
          "   compile N dependencies:     Compile dependencies\n"
          "   compileAndInstall [N]:      Compile all modules with N processors (8 by default) and install in the default directory\n"
          "   compile  N xmippCore:       Compile xmippCore\n"
          "   compile  N xmipp:           Compile xmipp\n"
          "   compile  N xmippViz:        Compile xmippViz\n"
          "   install [dir]:              Install at dir (./build by default)\n"
          "   get_models [dir]:           Download the Deep Learning Models at dir/models (./build/models by default).\n"
          "   test [--show] testName:     Run tests to check Xmipp programs (without args, it shows a detailed help).\n"
          "                                 if --show is activated without testName all are shown, \n"
          "                                 instead a grep of testName is done \n"
          "For developers:\n"
          "   create_devel_paths:         Create bashrc files for devel\n"
          "   git ...:                    Git command to all 4 repositories\n"
          "   gitConfig:                  Change the git config from https to git\n"
          "                                   uploads the .tgz according to the <login>. \n"
          "                                   Note that login=usr@server must have write permisions to Nolan machine.\n"
          "   tar <mode> [v=ver] [br=br]: Create a bundle of the xmipp (without arguments shows a detailed help)\n"
          "                                   <mode> can be 'Sources', 'BinDebian' or 'BinCentos', when Sources put a branch (default: master).'\n"
          "                                   <ver> usually X.YY.MM\n"
          )


def getVersion(onlyNumbers=False):
    import re  # To clean colors and format
    ansi_escape = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]')

    lines = subprocess.check_output(["src/xmipp/bin/xmipp_version"])
    verInfo = [line for line in lines.splitlines() if VERSION_TAG in line]
    cleanInfo = ansi_escape.sub('', verInfo[0])

    if onlyNumbers:
        cleanInfo = cleanInfo.split(':')[1]

    return cleanInfo.strip(' ')

def getGccVersion(compiler):
    log = []
    runJob(compiler + " -dumpversion", show_output=False, show_command=False, log=log)
    full_version = log[0].strip()
    tokens = full_version.split('.')
    if len(tokens) < 2:
        tokens.append('0')  # for version 5.0, only '5' is returned
    gccVersion = float(str(tokens[0] + '.' + tokens[1]))
    return gccVersion, full_version

def ensureCompilerVersion(compiler):
    if 'g++' in compiler or 'gcc' in compiler:
        ensureGCC_GPPVersion(compiler)
    else:
        print(red('Version detection for \'' + compiler + '\' is not implemented.'))


def ensureGCC_GPPVersion(compiler):
    if 'TRAVIS' in os.environ:
        return  # skip detection on TRAVIS
    if not checkProgram(compiler, True):
        sys.exit(-7)
    gccVersion, fullVersion = getGccVersion(compiler)

    if gccVersion < 4.8:  # join first two numbers, i.e. major and minor version
        print(red('Detected ' + compiler + " in version " + fullVersion + '. Version 4.8 or higher is required.'))
        sys.exit(-8)
    else:
        print(green(compiler + ' ' + fullVersion + ' detected'))

def ensureConfig():
    # assuming the config file is not loaded in the main(), i.e. it does not exists yet and has not been created
    # by another function
    if not configDict:
        print(red("There is no config file. Make sure to run config"))
        sys.exit(-6)


def ensureGit():
    if not checkProgram('git'):
        print(red('Git not found'))
        return False
    return True


def getScriptVersion():
    scriptName = os.path.basename(__file__)
    lastCommit = []
    # get hash of the last commit changing this script
    runJob('git log -n 1 --pretty=format:%H -- ' + scriptName, '.', False, lastCommit, False)
    return lastCommit[0].strip()

if __name__ == '__main__':
    # I comment this line to avoid problems if this script is run in XmippBundle/src/xmipp  or directly from XmippBundle/.
    # os.chdir(os.path.dirname(os.path.abspath(__file__)))

    XMIPP_SCRIPT_VERSION = getScriptVersion()

    if not ensureGit():
        sys.exit(-1)

    configDict = readConfigFile(CONFIG_FILE_NAME)

    n = len(sys.argv)
    if n == 2 and (sys.argv[1]=="help" or sys.argv[1]=="-help" or sys.argv[1]=="--help" or sys.argv[1]=="-h"):
        usage()
        sys.exit(0)
    for idx, arg in enumerate(sys.argv):
        if ' ' in arg:  # to preserve spaces between "comas"
            sys.argv[idx] = '"%s"' % sys.argv[idx]
    if n>=2:
        mode = sys.argv[1]
    else:
        mode = "all"
    if mode=="cleanAll":
        print("WARNING: This will DELETE ALL content from src and build")
        print("         Notice that if you have unpushed changes, \n"
              "           they will be deleted.\n"
              "         Still, if ./xmipp is a link to src/xmipp/xmipp, \n"
              "           it will also be deleted.")
        print("Are you sure you want to do this? (YeS/No) -case sensitive-")
        yesno = raw_input()
        if yesno == "YeS":
            print("Cleaning everything")
            cleanSources()
        else:
            print("Nothing cleaned")
            if yesno.lower()=="yes":
                print("Pay attention to capital letters of YeS")
    elif mode=="get_dependencies":
        getDependencies()
    elif mode=="cleanBin":
        cleanBinaries()
    elif mode=="version":
        if n>2 and sys.argv[2]=='--short':
            runJob("src/xmipp/bin/xmipp_version --short")
        else:
            runJob("src/xmipp/bin/xmipp_version")
    elif mode=="get_devel_sources":
        branch = None if n==2 else sys.argv[2]
        getSources(branch)
    elif mode=="config":
        configDict = config()
    elif mode=="check_config":
        if not checkConfig():
            print(red("\nCheck failed! Something wrong with the configuration.\n"))
            sys.exit(1)
    elif mode=="compile":
        Nproc = 8 if n<3 else sys.argv[2]
        if n<=3:
            ok = compile(Nproc)
            module = 'Xmipp'
        else:
            compileDependencies(Nproc)
            ok = compileModule(Nproc,sys.argv[3])
            module = sys.argv[3]
        if ok:
            print("\n"
                  "  * %s have been successfully compiled * \n"
                  "    > > > Don't forget to install! < < <   \n\n"
                  % module)
        else:
            print(red("\nSome error occurred during the compilation\n"))
            sys.exit(1)
    elif mode=="compileAndInstall":
        if n>2:
            Nproc=sys.argv[2]
        else:
            Nproc=8
        compile(Nproc)
        install("build")
    elif mode=="install":
        if n==3:
            dir=sys.argv[2]
        else:
            dir="build"
        install(dir)
    elif mode=="get_models":
        modelsDir = 'build' if n==2 else sys.argv[2]
    elif mode=="test" or mode=="tests":
        runTests(sys.argv[2:])
    elif mode=="all":
        Nproc = 8
        branch = ''
        buildDir = 'build'
        for arg in sys.argv[2:]:
            if arg.startswith("N="):
                Nproc = int(arg[2:])
            elif arg.startswith("br="):
                branch = arg[3:]
            elif arg.startswith("dir="):
                buildDir = arg[4:]
            else:
                usage("Unknown %s argument"%arg)
                sys.exit(1)

        # create config if not there
        if not os.path.isfile(CONFIG_FILE_NAME):
            config()
        # HACK: re-read it from file to resolve paths
        configDict = readConfigFile(CONFIG_FILE_NAME)
        ok = (checkConfig()
            and getDependencies()
            and getSources(branch)
            and compile(Nproc)
            and install('build'))
        if ok:
           sys.exit(0)
        else:
           sys.exit(1)
    elif mode=="create_devel_paths":
        if n==3:
            dir=sys.argv[2]
        else:
            dir="."
        writeDevelPaths(dir)
    elif mode=="git":
        runJob("(cd src/xmipp; git %s)"%" ".join(sys.argv[2:]))
    elif mode=="gitConfig":
        runJob("sed -i 's/https:\/\/github.com\//git@github.com:/g' src/xmipp/.git/config")
    elif mode=='tar':
        if len(sys.argv) < 3:
            runJob("src/xmipp/tar.py --help")
            sys.exit(0)
        ver = XMIPP_VERSION
        br = 'master'
        mode = sys.argv[2]
        for arg in sys.argv[3:]:
            if arg.startswith('br='):
                br = arg.split('br=')[1]
            if arg.startswith('v='):
                ver = arg.split('v=')[1]
        runJob("src/xmipp/tar.py %s %s %s" % (mode, ver, br))

    else:
        usage(" -> option not found <- \n")