#!/usr/bin/perl -w
####################################################################################################
#
# Helper script for Qt 5
#
# Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies).
# Contact: http://www.qt-project.org/legal
#
####################################################################################################

############################################################################################
#
# Convenience script working with a Qt 5 repository.
#
# Feel free to add useful options!
#
# To keep this script working with older Perl versions, please follow these guidelines:
# - Variables should not shadow others.
############################################################################################

use strict;

use Getopt::Long qw(:config no_ignore_case);
use File::Basename;
use File::Spec;
use File::Copy;
use POSIX;
use IO::File;
use File::Path;

my $CLEAN=0;
my $PULL=0;
my $BUILD=0;
my $MAKE=0;
my $RESET=0;
my $DIFF=0;
my $BOOTSTRAP=0;
my $STATUS=0;
my $UPDATE=0;
my $TEST=0;
my $REBUILD_CONFIGURE=0;
my $BUILD_WEBKIT = 0;
my $optModuleBranchArgument;
my $optGerritModule;
my $optGitHooks;
my $optDesiredBranch;
my $gitoriousURL = 'git://gitorious.org/qt/qt5.git';
my $codeReviewHost = 'codereview.qt-project.org';
my $codeReviewPort = 29418;
my $configFile;

my $USAGE=<<EOF;
Usage: qt5_tool [OPTIONS]

Utility script for working with Qt 5 modules.

Feel free to extend!

Options:
  -d  Diff (over all modules, relative to root)
  -s  Status
  -r  Reset hard to upstream state
  -c  Clean
  -u  Update
  -p  Pull (for development only)
    --Branch (dev/stable) to select which branches to check out when pulling
  -b  Build (configure + build)
  -m  Make
  -t  Test: Builds and runs tests located in tests/auto for each module,
      creates log file(s) in that directory as well as a summary log  file.
  -q  Quick bootstrap a new checkout under current folder.
  -w  Build WebKit
  -g <module> Set up gerrit for the module.

Example use cases:
  qt5_tool -c -u -b             Clean, update and build for nightly builds
  qt5_tool -d                   Generate modules diff relative to root directory
  qt5_tool -r                   Reset repository to upstream state (hard).
  qt5_tool -p --Branch stable   Check out all repos to the stable branches

qt_tool can be configured by creating a configuration file
"%CONFIGFILE%"
in the format key=value. It is possible to use repository-specific values
by adding a key postfixed by a dash and the repository folder base name.
Use 'true', '1' for Boolean keys.
Supported keys:

developerBuild:     Developer build (Boolean)
initArguments:      Arguments to init-repository for -q.
codeReviewUser:     User name for code review (Gerrit)
Branch:             'dev', 'stable' or other
configureArguments: Arguments to configure
shadowBuildPostfix: Postfix to use for shadow build directory.

Example:
shadowBuildPostfix=-build
shadowBuildPostfix-qt-5special=-special-build

specifies that for a checkout in '/home/user/qt-5', shadow builds are to be
done in '/home/user/qt-5-build'. It is overridden by a value for the checkout
'/home/user/qt-5special', which would result in '/home/user/qt-5-special-build'
EOF

# Empty entry means: Do not touch.
my %preferredBranches = (
    'qtrepotools' => 'master',
#   -- Unmaintained modules as of 4.12.2012. Qt3D currently only has 'dev'
    'qtfeedback' => 'master',
    'qtpim' => 'master',
    'qtqa' => 'master',
);

# --------------- Detect OS

my ($OS_LINUX, $OS_WINDOWS, $OS_MAC)  = (0, 1, 2);
my $os = $OS_LINUX;
if (index($^O, 'MSWin') >= 0) {
    $os = $OS_WINDOWS;
} elsif (index($^O, 'darwin') >= 0) {
   $os = $OS_MAC;
}

# -- Convenience for path search.
#    There is File::Which, but not by default installed on Linux.

sub which
{
    my ($needle) = @_;
    my $sep = $os == $OS_WINDOWS ? ';' : ':';
    my @needles = ($needle);
    push(@needles, $needle . '.exe', $needle . '.bat', $needle . '.cmd') if $os == $OS_WINDOWS;
    foreach my $path (split(/$sep/, $ENV{'PATH'})) {
        foreach my $n (@needles) {
            my $binary = File::Spec->catfile($path, $n);
            return $binary if (-f $binary);
        }
    }
    return undef;
}

# -- Locate an utility (grep, scp, etc) in MSYS git. This is specifically
#    for the setup case in which only git from the cmd folder and not the utilities
#    are in the path. We then look at the git and return ..\bin\<utility>.exe.
sub msysGitUtility
{
#   -- Look for 'cmd/git(.exe,.cmd)' and cd ..\bin. Note that as of msygit 1.8, git.exe
#      is in cmd.
    my ($git, $utility) = @_;
    my $gitBinDir = dirname($git);
    if ($gitBinDir =~ /cmd$/i) {
        my $msysGitBinFolder = File::Spec->catfile(dirname($gitBinDir), 'bin');
        return File::Spec->catfile($msysGitBinFolder, $utility . '.exe');
    }
    return $utility;
}

my $qmakeSpec = $ENV{'QMAKESPEC'};
my $make = 'make';
my @makeArgs = ('-s');
my $makeForceArg = '-k';
my $minGW = 0;

if ($os == $OS_WINDOWS) {
    $minGW = defined($qmakeSpec) && index($qmakeSpec,'g++') > 0;
    if ($minGW) {
        $make = 'mingw32-make';
    } else {
        @makeArgs = ('/s', '/l');
        $makeForceArg = '/k';
        my $jom = which('jom');
        if (defined $jom) {
            $make = $jom;
        } else {
            $make = 'nmake';
            # Switch cl compiler to multicore
            my $oldCL = $ENV{'CL'};
            if (defined $oldCL) {
                $ENV{'CL'} = $oldCL . ' /MP'
            } else {
                $ENV{'CL'} = '/MP'
            }
        } # jom
    } # !MinGW
}

my $git = which('git'); # TODO: Mac, Windows special cases?
die ('Unable to locate git') unless defined $git;

my $rootDir = '';
my $baseDir = '';
my $home = $os == $OS_WINDOWS ? ($ENV{'HOMEDRIVE'} . $ENV{'HOMEPATH'}) : $ENV{'HOME'};

# -- Set a HOME variable on Windows such that scp. etc. feel at home (locating .ssh).
$ENV{'HOME'} = $home if ($os == $OS_WINDOWS && not defined $ENV{'HOME'});

my $user = $os == $OS_WINDOWS ? $ENV{'USERNAME'} : $ENV{'USER'};

# -- Determine configuration file (~/.config if present).
if ($os == $OS_WINDOWS) {
    $configFile = File::Spec->catfile($ENV{'APPDATA'}, 'qt5_tool.conf');
} else {
    my $configFolder = File::Spec->catfile($home, '.config');
    $configFile = -d $configFolder ?
                  File::Spec->catfile($configFolder, 'qt5_tool.conf') :
                  File::Spec->catfile($home, '.qt5_tool');
}
$USAGE =~ s/%CONFIGFILE%/$configFile/; # Replace placeholder.

my @MODULES = ();

my ($developerBuildConfigKey, $branchConfigKey) = ('developerBuild', 'branch');

# --- Execute a command and print to log.
sub execute
{
    my @args = @_;
    print '### [',basename(getcwd()),'] ', join(' ', @args),"\n";
    return system(@args);
}

sub executeCheck
{
    my @args = @_;
    my $rc = execute(@args);
    die ($args[0] . ' failed ' . $rc . ':' . $!) if $rc != 0;
}

# --- Prompt for input with a default

sub prompt
{
    my ($promptText, $defaultValue) = @_;
    print $promptText,' [', $defaultValue, ']:';
    my $userInput = <STDIN>;
    chomp ($userInput);
    return $userInput eq '' ? $defaultValue : $userInput;
}

# --- Fix a diff line from a submodule such that it can be applied to
#     the root Qt 5 directory, that is:
#     '--- a/foo...' -> '--- a/<module>/foo...'

sub fixDiff
{
   my ($line, $module) = @_;
   if (index($line, '--- a/') == 0 || index($line, '+++ b/') == 0) {
       return substr($line, 0, 6) . $module . '/' . substr($line, 6);
   }
   if (index($line, 'diff --git ') == 0) {
       $line =~ s| a/| a/$module/|;
       $line =~ s| b/| b/$module/|;
   }
   return $line;
}

# --- Determine a suitable log file name for test log files.

sub formatTestLogBaseName
{
    my ($number, $module) = @_;
    my $result = 'test';
    $result .= '_win' if $os == $OS_WINDOWS;
    $result .= '_unix' if $os == $OS_LINUX;
    $result .= '_mac' if $os == $OS_MAC;
    if (defined $module) {
        $result .= '_';
        $result .=  index($module, 'qt') == 0 ? substr($module, 2) : $module;
    }
    $result .= '_' . strftime('%y%m%d', localtime());
    $result .= '_' . $number if $number > 0;
    return $result;
}

# ---- Generate a diff from all submodules such that it can be applied to
#      the root Qt 5 directory.

sub diff
{
    my $totalDiff = '';
    my ($rootDir,$modArrayRef) = @_;
    foreach my $MOD (@$modArrayRef) {
     chdir($MOD) or die ('Failed to chdir from' . $rootDir . ' to "' . $MOD . '":' . $!);
     my $diffOutput = `"$git" diff`;
     foreach my $line (split(/\n/, $diffOutput)) {
         chomp($line);
         $totalDiff .= fixDiff($line, $MOD);
         $totalDiff .= "\n";
     }
     chdir($rootDir);
  }
  return $totalDiff;
}

# ---- runGit() Run git in the current directory.

my $RUN_GIT_FAILMODE_EXIT = 0;
my $RUN_GIT_FAILMODE_IGNORE = 1;
my $RUN_GIT_FAILMODE_RETRY = 2;

sub runGit
{
    my ($argListRef, $failMode, $module) = @_;
    $failMode = $RUN_GIT_FAILMODE_EXIT unless defined $failMode;
    $module = '<root>' unless defined $module;
    my $exitCode = 1;
    for (my $r = 0; $r < 3; ++$r) {
        $exitCode = execute($git, @$argListRef);
        last if !$exitCode || $failMode != $RUN_GIT_FAILMODE_RETRY;
        warn('### Retrying ' . join(' ', @$argListRef) . ' in ' .  $module);
    }
    if ($exitCode) {
        my $reportString = join(' ', @$argListRef) . ' failed in ' . $module . '.';
        die ($reportString) if $failMode != $RUN_GIT_FAILMODE_IGNORE;
        warn ($reportString);
   }
   return $exitCode;
}

# ---- runGitAllModules() Run git in root and module folders.
#      Do not use 'git submodules foreach' to be able to work on
#      partially corrupt repositories.

sub runGitAllModules
{
    my ($argListRef, $failMode) = @_;
    $failMode = $RUN_GIT_FAILMODE_EXIT unless defined $failMode;
    print 'Running ', join(' ', @$argListRef), "\n";

    my $runRC = runGit($argListRef, $failMode);
    foreach my $MOD (@MODULES) {
        chdir($MOD) or die ('Failed to chdir from' . $rootDir . ' to "' . $MOD . '":' . $!);
        my $modRunRC = runGit($argListRef, $failMode, $MOD);
        $runRC = 1 if $modRunRC != 0;
        chdir($rootDir);
    }
    return $runRC;
}

# ---- Read a value from a configuration file of the form key=value.

sub readConfigFile
{
    my ($fileName, $key) = @_;

    my $configLine = '';
    my $configFile = new IO::File('<' . $fileName) or return $configLine;
    while (my $line = <$configFile>) {
        chomp($line);
        if ($line =~ /^\s*$key\s*=\s*(.*)$/) {
           $configLine .= $1;
           last;
        }
    }
    $configFile->close();
    return $configLine;
}

# ---- Read a value from a git config line.

sub readGitConfig
{
    my ($module, $key) = @_;
    return readConfigFile(File::Spec->catfile($rootDir, $module, '.git', 'config'), $key);
}

# ---- MinGW: Remove git from path (prevent sh.exe from throwing off mingw32-make).

sub winRemoveGitFromPath
{
    my @path = split(/;/, $ENV{'PATH'});
    my @cleanPath = grep(!/git/, @path);
    if (@path != @cleanPath) {
        print 'Removing git from path...';
        $ENV{'PATH'} = join(';', @cleanPath);
    }
}

# ---- Set up a tracking branch
sub initTrackingBranch
{
    my ($branchName, $remoteBranchName) = @_;
    my $strc = execute($git, ('fetch', '--all'));
    die 'fetch failed.' if $strc;
    print 'Switching to ', $branchName, ' from ', $remoteBranchName, "\n";
    $strc = execute($git, ('branch', '--track', $branchName, $remoteBranchName));
    die 'branch ' . $branchName . ' ' . $remoteBranchName . ' failed.' if $strc;
    $strc = execute($git, ('checkout', $branchName));
    die 'checkout failed.' if $strc;
}

# ---- Set 'MAKEFLAGS' which depends on OS
sub setMakeEnvironment
{
    my ($make, @makeArgs) = @_;
    my $makeFlags = $ENV{"MAKEFLAGS"};
    $makeFlags = '' unless defined $makeFlags;
    if ($make eq 'nmake' || $make eq 'jom') {
        # Windows: MAKEFLAGS=SLK
        for my $arg (@makeArgs) {
            $makeFlags .= substr($arg, 1);
        }
    } else {
        # UNIX: Concatenate
        $makeFlags .= ' ' unless $makeFlags eq '';
        $makeFlags .= join(' ', @makeArgs);
    }
    print 'Setting environment for ', $make, " '", $makeFlags, "'\n";
    $ENV{"MAKEFLAGS"} = $makeFlags;
}

# ----- Create ls -l like listing for a file name
sub ls
{
    my ($fileName) = @_;
    my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size, $atime,$mtime,$ctime,$blksize,$blocks) = stat($fileName);
    return $fileName . ' not found' unless defined $dev;
    return $fileName . ' ' . $size . ' ' . scalar(localtime($mtime));
}

# ---- Read a value from the '$HOME/.qt5_tool' configuration file
#      When given a key 'key' for the repository directory '/foo/qt-5',
#      check for the repo-specific value 'key-qt5' and then for the general
#      'key'.

sub readQt5ToolConfig
{
    my ($key) = @_;
    my $repoKey = $key . '-' . $baseDir;
    my $repoValue = readConfigFile($configFile, $repoKey);
    return $repoValue if $repoValue ne '';
    return readConfigFile($configFile, $key);
}

sub readQt5ToolConfigBool
{
   my ($key) = @_;
   my $value = readQt5ToolConfig($key);
   return (length($value) > 0 && ($value eq '1' || $value =~ /true/i)) ? 1 : 0;
}

sub shadowBuildFolder
{
    my $shadowBuildPostfix = readQt5ToolConfig('shadowBuildPostfix');
    return $shadowBuildPostfix ne '' ? $rootDir . $shadowBuildPostfix : '';
}

# ---- Check for absolute path names.

sub isAbsolute
{
    my ($file) = @_;
    return index($file, ':') == 1 if ($os == $OS_WINDOWS);
    return index($file, '/') == 0;
}

# ---- Determine modules by trying to find <module>/.git/config.
sub determineModules
{
    my ($rootFolder) = @_;
    opendir (DIR, $rootFolder) or die ('Cannot read ' . $rootFolder . $!);
    my @mods = ();
    while (my $e = readdir(DIR)) {
        if ($e ne '.' && $e ne '..' && -d $e && $e ne 'qtquick3d') {
            my $gitFolder = File::Spec->catfile($e, '.git');
            push(@mods, $e) if -e $gitFolder;
       }
    }
    closedir(DIR);
    die ('Failed to detect modules in ' . $rootFolder . ".\nNeeds to be called from the root directory.") if @mods == 0;
    # Sort & Put a (potentially failing) webkit last.
    my @nonWebKit = sort(grep(!/^qtwebkit$/, @mods));
    my @webKit = grep(/^qtwebkit$/, @mods);
    return (@nonWebKit, @webKit)
}

# ---- Helper to be called before pull. Checks if
#      the module is in a 'no branch' state after init-repository or in the wrong
#      branch as from the configuration file.
#      If so, check out a branch, checking preferredBranches hash.

sub checkoutBranch
{
    my ($mod, $desiredBranchIn) = @_;
    # -- Special treatment for webkit: Do not touch.
    my $desiredBranch = $preferredBranches{$mod};
    $desiredBranch = $desiredBranchIn unless defined $desiredBranch;
    if ($desiredBranch eq '') {
       print 'Skipping ', $mod, ".\n";
       return 0;
    }
    # Determine branch
    my @branches = split("\n", `"$git" branch`);
    my @currentBranches = grep(/^\* /, @branches);
    die ('Unable to determine branch of ' . $mod) if @currentBranches != 1;
    my $currentBranch = substr($currentBranches[0], 2);
    #  We have one, no need to act.
    if ($currentBranch eq $desiredBranch) {
        print $mod, ' is already on branch: "',$currentBranch,"\".\n";
        return 1;
    }
    # Does the local branch exist?
    my $rc = 0;
    if (!grep(/^  $desiredBranch$/, @branches)) {
        my $remoteBranchName = 'origin/' . $desiredBranch;
        # Does the remote branch exist?
        $rc = execute($git, ('fetch', '--all'));
        die 'fetch of ' . $mod . ' failed'  if ($rc);
        my @availableRemoteBranches = split("\n", `"$git" branch -r`);
        if (!grep(/^  $remoteBranchName$/, @availableRemoteBranches)) {
            print $mod, ' does not have the remote branch of "', $remoteBranchName, "\" set up, skipping.\n";
            return 0;
        }
        $rc = execute($git, ('branch', '--track', $desiredBranch, $remoteBranchName));
        die 'Creation of ' . $desiredBranch . ' tracking ', $remoteBranchName, ' failed'  if ($rc);
    }
    print 'switching ',$mod, ' from "', $currentBranch,'" to "',$desiredBranch,"\".\n";
    $rc = execute($git, ('checkout', $desiredBranch));
    die 'Checkout of ' . $desiredBranch . ' failed'  if ($rc);
    return 1;
}

# --------------- MAIN: Parse arguments

$Getopt::ignoreCase = 0;
if (!GetOptions('clean' => \$CLEAN,
     'pull' => \$PULL, 'Branch=s' => \$optDesiredBranch, 'update' => \$UPDATE, 'reset' => \$RESET, 'diff' => \$DIFF, 's' => \$STATUS,
     'build' => \$BUILD, 'make' => \$MAKE, 'test' => \$TEST,
     'acking=s' => \$optModuleBranchArgument, 'gerrit=s' => \$optGerritModule, 'hooks' => \$optGitHooks,
     'quick-bootstrap'  => \$BOOTSTRAP,
     'webkit'  => \$BUILD_WEBKIT, 'x' => \$REBUILD_CONFIGURE)
    || ($CLEAN + $PULL + $UPDATE + $BUILD + $MAKE + $RESET + $DIFF + $BOOTSTRAP + $STATUS
        + $REBUILD_CONFIGURE + $TEST + $BUILD_WEBKIT== 0
        && ! defined $optModuleBranchArgument && !defined $optGerritModule && !defined $optGitHooks)) {
    print $USAGE;
    exit (1);
}

sub defaultConfigureArguments
{
    my ($developerBuild) = @_;
    my $result = '-confirm-license';
    $result .= ' -developer-build' if ($developerBuild);
    $result .= ' -opensource -debug';
#   On Mac, -debug requires -no-framework (or use -debug-and-release)?
    $result .= ' -no-framework' if  $os == $OS_MAC;
    $result .= ' -xcb' if $os == $OS_LINUX && defined $ENV{'DISPLAY'};
    $result .= ' -nomake tests -nomake examples';
    return $result;
}

#   -- Prompt to create configuration file for options that require it.
if (($BOOTSTRAP + $BUILD != 0 || defined $optGerritModule) && ! -f $configFile) {
    print "\n### This appears to be the first use of qt5_tool on this machine.\n\nCreating configuration file '",$configFile, "'...\n";
    my $newDeveloperBuild = prompt('Developer build (y/n)', 'y') =~ /y/i;
    my $newCodeReviewUser = '';
    my $newInitArgumentsDefault = '--no-webkit';
    $newCodeReviewUser = prompt('Enter CodeReview user name', $user) if ($newDeveloperBuild);
    my $newInitArguments = prompt('Enter arguments to init-repository', $newInitArgumentsDefault);
    my $newConfigureArguments = prompt('Enter arguments to configure', defaultConfigureArguments($newDeveloperBuild));
    my $newBranch = prompt('Branch', 'dev');
    my $configFileHandle = new IO::File('>' . $configFile) or die ('Unable to write to ' . $configFile . ':' . $!);
    print $configFileHandle 'configureArguments=', $newConfigureArguments, "\n" if $newConfigureArguments ne '';
    print $configFileHandle 'initArguments=',$newInitArguments, "\n" if $newInitArguments ne '';
    print $configFileHandle 'codeReviewUser=', $newCodeReviewUser,"\n" if $newCodeReviewUser ne '';
    print $configFileHandle $branchConfigKey, '=', $newBranch,"\n";
    print $configFileHandle $developerBuildConfigKey, "=true\n" if $newDeveloperBuild;
    $configFileHandle->close();
    print "Wrote '",$configFile, "'\n\n";
}

#  --- read config file
my $codeReviewUser = readQt5ToolConfig('codeReviewUser');

# --------------- Bootstrap

if ( $BOOTSTRAP != 0 ) {
    my $developerBuild = readQt5ToolConfigBool($developerBuildConfigKey);
    my $targetFolder = prompt('Enter target folder', 'qt-5');
    my @initOptions;
    push (@initOptions, '--codereview-username=' . $codeReviewUser) if $codeReviewUser ne '';
    my $initArgumentsFromConfig = readQt5ToolConfig('initArguments');
#   -- Webkit is usually too slow to clone unless something is configured.
    if ($initArgumentsFromConfig ne '') {
        push(@initOptions, split(/ /, $initArgumentsFromConfig));
    }
    my $toolsFolder = 'qtrepotools';
    # -- Clone
    my $cloneRc = execute($git, ('clone',  $gitoriousURL, $targetFolder));
    die 'clone '.  $gitoriousURL . ' failed.' if $cloneRc;
    chdir($targetFolder) or die ('Failed to chdir to "' . $targetFolder . '":' . $!);
    # -- Run init
    my $initRc = 0;
    if ($os == $OS_WINDOWS) {
        $initRc = execute('perl', 'init-repository', @initOptions);
    } else {
        $initRc = execute('./init-repository', @initOptions);
    }
    die 'init '. $gitoriousURL . ' failed.' if $initRc;
    exit 0;
}

# --- Change to root: Assume we live in qtrepotools below root.
#     Note: Cwd::realpath is broken in the Symbian-perl-version.
my $prog = $0;
$prog = Cwd::realpath($0) unless isAbsolute($prog);
$rootDir = dirname(dirname(dirname($prog)));
$baseDir = basename($rootDir);
chdir($rootDir) or die ('Failed to chdir to' . $rootDir . '":' . $!);
my $exitCode = 0;

@MODULES = determineModules($rootDir);

print diff($rootDir, \@MODULES) if $DIFF;

# --------------- Reset: Save to a patch in HOME dir indicating date in
#                 file name should there be a diff.
if ( $RESET !=  0 ) {
  print 'Resetting Qt 5 in ',$rootDir,"\n";
  my $changes = diff($rootDir, \@MODULES);
  if ($changes ne '') {
     my $patch = File::Spec->catfile($home, POSIX::strftime('qt5_d%Y%m%d%H%M.patch',localtime));
     my $patchFile = new IO::File('>' . $patch) or die ('Unable to open for writing ' .  $patch . ' :' . $!);
     print $patchFile $changes;
     $patchFile->close();
     print 'Saved ', $patch, "\n";
  }
  my @resetArgs = ('reset', '--hard', '@{upstream}');
  runGitAllModules(\@resetArgs, $RUN_GIT_FAILMODE_IGNORE);
}

# --------------- Status.
if ( $STATUS !=  0 ) {
    my @branchArgs = ('branch', '-v');
    runGitAllModules(\@branchArgs, $RUN_GIT_FAILMODE_IGNORE);
    my @statusArgs = ('status');
    runGitAllModules(\@statusArgs, $RUN_GIT_FAILMODE_IGNORE);
}

#   --------------- TEST: Run auto-tests in relevant modules
#   and create a summary log, and optionally, if the converter can be
#   found, a tasks file for Qt Creator..

if ( $TEST !=  0 ) {
    my %testPaths;
    my $testRelativePath = File::Spec->catfile('tests', 'auto');
    my $catCommand = $os == $OS_WINDOWS ? 'type' : 'cat';
#   ---  Find relevant modules.
    foreach my $MOD (@MODULES) {
        my $excluded = ($MOD eq 'qtactiveqt'   && $os != $OS_WINDOWS)
                       || ($MOD eq 'qtjsondb'  && $os == $OS_WINDOWS)
                       || ($MOD eq 'qtwayland' && $os == $OS_WINDOWS);
        if (!$excluded) {
            my $testPath = File::Spec->catfile($MOD, $testRelativePath);
            $testPaths{$MOD} = $testPath if -d $testPath;
        }
    }
#   -- Locate 'test2tasks.pl'. This is a script located in the Qt Creator repository
#      that converts test output into task files that can be loaded into Qt  Creator's
#      'Build issues' pane.
    my $test2tasks = which('test2tasks.pl');
#   -- Initialize logging, find a unique log file name.
    my $testLogUniqueNumber = 0;
    my ($summaryLogFile, $summaryTasksFile);
    for ( ; ; $testLogUniqueNumber++) {
        my $testLogBaseName = formatTestLogBaseName($testLogUniqueNumber);
        $summaryLogFile = File::Spec->catfile($rootDir, $testLogBaseName . '.log');
        $summaryTasksFile = File::Spec->catfile($rootDir, $testLogBaseName . '.tasks');
        last unless -f $summaryLogFile || -f $summaryTasksFile;
    }
    if (-e $summaryTasksFile) {
        unlink($summaryTasksFile) or die ('Unable to delete existing tasks file ' . $summaryTasksFile . ' ' . $!);
    }
    print '### Testing ', scalar(keys(%testPaths)), ' modules: ', join(', ', keys(%testPaths)), "\n### Logging to: ",
          $summaryLogFile, "\n";
#   -- Build the tests, qmake, make
    my @buildFailures;
    my $tn = 0;
    foreach my $testPath (values(%testPaths)) {
        chdir($testPath) || die ('Failed to chdir from' . $rootDir . ' to "' . $testPath);
        print '### Building #', ++$tn, ' of ', scalar(keys(%testPaths)), ' ', $testPath, "\n";
        my $rc = execute('qmake');
        die ('qmake failed') unless $rc == 0;
        $rc = execute($make, @makeArgs);
        if ($rc != 0) {
            print '### Warning: Build failed in ', $testPath, "\n";
            push (@buildFailures, $testPath);
        }
        chdir($rootDir);
    }
#   -- Run the test: 'make check'. Explicitly set 'Keep' on Windows as flags are not propagated.
    $tn = 0;
    $ENV{'MAKEFLAGS'} = 'K' if  $make eq 'nmake';
    foreach my $module (keys(%testPaths)) {
        my $testPath = $testPaths{$module};
        ++$tn;
        if (grep (/$testPath/, @buildFailures)) {
            print '### Skipping ', $testPath, "\n";
            next;
        }
        chdir($testPath) || die ('Failed to chdir from' . $rootDir . ' to "' . $testPath);
        print '### Running #', ++$tn, ' of ', scalar(keys(%testPaths)), ' ', $testPath, "\n";
        my $moduleLogBaseName = formatTestLogBaseName($testLogUniqueNumber, $module);
        my $moduleLogFile = File::Spec->catfile(getcwd(), $moduleLogBaseName . '.log');
        my $moduleTasksFile = File::Spec->catfile(getcwd(),  $moduleLogBaseName . '.tasks');
        my $cmd = $make . ' ' . join(' ', @makeArgs) . ' ' . $makeForceArg . ' check > ' .  $moduleLogFile;
        print '### Running: ', $cmd, "\n";
        system($cmd);
#       -- concat log file.
        $cmd = $catCommand . ' ' . $moduleLogFile . ' >> ' . $summaryLogFile;
        print '### Running: ', $cmd, "\n";
        system($cmd);
#       --- Create a tasks file for Qt Creator
        if (defined $test2tasks) {
#           -- Local file with relative paths
                $cmd = $test2tasks . ' < ' . $moduleLogFile . ' > ' . $moduleTasksFile;
            print '### Running: ', $cmd, "\n";
                   system($cmd);
#           -- Append to summary file with path relative to its location
            $cmd = $test2tasks . ' -r ' . $testPath . ' < ' . $moduleLogFile . ' >> ' . $summaryTasksFile;
            print '### Running: ', $cmd, "\n";
            system($cmd);
        }
        chdir($rootDir);
    }
    my $report = "\n### Testing done:\n";
    $report .= '- Build failures: ' . join(', ', @buildFailures) . "\n" if (scalar(@buildFailures));
    $report .= '- ' . ls($summaryLogFile) . "\n";
    $report .= '- ' . ls($summaryTasksFile) . "\n" if defined $test2tasks;
    print $report;
}

# --------------- Init gerrit

if (defined $optGerritModule) {
    my $scp = which('scp');
    $scp = msysGitUtility($git, 'scp') unless defined $scp || $os != $OS_WINDOWS;
    die ('Unable to locate scp.') unless defined $scp;
    die ('This option requires a codereview-user name') if $codeReviewUser eq '';
    chdir($optGerritModule) or die ('Failed to chdir from' . $rootDir . ' to "' . $optGerritModule);
    my $remoteRepo = $codeReviewUser . '@' . $codeReviewHost . ':qt/' . $optGerritModule;
    print 'Configuring ', $remoteRepo, "\n";
    my $grc = execute($git, ('config', 'remote.origin.url',  $remoteRepo));
    die 'Config failed'  if ($grc);
    print 'Initializing hooks', "\n";
    $grc = execute($scp, ('-p',  $codeReviewUser . '@' . $codeReviewHost . ':hooks/commit-msg', '.git/hooks'));
    die 'Copying of commit hook failed'  if ($grc);
    print 'Fetch all...', "\n";
    $grc = execute($git, ('fetch', '--all'));
    chdir($rootDir);
}

if (defined $optGitHooks) {
    my $qtrepotoolsdir = dirname(dirname($prog));
    my $postcommitpath = File::Spec->canonpath("$qtrepotoolsdir/git-hooks");
    if ($os == $OS_WINDOWS) {
        # rewrite from "C:/Users/qt" to "/C/Users/qt" so that msysgit understand
        $postcommitpath =~ s,([A-Za-z]):,/$1,;
        $postcommitpath =~ s,\\,/,g;
    }
    print "Installing post-commit hooks\n";
    foreach my $MOD (@MODULES) {
        print 'Examining: ', $MOD, ' url: ',readGitConfig($MOD, 'url'), ' ';
        chdir($MOD) or die ('Failed to chdir from' . $rootDir . ' to "' . $MOD . '":' . $!);
        my $postCommitFile = '.git\hooks\post-commit';
        my $overwrite = 1;
        if (-f $postCommitFile) {
            $overwrite = prompt('Overwrite existing post-commit file? (y/n)', 'n') =~ /y/i;
        }
        if ($overwrite) {
            my $postCommitFileHandle = new IO::File('>' . $postCommitFile) or die ('Unable to write to ' . $postCommitFile . ':' . $!);
            print $postCommitFileHandle "#!/bin/sh\n";
            print $postCommitFileHandle "export PATH=\$PATH:$postcommitpath\n"; #needed
            print $postCommitFileHandle "exec $postcommitpath/git_post_commit_hook\n";
            $postCommitFileHandle->close();
        }
        chdir($rootDir);
    }
}

# --------------- Clean if desired

if ( $CLEAN !=  0 ) {
  print 'Cleaning Qt 5 in ',$rootDir,"\n";
  my @cleanArgs = ('clean','-dxfq');
  runGitAllModules(\@cleanArgs, $RUN_GIT_FAILMODE_RETRY);
}

# ---- Pull: Switch to branch unless there is one (check preferred
#      branch hash, default to branch n+1, which is mostly master).

if ( $PULL !=  0 ) {
    print 'Pulling Qt 5 in ',$rootDir,"\n";
    my $desiredBranch = defined $optDesiredBranch ? $optDesiredBranch : readQt5ToolConfig($branchConfigKey);
    if (!defined $desiredBranch || $desiredBranch eq '') {
        $desiredBranch = 'dev';
        print 'No branch has been set up in ', $configFile, ' (key: ', $branchConfigKey, '), assuming ', $desiredBranch, "\n";
    }
    my @pullArgs = ('pull');
    my $prc = runGit(\@pullArgs, $RUN_GIT_FAILMODE_RETRY);
    die 'Pull failed'  if ($prc);
    foreach my $MOD (@MODULES) {
        print 'Examining: ', $MOD, ' url: ',readGitConfig($MOD, 'url'), ' ';
        chdir($MOD) or die ('Failed to chdir from' . $rootDir . ' to "' . $MOD . '":' . $!);
        if (checkoutBranch($MOD, $desiredBranch)) {
            print 'Pulling ', $MOD, "\n";
            $prc = runGit(\@pullArgs, $RUN_GIT_FAILMODE_RETRY, $MOD);
            die 'Pull ' . $MOD . ' failed'  if ($prc);
        }
        chdir($rootDir);
   } # foreach
} # pull

# --------------- Rebuild configure.exe
if ( $REBUILD_CONFIGURE !=  0 ) {
    rebuildConfigure($rootDir);
}

# ---- Update

if ( $UPDATE !=  0 ) {
    print 'Updating Qt 5 in ',$rootDir,"\n";
    my $urc = execute($git, ('pull'));
    die 'pull failed'  if ($urc);
    $urc = execute($git, ('submodule', 'update'));
    die 'update failed'  if ($urc);
}

# ---- Configure and build

my $makeInstallRequired = 0;

if ( $BUILD !=  0 ) {
    print 'Building Qt 5 in ',$rootDir,"\n";
    winRemoveGitFromPath() if $minGW;
    my @configureArguments;
    my $configureArgumentsFromConfig = readQt5ToolConfig('configureArguments');
    push(@configureArguments, split(/ /, $configureArgumentsFromConfig)) unless $configureArgumentsFromConfig eq '';
    if (!$BUILD_WEBKIT) {
        push(@configureArguments, "-skip");
        push(@configureArguments, "qtwebkit");
    }
    $makeInstallRequired = grep(/^-prefix$/, @configureArguments);
#   --- Shadow builds: Remove and re-create directory
    my $shadowBuildDir = shadowBuildFolder();
    if ($shadowBuildDir ne '') {
        print 'Shadow build: "', $shadowBuildDir,"\"\n";
        if (-d $shadowBuildDir) {
            File::Path::rmtree($shadowBuildDir) or die ('Unable to remove ' . $shadowBuildDir . ' :' . $!);
        }
        mkdir($shadowBuildDir) or die  ('Unable to create ' . $shadowBuildDir . ' :' . $!);
        chdir($shadowBuildDir) or die  ('Unable to chdir ' . $shadowBuildDir . ' :' . $!);
    }
#   ---- Configure and build
    my $brc = execute(File::Spec->catfile($rootDir, 'configure'), @configureArguments);
    die 'Configure failed'  if ($brc);
} # BUILD

if ( $BUILD + $MAKE !=  0) {
    my $makeShadowBuildDir = shadowBuildFolder();
    if ($BUILD == 0) { # Did not go through configure, cd
        if ($makeShadowBuildDir ne '') {
            print 'Shadow build: "', $makeShadowBuildDir,"\"\n";
            chdir($makeShadowBuildDir) or die  ('Unable to chdir ' . $makeShadowBuildDir . ' :' . $!);
        }
    }
#   Run a global make for non-shadow developer build, else call 'build'.
    executeCheck($make, @makeArgs);
} # MAKE

if ( $BUILD && $makeInstallRequired ) {
    print 'Installing Qt 5 from ',$rootDir,"\n";
    my @installArgs = @makeArgs;
    # Turn of parallelization for make install, which is known to fail with jom.
    push (@installArgs, '-j', '1') unless (index($make, 'nmake') >= 0);
    push (@installArgs, 'install');
    executeCheck($make, @installArgs);
}

exit($exitCode);
