path: root/tests/prebuild/license/tst_licenses.pl

#!/usr/bin/env perl
#############################################################################
##
## Copyright (C) 2012 Nokia Corporation and/or its subsidiary(-ies).
## Contact: http://www.qt-project.org/
##
## This file is part of the test suite of the Qt Toolkit.
##
## $QT_BEGIN_LICENSE:LGPL$
## GNU Lesser General Public License Usage
## This file may be used under the terms of the GNU Lesser General Public
## License version 2.1 as published by the Free Software Foundation and
## appearing in the file LICENSE.LGPL included in the packaging of this
## file. Please review the following information to ensure the GNU Lesser
## General Public License version 2.1 requirements will be met:
## http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
##
## In addition, as a special exception, Nokia gives you certain additional
## rights. These rights are described in the Nokia Qt LGPL Exception
## version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
##
## GNU General Public License Usage
## Alternatively, this file may be used under the terms of the GNU General
## Public License version 3.0 as published by the Free Software Foundation
## and appearing in the file LICENSE.GPL included in the packaging of this
## file. Please review the following information to ensure the GNU General
## Public License version 3.0 requirements will be met:
## http://www.gnu.org/copyleft/gpl.html.
##
## Other Usage
## Alternatively, this file may be used in accordance with the terms and
## conditions contained in a signed written agreement between you and Nokia.
##
##
##
##
##
##
## $QT_END_LICENSE$
##
#############################################################################

use 5.010;
use strict;
use warnings;
use utf8;
use File::Find;
use File::Basename;
use File::Spec::Functions;
use Cwd qw( abs_path getcwd );
use List::Util qw( first );
use Test::More;

=head1 NAME

tst_licenses.pl - verify that source files contain valid license headers

=head1 SYNOPSIS

  perl ./tst_licenses.pl

This test expects the environment variable QT_MODULE_TO_TEST to contain
the path to the Qt module to be tested.

This test inspects all source files that are expected to have a valid Qt
license header and complains about any files that do not match the reference
headers.

=cut

# These modules are not expected to contain any files that need
# Qt license headers.  They are entirely excluded from license checking.
my @excludedModules = qw{
    qtwebkit
};

# If you add to the following lists of regexes, please
# make the patterns as specific as possible to avoid excluding more files
# than intended -- for directories, add the leading and trailing /, and
# for files, add the trailing $ and don't forget to escape the '.'.
# It is also helpful to document why files are excluded.

# The following regex patterns designate directories and files for which
# license headers are not checked at all.  Valid uses of this should be
# very rare - use %optionalFiles where possible.
my %excludedFiles = (
    "qtqa"           => [
                          # contains deliberately incorrect license headers, for testing
                          qr{^scripts/t/license-testdata/bad/},
                        ],
);

# The following regex patterns designate directories and files for which
# license headers are optional.  These files are permitted, but not required,
# to have a license header.  This script will check the correctness of any
# found license headers.
my %optionalFiles = (
    "all"            => [
                          # Third-party files are not expected to have a Qt license
                          qr{/3rdparty/},
                          qr{^3rdparty/},
                          # Don't look at git's metadata
                          qr{^\.git/},
                          # These are qt5 third-party files.
                          qr{^gnuwin32/share/bison/glr\.c$},
                          qr{^gnuwin32/share/bison/yacc\.c$},
                        ],
    "qtbase"         => [
                          # These are two-line wrappers around perl programs.
                          qr{^bin/elf2e32_qtwrapper\.bat$},
                          qr{^bin/patch_capabilities\.bat$},
                          # change logs
                          qr{^dist/},
                          # Wrappers around non-Qt headers
                          qr{^mkspecs/common/symbian/header-wrappers/},
                          # These three are for assemblers that don't allow comments
                          qr{^src/corelib/arch/i386/qatomic_i386\.s$},
                          qr{^src/corelib/arch/vxworks/qatomic_ppc\.s$},
                          qr{^src/corelib/arch/x86_64/qatomic_sun\.s$},
                          # This is a 3rdparty file
                          qr{^src/corelib/io/qurltlds_p\.h$},
                          # This is a 3rdparty file
                          qr{^src/corelib/global/qtypetraits\.h$},
                          # This is a 3rdparty file
                          qr{^src/gui/text/qharfbuzz_copy_p\.h$},
                          # these files are generated by the qdbusxml2cpp utility
                          qr{^src/plugins/platforminputcontexts/ibus/qibusinputcontextproxy\.cpp$},
                          qr{^src/plugins/platforminputcontexts/ibus/qibusinputcontextproxy\.h$},
                          qr{^src/plugins/platforminputcontexts/ibus/qibusproxy\.cpp$},
                          qr{^src/plugins/platforminputcontexts/ibus/qibusproxy\.h$},
                          # This is a list of classes generated by a script
                          qr{^src/tools/uic/qclass_lib_map\.h$},
                          # This is test data, of which some files have a .c extension
                          qr{^tests/auto/corelib/io/qdir/types/},
                          # This directory contains small .bat files that are test data
                          qr{^tests/auto/corelib/io/qprocess/testBatFiles/},
                          # These are test data which has the same .1 extension as man page sources
                          qr{^tests/auto/corelib/xml/qxmlstream/XML-Test-Suite/xmlconf/.*/.*\.1$},
                          # This is an empty test data file
                          qr{^tests/auto/tools/qmake/testdata/bundle-spaces/main\.cpp$},
                          # These test data files are generated by uic
                          qr{^tests/auto/tools/uic/baseline/.*\.h$},
                          # These are generated test data files
                          qr{^tests/benchmarks/corelib/tools/qstring/data\.cpp$},
                          qr{^tests/benchmarks/corelib/tools/qstring/fromutf8\.cpp$},
                          qr{^tests/benchmarks/corelib/tools/qstring/fromlatin1\.cpp$},
                        ],
    "qtconnectivity" => [
                          # This directory contain generated files
                          qr{^src/bluetooth/bluez/},
                        ],
    "qtdeclarative"  => [
                          # This is a single line forwarding header
                          qr{^src/qml/qml/v8/qv8debug_p\.h$},
                          # Nokia Legal has agreed that .qml files in test data
                          # need not have a license header.
                          qr{^tests/auto/qml/},
                          qr{^tests/auto/quick/},
                          qr{^tests/benchmarks/qml/javascript/data/},
                          # The following can be removed when the paths become obsolete:
                          qr{^src/declarative/qml/v8/qv8debug_p\.h$},
                          qr{^tests/auto/declarative/},
                          qr{^tests/auto/qtquick2/},
                          qr{^tests/benchmarks/declarative/javascript/data/},
                        ],
    "qtdoc"          => [
                          # This is a 3rdparty file from KDE
                          qr{^doc/src/classes/phonon-api\.qdoc$},
                        ],
    "qtfeedback"     => [
                          # Nokia Legal has agreed that .qml files in test data
                          # need not have a license header.
                          qr{^tests/auto/qdeclarativefeedback/data/},
                        ],
    "qtquick1"       => [
                          # Nokia Legal has agreed that .qml files in test data
                          # need not have a license header.
                          qr{^tests/auto/declarative/},
                        ],
    "qtscript"       => [
                          # These tests are 3rdparty code.
                          qr{^tests/auto/qscriptjstestsuite/tests/},
                          qr{^tests/auto/qscriptv8testsuite/tests/},
                        ],
    "qttools"        => [
                          # This directory contain 3rdparty code
                          qr{^src/assistant/clucene/},
                          # This directory is a copy of a 3rdparty library
                          qr{^src/assistant/lib/fulltextsearch/},
                          # Nokia Legal has agreed that .qml files in test data
                          # need not have a license header.
                          qr{^tests/auto/linguist/lupdate/testdata/good/parseqml/main\.qml$},
                        ],
);

# Unless specifically excluded, all files matching these patterns are
# required to contain a license header.
my @mandatoryFiles=(
    qr{^configure$},
    qr{^bin/findtr$},
    qr{^bin/qtmodule-configtests$},
    qr{^bin/syncqt$},
    qr{\.h$},
    qr{\.cpp$},
    qr{\.c$},
    qr{\.mm$},
    qr{\.qml$},
    qr{\.qtt$},
    qr{\.fsh$},
    qr{\.vsh$},
    qr{\.qdoc$},
    qr{\.g$},
    qr{\.l$},
    qr{\.s$},
    qr{\.1$},
    qr{\.pl$},
    qr{\.pm$},
    qr{\.bat$},
    qr{\.py$},
    qr{\.sh$},
);

my $QT_MODULE_TO_TEST;
my $moduleName;

#
# These regexes define the expected patterns for the various parts of a
# license header.
#

# Each line of the license header will have a comment delimiter or literal
# string delimiter at the beginning of the line.
# Delimiters we're likely to see in Qt are '*' (C/C++/qdoc), ';' (assembly),
# '!' (SPARC assembly), ':' (batch files), '#' (shell/perl scripts),
# '--' (flex/bison), '.\"' (man page source).
my $leadingDelimiter = qr/^(\s*[\*!;:#\-\.\\\"]+)/;

# These lines appear before the legal text in each license header.
# Where these are embedded in literals in a perl script, the @ in the
# contact email address will be escaped.
my @copyrightBlock = (
    qr/\s\bCopyright \(C\) 2[0-9][0-9][0-9] .*/,
    qr/\s\b(?#optional)All rights reserved.*/,
    qr/\s\bContact: http:\/\/www\.qt-project\.org\/.*/,
    qr//,
    qr/\s\bThis file is (the|part of the)\s*\b(\w*)\b.*/,
    qr//,
);

# These patterns represent the markers at the beginning and end of the
# license text. Where these are embedded in literals in scripts, the dollar
# signs may be escaped by one literal backslash (perl scripts) or two
# literal backslashes (for shell scripts).
my $licenseBeginMarker = qr/\s\\{0,2}\$QT_BEGIN_LICENSE:([A-Z0-9\-]*)\\{0,2}\$(.*)$/;
my $licenseEndMarker   = qr/\s\\{0,2}\$QT_END_LICENSE\\{0,2}\$/;

#
# The following subroutine loads the legal text from a reference header
# into the %licenseTexts map.
#
my %licenseTexts;   # Map from license name to the associated legal text

sub loadLicense {
    my $licenseFile = shift;

    # Read the sample license
    my $fileHandle;
    if (!open($fileHandle, '<', $licenseFile)) {
        fail("Cannot open license file: $licenseFile");
        return 0;
    }

    # Skip lines up to the QT_BEGIN_LICENSE marker.
    my $foundBeginMarker = 0;
    my $licenseType;
    my $beginDelimiter;
    my $endDelimiter;
    while (!$foundBeginMarker and $_ = <$fileHandle>) {
        chomp;
        if (/$leadingDelimiter$licenseBeginMarker/) {
            ($beginDelimiter,$licenseType,$endDelimiter) = ($1, $2, $3);
            $foundBeginMarker = 1;
        }
    }
    if (!$foundBeginMarker) {
        fail("$licenseFile has no QT_BEGIN_LICENSE marker");
        close $fileHandle;
        return 0;
    }

    # Strip the delimiters from lines up to the QT_END_LICENSE marker
    my $foundEndMarker = 0;
    my @licenseText;
    while (!$foundEndMarker and $_ = <$fileHandle>) {
        chomp;
        if (/$licenseEndMarker/) {
            $foundEndMarker = 1;
        } else {
            # Strip delimiters -- wrapping \Q and \E stops perl trying to treat the contents as a regex
            s/^\Q$beginDelimiter\E\s?//;
            s/\s*\Q$endDelimiter\E$//;
            push @licenseText, $_;
        }
    }
    close $fileHandle;
    if (!$foundEndMarker) {
        fail("$licenseFile has no QT_END_LICENSE marker");
        return 0;
    }

    $licenseTexts{$licenseType} = \@licenseText;
    return 1;
}

#
# Check whether the nominated file has a valid license header with legal text
# that matches one of the reference licenses.
#
sub checkLicense
{
    my $filename = shift;

    # Use short filename for reporting purposes (remove useless noise from failure message)
    my $shortfilename = $filename;
    $shortfilename =~ s/^\Q$QT_MODULE_TO_TEST\E\///;

    # Read in the whole file
    my $fileHandle;
    if (!open($fileHandle, '<', $filename)) {
        fail("Cannot open $filename");
        return 0;
    }
    my @lines = <$fileHandle>;
    close $fileHandle;

    # Convert from Mac OS9 format if needed (an OS9 file will look like
    # it's all on a single line due to lack of newlines).
    if ($#lines == 0) {
        @lines = split /\r/, $lines[0];
    }

    # Process the entire file, because it is possible that more than one
    # license header may be present, e.g. one at the top of the file and
    # one embedded in a printf for a utility that generates other files.
    my $matchedLicenses = 0;
    my $inLicenseText = 0;
    my $linesMatched = 0;
    my $currentLine = 0;
    my $beginDelimiter;
    my $endDelimiter;
    my $licenseType;
    my @text;

    while ($currentLine <= $#lines) {
        $_ = $lines[$currentLine];
        $currentLine++;

        chomp;
        s/\r$//;    # Strip DOS carriage return if present

        if ($linesMatched == 0) {
            # Can we match the first line of the copyright block?
            if (/$leadingDelimiter$copyrightBlock[0]/) {
                # Found first line of copyright block
                $beginDelimiter = $1;
                $linesMatched = 1;
            }
            # ...else this is not the beginning of copyright block -- do nothing
        } elsif ($linesMatched == 1 and /$leadingDelimiter$copyrightBlock[0]/ ) {
            # more copyright lines, do nothing
        } elsif ($linesMatched >= 1 and $linesMatched <= $#copyrightBlock) {
            if (/^\Q$beginDelimiter\E$copyrightBlock[$linesMatched]/) {
                # We matched the next line of the copyright block
                ++$linesMatched;
            } elsif ($copyrightBlock[$linesMatched] =~ m{\Q(?#optional)\E}) {
                # We didn't match, but it's OK - this part of the block is optional anyway.
                # We need to move on to the next pattern and rescan this line.
                ++$linesMatched;
                --$currentLine;
            }
            else {
                # If the line doesn't match the delimiter or the expected pattern,
                # don't error out (because other copyright messages are allowed),
                # just go back to looking for the beginning of a license block.
                $linesMatched = 0;
            }
        } elsif ($linesMatched == $#copyrightBlock + 1) {
            # The next line should contain the QT_BEGIN_LICENSE marker
            if (/^\Q$beginDelimiter\E$licenseBeginMarker/) {
                ($licenseType,$endDelimiter) = ($1, $2);
                # Verify that we have reference text for the license type
                if (!defined(@{$licenseTexts{$licenseType}})) {
                    fail("No reference text for license type $licenseType in $shortfilename, line $currentLine");
                    return 0;
                }
                $inLicenseText = 1;
            } elsif (/^\Q$beginDelimiter\E/) {
                fail("QT_BEGIN_LICENSE does not follow Copyright block in $shortfilename, line $currentLine");
                return 0;
            } else {
                fail("$shortfilename has license header with inconsistent comment delimiters, line $currentLine");
                return 0;
            }
            $linesMatched++;
        } else {
            # The next line should contain either license text or the
            # QT_END_LICENSE marker.
            if (/^\Q$beginDelimiter\E$licenseEndMarker\Q$endDelimiter\E/) {
                # We've got all the license text, does it match the reference?
                my @referenceText = @{$licenseTexts{$licenseType}};
                if ($#text != $#referenceText) {
                    fail("License text and reference text have different number of lines in $shortfilename");
                    return 0;
                }
                my $n = 0;
                while ($n <= $#text) {
                    if ($text[$n] ne $referenceText[$n]) {
                        fail("Mismatch in license text in $shortfilename\n".
                             "    Actual: $text[$n]\n".
                             "  Expected: $referenceText[$n]");
                        return 0;
                    }
                    $n++;
                }
                $matchedLicenses++;

                # Reset to begin searching for another license header
                $inLicenseText = 0;
                $linesMatched = 0;
                @text = ();
            } elsif (/^\Q$beginDelimiter\E\s?(.*)\Q$endDelimiter\E/) {
                # We've got a line of license text
                push @text, $1;
            } else {
                # We didn't recognize the line -- it mustn't be wrapped in the
                # same delimiters as the QT_BEGIN_LICENSE line.
                fail("$shortfilename has license header with inconsistent comment delimiters, line $currentLine");
                return 0;
            }
        }
    }

    # Were we in the middle of a license when we reached EOF?
    if ($inLicenseText) {
        fail("$shortfilename has QT_BEGIN_LICENSE, but no QT_END_LICENSE");
        return 0;
    }

    # Did we find any valid licenses?
    if ($matchedLicenses == 0) {
        fail("$shortfilename does not appear to contain a license header");
        return 0;
    }

    # If we get here and matched at least one license then the file is OK.
    pass($shortfilename);
    return 1;
}

#
# Decide whether the nominated file needs to be scanned for a license header.
# We will scan the file if it is obliged to have a license header (i.e. it is
# in one of the mandatory categories and isn't specifically excluded) or if
# it isn't mandatory, but we can see that it contains a QT_BEGIN_LICENSE marker.
#
sub shouldScan
{
    my $fullPath = shift;

    # Is this an existing file?
    return 0 unless (-f $fullPath);

    # Strip the module path from the filename
    my $file = $fullPath;
    $file =~ s/^\Q$QT_MODULE_TO_TEST\E\///;

    # Does the filename match a mandatory pattern?
    my $isMandatory = first { $file =~ qr{$_} } @mandatoryFiles;

    # Is the file excluded or optional?
    my $isExcluded = first { $file =~ qr{$_} } @{$excludedFiles{"all"}}, @{$excludedFiles{$moduleName} || []};
    my $isOptional = first { $file =~ qr{$_} } @{$optionalFiles{"all"}}, @{$optionalFiles{$moduleName} || []};

    return 0 if ($isExcluded);

    # Skip opening the file if we already know we'll have to scan it later
    return 1 if ($isMandatory and !$isOptional);

    # The file is neither excluded nor mandatory - we only check it if it has a license marker
    my $fileHandle;
    if (!open ($fileHandle, '<', $fullPath)) {
        fail("Cannot open $fullPath");
        return 0;
    }
    my @lines = <$fileHandle>;
    close $fileHandle;

    return grep(/QT_BEGIN_LICENSE/, @lines);
}

sub run
{
    #
    # Phase 1: Check prerequisites
    #

    # The QT_MODULE_TO_TEST environment variable must be set and must point to
    # a path that exists
    $QT_MODULE_TO_TEST=$ENV{QT_MODULE_TO_TEST};
    if (!$QT_MODULE_TO_TEST) {
        fail("Environment variable QT_MODULE_TO_TEST has not been set");
        return;
    }
    if (!-d $QT_MODULE_TO_TEST) {
        fail("Environment variable QT_MODULE_TO_TEST is set to \"$QT_MODULE_TO_TEST\", which does not exist");
        return;
    }
    $QT_MODULE_TO_TEST = abs_path($QT_MODULE_TO_TEST);

    # Get module name without the preceding path
    $moduleName = basename ($QT_MODULE_TO_TEST);

    # Skip the test (and return success) if we don't want to scan this module
    if (grep { $_ eq $moduleName } @excludedModules) {
        plan skip_all => "$moduleName is excluded from license checks";
        return;
    }

    #
    # Pase 2: Read the reference license texts
    #

    # Check that qtbase is a peer directory of QT_MODULE_TO_TEST
    my @qtbase_paths = (
        catfile( $QT_MODULE_TO_TEST, '..', 'qtbase' ),  # qt5 submodule case
        catfile( $QT_MODULE_TO_TEST, 'qtbase' ),        # qt5 case
    );

    my $qtbase_path = first { -d $_ } @qtbase_paths;
    if (! $qtbase_path) {
        fail("Cannot find qtbase module, looked at: @qtbase_paths");
        return;
    }

    # Treat all header.* files in the root of qtbase as reference license text
    foreach (glob "$qtbase_path/header.*") {
        loadLicense($_) || return;
    }

    my $numLicenses = keys %licenseTexts;
    if ($numLicenses == 0) {
        fail("No reference licenses were found.");
        return;
    }

    #
    # Phase 3: Decide which files we are going to scan.
    #

    my @filesToScan;
    if (-d "$QT_MODULE_TO_TEST/.git") {
        # We're scanning a git repo, only examine files that git knows
        my $oldpwd = getcwd();
        if (!chdir $QT_MODULE_TO_TEST) {
            fail("Cannot change directory to $QT_MODULE_TO_TEST: $!");
            return;
        }

        my @allFiles = `git ls-files`;

        if ($? != 0) {
            fail("There was a problem running 'git ls-files' on the repository");
            return;
        }

        foreach (@allFiles) {
            chomp;
            shouldScan("$QT_MODULE_TO_TEST/$_") && push @filesToScan, "$QT_MODULE_TO_TEST/$_";
        }
        chdir $oldpwd;
    } else {
        # We're scanning something other than a git repo, examine all files
        find( sub{
            shouldScan($File::Find::name) && push @filesToScan, $File::Find::name;
        }, $QT_MODULE_TO_TEST);
    }

    # sort the files so we get predictable (and testable) output
    @filesToScan = sort @filesToScan;

    #
    # Phase 4: Scan the files
    #
    my $numTests = $#filesToScan + 1;
    if ($numTests <= 0) {
        plan skip_all => "Module $moduleName appears to have no files that must be scanned";
    } else {
        plan tests => $#filesToScan + 1;
        foreach ( @filesToScan ) {
            checkLicense($_);
        }
    }

}

run();
done_testing();