path: root/tools/bzl/license-map.py

#!/usr/bin/env python3

# reads bazel query XML files, to join target names with their licenses.

from __future__ import print_function
from collections import namedtuple

import argparse
import json
from collections import defaultdict
from sys import stdout, stderr
import xml.etree.ElementTree as ET

DO_NOT_DISTRIBUTE = "//lib:LICENSE-DO_NOT_DISTRIBUTE"

LICENSE_PREFIX = "//lib:LICENSE-"

parser = argparse.ArgumentParser()
parser.add_argument("--asciidoctor", action="store_true")
parser.add_argument("--json-map", action="append", dest="json_maps")
parser.add_argument("xmls", nargs="+")
args = parser.parse_args()


def read_file(filename):
    "Reads file and returns its content"
    with open(filename) as fd:
        return fd.read()

# List of files in package to which license is applied.
# kind - enum, one of
#   AllFiles - license applies to all files in package
#   OnlySpecificFiles - license applies to all files from "files" list
#   AllFilesExceptSpecific - license applies to all files in package
#      except several files. "files" contains list of exceptions
# files - defines list of files for the following kinds:
#   OnlySpecificFiles, AllFilesExceptSpecific.
#   Each item is a string, but not necessary a real name of a file.
#   It can be any string, understandable by human (like directory name)
LicensedFiles = namedtuple("LicensedFiles", ["kind", "files"])

# PackageInfo - contains information about pacakge/files in packages to
#   which license is applied.
# name - name of the package, as specified in package.json file
# version - optional package version. Exists only if different versions
#   of the same package have different licenses
# licensed_files - instance of LicensedFiles
PackageInfo = namedtuple("PackageInfo", ["name", "version", "licensed_files"])

# LicenseMapItem - describe one type of license and a list of packages
#   under this license
# name - name of the license
# safename - name which is safe to use as an asciidoc bookmark name
# packages - list of PackageInfo
# license_text - license text as string
LicenseMapItem = namedtuple("LicenseMapItem",
                            ["name", "safename", "packages", "license_text"])


def load_xmls(xml_filenames):
    """Load xml files produced by bazel query
     and converts them to a list of LicenseMapItem

    Args:
         xml_filenames: list of string; each string is a filename
    Returns:
        list of LicenseMapItem
    """
    entries = defaultdict(list)
    graph = defaultdict(list)
    handled_rules = set()
    for xml in xml_filenames:
        tree = ET.parse(xml)
        root = tree.getroot()

        for child in root:
            rule_name = child.attrib["name"]
            if rule_name in handled_rules:
                # already handled in other xml files
                continue

            handled_rules.add(rule_name)
            for c in list(child):
                if c.tag != "rule-input":
                    continue

                license_name = c.attrib["name"]
                if LICENSE_PREFIX in license_name:
                    entries[rule_name].append(license_name)
                    graph[license_name].append(rule_name)

    if len(graph[DO_NOT_DISTRIBUTE]):
        print("DO_NOT_DISTRIBUTE license found in:", file=stderr)
        for target in graph[DO_NOT_DISTRIBUTE]:
            print(target, file=stderr)
        exit(1)

    result = []
    for n in sorted(graph.keys()):
        if len(graph[n]) == 0:
            continue

        name = n[len(LICENSE_PREFIX):]
        safename = name.replace(".", "_")
        packages_names = []
        for d in sorted(graph[n]):
            if d.startswith("//lib:") or d.startswith("//lib/"):
                p = d[len("//lib:"):]
            else:
                p = d[d.index(":") + 1:].lower()
            if "__" in p:
                p = p[:p.index("__")]
            packages_names.append(p)

        filename = n[2:].replace(":", "/")
        content = read_file(filename)
        result.append(LicenseMapItem(
            name=name,
            safename=safename,
            license_text=content,
            packages=[PackageInfo(name=name, version=None,
                                  licensed_files=LicensedFiles(kind="All",
                                                               files=[])) for
                      name
                      in packages_names]
        )
        )

    return result

def main():
    xml_data = load_xmls(args.xmls)
    json_map_data = load_jsons(args.json_maps)

    if args.asciidoctor:
        # We don't want any blank line before "= Gerrit Code Review - Licenses"
        print("""= Gerrit Code Review - Licenses

// DO NOT EDIT - GENERATED AUTOMATICALLY.

Gerrit open source software is licensed under the <<Apache2_0,Apache
License 2.0>>.  Executable distributions also include other software
components that are provided under additional licenses.

[[cryptography]]
== Cryptography Notice

This distribution includes cryptographic software.  The country
in which you currently reside may have restrictions on the import,
possession, use, and/or re-export to another country, of encryption
software.  BEFORE using any encryption software, please check
your country's laws, regulations and policies concerning the
import, possession, or use, and re-export of encryption software,
to see if this is permitted.  See the
link:http://www.wassenaar.org/[Wassenaar Arrangement]
for more information.

The U.S. Government Department of Commerce, Bureau of Industry
and Security (BIS), has classified this software as Export
Commodity Control Number (ECCN) 5D002.C.1, which includes
information security software using or performing cryptographic
functions with asymmetric algorithms.  The form and manner of
this distribution makes it eligible for export under the License
Exception ENC Technology Software Unrestricted (TSU) exception
(see the BIS Export Administration Regulations, Section 740.13)
for both object code and source code.

Gerrit includes an SSH daemon (Apache SSHD), to support authenticated
uploads of changes directly from `git push` command line clients.

Gerrit includes an SSH client (JSch), to support authenticated
replication of changes to remote systems, such as for automatic
updates of mirror servers, or realtime backups.

== Licenses
""")

    for data in xml_data + json_map_data:
        name = data.name
        safename = data.safename
        print()
        print("[[%s]]" % safename)
        print(name)
        print()
        for p in data.packages:
            package_notice = ""
            if p.licensed_files.kind == "OnlySpecificFiles":
                package_notice = " - only the following file(s):"
            elif p.licensed_files.kind == "AllFilesExceptSpecific":
                package_notice = " - except the following file(s):"

            print("* " + get_package_display_name(p) + package_notice)
            for file in p.licensed_files.files:
                print("** " + file)
        print()
        print("[[%s_license]]" % safename)
        print("----")
        license_text = data.license_text
        print(data.license_text.rstrip("\r\n"))
        print()
        print("----")
        print()

    if args.asciidoctor:
        print("""
GERRIT
------
Part of link:index.html[Gerrit Code Review]
""")

def load_jsons(json_filenames):
    """Loads information about licenses from jsons files.
    The json files are generated by license-map-generator.ts tool

    Args:
         json_filenames: list of string; each string is a filename
    Returns:
        list of LicenseMapItem
    """
    result = []
    for json_map in json_filenames:
        with open(json_map, 'r') as f:
            licenses_list = json.load(f)
        for license_id, license in licenses_list.items():
            name = license["licenseName"]
            safename = name.replace(".", "_")
            packages = []
            for p in license["packages"]:
                package = PackageInfo(name=p["name"], version=p["version"],
                                      licensed_files=get_licensed_files(
                                          p["licensedFiles"]))
                packages.append(package)
            result.append(LicenseMapItem(
                name=name,
                safename=safename,
                license_text=license["licenseText"],
                packages=sorted(remove_duplicated_packages(packages),
                                key=lambda package: get_package_display_name(
                                    package)),
            ))
    return result

def get_licensed_files(json_licensed_file_dict):
    """Convert json dictionary to LicensedFiles"""
    kind = json_licensed_file_dict["kind"]
    if kind == "AllFiles":
        return LicensedFiles(kind="All", files=[])
    if kind == "OnlySpecificFiles" or kind == "AllFilesExceptSpecific":
        return LicensedFiles(kind=kind, files=sorted(json_licensed_file_dict["files"]))
    raise Exception("Invalid licensed files kind: %s".format(kind))

def get_package_display_name(package):
    """Returns a human-readable name of package with optional version"""
    if package.version:
        return package.name + " - " + package.version
    else:
        return package.name


def can_merge_packages(package_info_list):
    """Returns true if all versions of a package can be replaced with
    a package name

    Args:
        package_info_list: list of PackageInfo. Method assumes,
        that all items in package_info_list have the same package name,
        but different package version.

    Returns:
        True if it is safe to print only a package name (without versions)
        False otherwise
    """
    first = package_info_list[0]
    for package in package_info_list:
        if package.licensed_files != first.licensed_files:
            return False
    return True


def remove_duplicated_packages(package_info_list):
    """ Keep only the name of a package if all versions of the package
    have the same licensed files.

    Args:
        package_info_list: list of PackageInfo. All items in the list
           have the same license.

    Returns:
        list of PackageInfo with removed/replaced items.

    Keep single version of package if all versions have the same
    license files."""
    name_to_package = defaultdict(list)
    for package in package_info_list:
        name_to_package[package.name].append(package)

    result = []
    for package_name, packages in name_to_package.items():
        if can_merge_packages(packages):
            package = packages[0]
            result.append(PackageInfo(name=package.name, version=None,
                                      licensed_files=package.licensed_files))
        else:
            result.extend(packages)
    return result

if __name__ == "__main__":
    main()