path: root/app.py

# Copyright (C) 2021 The Qt Company Ltd.
# Contact: https://www.qt.io/licensing/
#
# You may use this file under the terms of the CC0 license.
# See the file LICENSE.CC0 from this package for details.

from pathlib import Path

import dash
import dash_core_components as dcc
import dash_html_components as html
import pandas as pd
import requests
import os
import io
import sys
import zipfile
import subprocess
from dash.dependencies import Input, Output

external_stylesheets = [
    {
        "href": "https://codepen.io/chriddyp/pen/bWLwgP.css",
        "rel": "stylesheet",
    },
    {
        "href": "https://fonts.googleapis.com/css2?" "family=Titillium+Web&display=swap",
        "rel": "stylesheet",
    },
]

# Main application
app = dash.Dash(__name__, external_stylesheets=external_stylesheets)
server = app.server
app.title = "The Qt Project"


def partially_hide(x):
    words = []
    x_split = x.split()
    len_split = len(x_split)

    for i, name in enumerate(x_split):
        # Replace with dots
        # filled = '.' * (len(name) - 1)
        # words.append(f'{name[0]}{filled}')

        # Use first letter
        # words.append(name[0])

        # User F. Lastime
        if i != len_split - 1:
            words.append(f"{name[0]}.")
        else:
            words.append(name)
    return " ".join(words)


def get_domain_chart_data(df):
    d = df.groupby("domain")["domain"].count()
    d = d.to_frame()
    d.rename(columns={"domain": "domain_count"}, inplace=True)
    d = d.sort_values(by="domain_count", ascending=False)
    TOP = 5
    d3 = d[:TOP].copy()
    d3.loc["Other"] = d[TOP:]["domain_count"].sum()
    d3 = d3.iloc[::-1]
    total = d3["domain_count"].sum()
    p = (d3["domain_count"] / total) * 100

    return {
        "data": [
            {
                "x": d3["domain_count"],
                "y": [f"{i} " for i in d3.index],
                "text": p,
                "type": "bar",
                "orientation": "h",
                # "marker": {"color": "#f9e56d"},
                "marker": {"color": "#41CD52"},
                "texttemplate": "%{value} (%{text:.2f} %)",
                "textposition": "auto",
            },
        ],
        "layout": {
            "title": "Commits per email domain",
            "height": "350",
            "padding": {
                "r": "150",
            },
        },
    }


def get_ranking_chart_data(df, column="commit_count"):
    TOP = 10
    d4 = (
        df.groupby("name")
        .agg({"name": "count", "files_changed": "sum", "insertions": "sum", "deletions": "sum"})
        .rename(columns={"name": "commit_count"})
    )
    d4 = d4.sort_values(by=column, ascending=False)[:TOP]
    d4 = d4.reset_index()
    d4["name"] = d4["name"].apply(partially_hide)
    d4 = d4.iloc[::-1]

    titles = {
        "files_changed": "Files changed",
        "commit_count": "Commits",
        "insertions": "Insertions",
        "deletions": "Deletions",
    }

    colors = {
        "files_changed": "#53586b",
        "commit_count": "#222840",
        "insertions": "#41CD52",
        "deletions": "#fb6761",
    }

    return {
        "data": [
            {
                "y": d4["name"],
                "x": d4[column],
                "type": "bar",
                "name": "Commits",
                "orientation": "h",
                "marker": {"color": colors[column]},
            },
        ],
        "layout": {
            "title": f"Contributors ({titles[column]})",
            "yaxis": {"dtick": "1"},
            "bargap": "2",
            "margin": {
                "l": "100",
            },
        },
    }


def get_commit_chart_data(df):
    d = df.groupby("date_week")["date_week"].count()
    d = d.to_frame()
    d.rename(columns={"date_week": "date_count"}, inplace=True)
    d = d.reset_index()
    # d["week"] = d["date_week"].copy().apply(lambda x : int(x.split("W")[-1]))
    # d["year"] = d["date_week"].copy().apply(lambda x : int(x.split("W")[0]))

    return {
        "data": [
            {
                "x": d["date_week"],
                "y": d["date_count"],
                "line": {"color": "#222840"},
                "type": "lines",
                "xaxis": "x1",
            },
        ],
        "layout": {
            "title": "Number of commits",
            "height": "300",
            "xaxis": {"tickangle": "45"},
        },
    }


def get_collab_chart_data(df):
    d = df.groupby("date_week")["email"].nunique()
    d = d.to_frame()
    d.rename(columns={"email": "collaborators_count"}, inplace=True)

    return {
        "data": [
            {
                "x": d.index,
                "y": d["collaborators_count"],
                "line": {"color": "#41CD52"},
                "type": "lines",
            },
        ],
        "layout": {
            "title": "Number of contributors",
            "height": "300",
            "xaxis": {"tickangle": "45"},
        },
    }


@app.callback(
    [
        Output("commits-chart", "figure"),
        Output("collaborators-chart", "figure"),
        Output("files-changed-chart", "figure"),
        Output("commit-count-chart", "figure"),
        Output("insertions-chart", "figure"),
        Output("deletions-chart", "figure"),
        Output("domain-chart", "figure"),
    ],
    [
        Input("module-filter", "value"),
        Input("year-filter", "value"),
        Input("tqtc-filter", "value"),
    ],
)
def update_charts(module, year, tqtc):
    """
    This function is in charge of updating the data for all the charts,
    and the connection is done by the chart-id from each.

    As Input, we get the values from the combobox: 'module', and 'year',
    and then we filter the main dataframe, to re-generate the data
    for all the different plots.
    """
    df = data[module][data[module]["datetime"].dt.year >= int(year)]

    if "TQtC" not in tqtc:
        df = df[df.domain != "qt"]

    commit_data = get_commit_chart_data(df)
    collab_data = get_collab_chart_data(df)
    files_changed_data = get_ranking_chart_data(df, column="files_changed")
    commit_count_data = get_ranking_chart_data(df, column="commit_count")
    insertions_data = get_ranking_chart_data(df, column="insertions")
    deletions_data = get_ranking_chart_data(df, column="deletions")
    domain_data = get_domain_chart_data(df)

    return (
        commit_data,
        collab_data,
        files_changed_data,
        commit_count_data,
        insertions_data,
        deletions_data,
        domain_data,
    )


def get_header():
    """
    This is in charge of return the divs that form the header,
    both the left logo/title and the right side menu.
    """
    return html.Div(
        children=[
            html.Div(
                children=[
                    html.Img(src="assets/theqtproject.png", className="header-logo"),
                ],
                className="header-title-left",
            ),
            html.Div(
                children=[
                    html.A(children="Code Review", href="https://codereview.qt-project.org/"),
                    html.A(children="Bug Tracker", href="https://bugreports.qt.io"),
                    html.A(children="Wiki", href="https://wiki.qt.io"),
                    html.A(children="Docs", href="https://doc.qt.io"),
                    html.A(children="Mailing List", href="https://lists.qt-project.org"),
                    html.A(children="Forum", href="https://forum.qt.io"),
                    html.A(children="Qt.io", href="https://qt.io", style={"color": "#41CD52"}),
                ],
                className="header-menu-right",
            ),
        ],
        className="header",
    )


def get_left_column(divs=[]):
    """
    This returns the left divs which contain explanatory text
    about different topics, defined by different markdown files
    on this project.
    """

    def get_div(i):
        return html.Div(
            children=[
                dcc.Markdown(i),
            ],
            className="card pad",
        )

    content = [get_div(i) for i in divs if i]

    return html.Div(
        children=content,
        className="six columns",
    )


def get_services_status():
    gerrit_url = "https://codereview.qt-project.org/projects/qt%2Fqtbase/HEAD"
    coin_url = "https://testresults.qt.io/coin/api/capabilities"

    def get_st(url):
        response = requests.get(url)
        if response.status_code == 200:
            return "Online   🟩"
        elif response.status_code == 404:
            return "Offline  🟥"
        else:
            return "Undefined  🟧"

    return html.Div(
        children=[
            html.H4(children="Services Status"),
            html.Div(
                children=[
                    html.Div(
                        children=[
                            html.B(
                                children="Gerrit: ",
                                style={"display": "inline-block", "margin-right": "5px"},
                            ),
                            html.A(
                                children=f"{get_st(gerrit_url)}",
                                href="https://codereview.qt-project.org",
                                style={"display": "inline-block"},
                            ),
                        ],
                    ),
                ],
                className="five columns",
            ),
            html.Div(
                children=[
                    html.Div(
                        children=[
                            html.B(
                                children="COIN: ",
                                style={"display": "inline-block", "margin-right": "5px"},
                            ),
                            html.A(
                                children=f"{get_st(coin_url)}",
                                href="https://testresults.qt.io",
                                style={"display": "inline-block"},
                            ),
                        ],
                    ),
                ],
                className="five columns",
            ),
        ],
        className="row card",
    )


def get_filter(modules, years):
    """
    Get div containing the combobox to filter the modules
    and years, to trigger the charts update.
    """
    return html.Div(
        children=[
            html.Div(
                children=[
                    html.Div(
                        children=[
                            html.Div(children="Module", className="menu-title"),
                            dcc.Dropdown(
                                id="module-filter",
                                options=[{"label": m, "value": m} for m in modules],
                                value="qtbase",
                                clearable=False,
                                className="dropdown",
                            ),
                        ],
                    ),
                    dcc.Checklist(
                        id="tqtc-filter",
                        options=[
                            {"label": "Include commits from The Qt Company", "value": "TQtC"},
                        ],
                        value=["TQtC"],
                        className="tqtc-filter",
                    ),
                ],
                className="six columns",
            ),
            html.Div(
                children=[
                    html.Div(
                        children=[
                            html.Div(children="Year", className="menu-title"),
                            dcc.Dropdown(
                                id="year-filter",
                                options=[{"label": m, "value": m} for m in years],
                                value="2018",
                                clearable=False,
                                className="dropdown",
                            ),
                        ],
                    ),
                ],
                className="six columns",
            ),
        ],
        className="row card option-select",
    )


def get_filter_email(lists):
    """
    Get div containing the combobox to filter the mailing lists,
    to trigger the charts update.
    """
    return html.Div(
        children=[
            html.Div(
                children=[
                    html.Div(
                        children=[
                            html.Div(children="Mailing List", className="menu-title"),
                            dcc.Dropdown(
                                id="mailing-list-filter",
                                options=[{"label": m, "value": m} for m in lists],
                                value="development",
                                clearable=False,
                                className="dropdown",
                            ),
                        ],
                    ),
                ],
                className="six columns",
            ),
            html.Div(
                children=[
                    html.Div(
                        children=[
                            html.Div(children="Year", className="menu-title"),
                            dcc.Dropdown(
                                id="mailing-list-year-filter",
                                options=[{"label": m, "value": m} for m in years],
                                value="2018",
                                clearable=False,
                                className="dropdown",
                            ),
                        ],
                    ),
                ],
                className="six columns",
            ),
        ],
        className="row card option-select",
    )


def get_markdown_content(filename):
    print(f"Reading '{filename}'...")
    content = ""
    with open(filename, "r") as f:
        content = f.read()
    return content


# Download the data from an external source which runs a cronjob
# to keep an updated ZIP file with the processed data.
# Heroku will restart every day, so the data will be updated on
# a daily basis.
print("Downloading data...")
r = requests.get("https://qtstats.info/data_csv.zip")
if r.status_code == 404:
    print("Error: Problem downloading the Data")
    sys.exit(-1)

with zipfile.ZipFile(io.BytesIO(r.content)) as z:
    z.extractall(".")

print(f"--\nCSV files found: {len(os.listdir('data/'))}\n--")


# Loading all the data
print("Loading all the CSV files...")
modules = []
years = set()
lists = set()
data = {}
email_data = {}
for f in Path("data").glob("*.csv"):
    if "email" in f.name:
        ml = f.stem.replace("emails_", "")
        email_data[ml] = pd.read_csv(f.relative_to(Path(".")), sep=";")
        email_data[ml]["datetime"] = pd.to_datetime(email_data[ml]["date"], format="%Y-%m-%d")
    else:
        data[f.stem] = pd.read_csv(f.relative_to(Path(".")), sep=";")
        data[f.stem]["datetime"] = pd.to_datetime(data[f.stem]["date"], format="%Y-%m-%d")
        data[f.stem].sort_values("datetime", inplace=True)
        data[f.stem]["date_week"] = data[f.stem]["datetime"].apply(
            lambda x: f"{x.year} W{str(x.week).zfill(3)}"
        )
        modules.append(f.stem)
        years.update(set(data[f.stem]["datetime"].dt.year))
        print(f"-- Read '{f.stem}': {data[f.stem].shape}")

# Generate entry with all the content
data["All Qt"] = pd.concat([i for _, i in data.items()], ignore_index=True)
modules.append("All Qt")

modules = sorted(modules)
years = [i for i in sorted(years)]
lists = sorted(i.replace("emails_", "") for i in email_data.keys())
print("Done")

# Reading 'markdown' files
left_boxes = [
    get_markdown_content("learn_more.md"),
    get_markdown_content("contribute.md"),
    get_markdown_content("guidelines.md"),
    get_markdown_content("quips.md"),
]

# Layout
app.layout = html.Div(
    children=[
        get_header(),
        html.Div(
            children=[
                html.Div(
                    children=[
                        get_left_column(divs=left_boxes),
                        html.Div(
                            children=[
                                get_services_status(),
                                get_filter(modules, years),
                                html.Div(
                                    children=dcc.Graph(id="commits-chart"),
                                    className="card",
                                ),
                                html.Div(
                                    children=dcc.Graph(id="collaborators-chart"),
                                    className="card",
                                ),
                                html.Div(
                                    children=dcc.Graph(id="domain-chart"),
                                    className="card",
                                ),
                                html.Div(
                                    children=[
                                        html.Div(
                                            children=dcc.Graph(id="files-changed-chart"),
                                            className="six columns card",
                                        ),
                                        html.Div(
                                            children=dcc.Graph(id="commit-count-chart"),
                                            className="six columns card",
                                        ),
                                    ],
                                    className="row",
                                ),
                                html.Div(
                                    children=[
                                        html.Div(
                                            children=dcc.Graph(id="insertions-chart"),
                                            className="six columns card",
                                        ),
                                        html.Div(
                                            children=dcc.Graph(id="deletions-chart"),
                                            className="six columns card",
                                        ),
                                    ],
                                    className="row",
                                ),
                            ],
                            className="six columns",
                        ),
                    ],
                    className="row",
                ),
            ],
            className="wrapper",
        ),
    ],
)

if __name__ == "__main__":
    app.run_server(debug=False, threaded=True)