diff options
208 files changed, 3392 insertions, 900 deletions
@@ -3,29 +3,49 @@ build --repository_cache=~/.gerritcodereview/bazel-cache/repository build --action_env=PATH build --disk_cache=~/.gerritcodereview/bazel-cache/cas +# Define configuration using remotejdk_11, executes using remotejdk_11 or local_jdk +build:build_shared --java_language_version=11 +build:build_shared --java_runtime_version=remotejdk_11 +build:build_shared --tool_java_language_version=11 +build:build_shared --tool_java_runtime_version=remotejdk_11 + # Builds using remotejdk_11, executes using remotejdk_11 or local_jdk +# Avoid warnings for non default configurations: +# build --config=build_shared build --java_language_version=11 build --java_runtime_version=remotejdk_11 build --tool_java_language_version=11 build --tool_java_runtime_version=remotejdk_11 -# Builds using remotejdk_17, executes using remotejdk_17 or local_jdk -build:java17 --java_language_version=17 -build:java17 --java_runtime_version=remotejdk_17 -build:java17 --tool_java_language_version=17 -build:java17 --tool_java_runtime_version=remotejdk_17 - -# Builds and executes on RBE using remotejdk_11 -build:remote --java_language_version=11 -build:remote --java_runtime_version=remotejdk_11 -build:remote --tool_java_language_version=11 -build:remote --tool_java_runtime_version=remotejdk_11 - -# Builds and executes on RBE using remotejdk_17 -build:remote17 --java_language_version=17 -build:remote17 --java_runtime_version=remotejdk_17 -build:remote17 --tool_java_language_version=17 -build:remote17 --tool_java_runtime_version=remotejdk_17 +# Builds and executes on Google GCP RBE using remotejdk_11 +build:remote --config=config_gcp +build:remote --config=build_shared + +# Define remote configuration alias +build:remote_gcp --config=remote + +# Builds and executes on BuildBuddy RBE using remotejdk_11 +build:remote_bb --config=config_bb +build:remote_bb --config=build_shared + +# Define configuration using remotejdk_17, executes using remotejdk_17 or local_jdk +build:build_java17_shared --java_language_version=17 +build:build_java17_shared --java_runtime_version=remotejdk_17 +build:build_java17_shared --tool_java_language_version=17 +build:build_java17_shared --tool_java_runtime_version=remotejdk_17 + +build:java17 --config=build_java17_shared + +# Builds and executes on Google GCP RBE using remotejdk_17 +build:remote17 --config=config_gcp +build:remote17 --config=build_java17_shared + +# Define remote17 configuration alias +build:remote17_gcp --config=remote17 + +# Builds and executes on BuildBuddy RBE using remotejdk_17 +build:remote17_bb --config=config_bb +build:remote17_bb --config=build_java17_shared # Enable strict_action_env flag to. For more information on this feature see # https://groups.google.com/forum/#!topic/bazel-discuss/_VmRfMyyHBk. diff --git a/.bazelversion b/.bazelversion index 6abaeb2f90..91e4a9f262 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -6.2.0 +6.3.2 diff --git a/Documentation/BUILD b/Documentation/BUILD index af355ca56d..85ddbe7413 100644 --- a/Documentation/BUILD +++ b/Documentation/BUILD @@ -126,3 +126,13 @@ genasciidoc_zip( directory = DOC_DIR, searchbox = False, ) + +genasciidoc_zip( + name = "searchfree_safe", + srcs = SRCS, + attributes = documentation_attributes(), + backend = "html5", + directory = DOC_DIR, + searchbox = False, + webfonts = False, +) diff --git a/Documentation/cmd-index.txt b/Documentation/cmd-index.txt index c959a07ba0..b92a89d2f4 100644 --- a/Documentation/cmd-index.txt +++ b/Documentation/cmd-index.txt @@ -58,9 +58,6 @@ link:cmd-apropos.html[gerrit apropos]:: link:cmd-ban-commit.html[gerrit ban-commit]:: Bans a commit from a project's repository. -link:cmd-copy-approvals.html[gerrit copy-approvals]:: - Copy all inferred approvals labels to the latest patch-set. - link:cmd-check-project-access.html[gerrit check-project-access]:: Check if user(s) can read non-config refs of a project diff --git a/Documentation/cmd-ls-projects.txt b/Documentation/cmd-ls-projects.txt index 1dd6720148..72e7630a7b 100644 --- a/Documentation/cmd-ls-projects.txt +++ b/Documentation/cmd-ls-projects.txt @@ -16,6 +16,10 @@ _ssh_ -p <port> <host> _gerrit ls-projects_ [--limit <N>] [--prefix | -p <prefix>] [--has-acl-for GROUP] + [--match | -m] + [-r REGEX] + [--start | -S] + [--state | -s ] -- == DESCRIPTION @@ -58,6 +62,37 @@ used to unescape the output. Displays project inheritance in a tree-like format. This option does not work together with the show-branch option. +--match:: +-m + Match project substring + +-r:: + Match project regex + +--start:: +-S:: + Number of projects to skip + +--state:: +-s:: + Filter by project state. [ACTIVE | READON_ONLY | HIDDEN] + + +[NOTE] +If the calling user does not meet any of the following criteria: + +* The state of the parent project is either "ACTIVE" or "READ ONLY", +and the calling user has READ permission to at least one ref. +* The state of the parent project is "HIDDEN" and the calling user +has READ permission for 'refs/meta/config'. + +Then the 'parent' field will be labeled as '?-N', where N represents the +nesting level within the project's tree structure. In the provided example, +'All-Projects' corresponds to level 1, 'parent-project' to level 2, and +'child-project' to level 3. + +The output format to display the results should be `json` or `json_compact`. + --type:: Display only projects of the specified type. If not specified, defaults to `all`. Supported types: diff --git a/Documentation/config-gerrit.txt b/Documentation/config-gerrit.txt index 6b8f10a3db..23455b20d4 100644 --- a/Documentation/config-gerrit.txt +++ b/Documentation/config-gerrit.txt @@ -37,9 +37,10 @@ flags. [[accountPatchReviewDb.url]]accountPatchReviewDb.url:: + -The url of accountPatchReviewDb. Supported types are `H2`, `POSTGRESQL`, -`MARIADB`, and `MYSQL`. Drop the driver jar in the lib folder of the site path -if the Jdbc driver of the corresponding Database is not yet in the class path. +The url of accountPatchReviewDb. Supported types are `CLOUDSPANNER`, `H2`, +`POSTGRESQL`, `MARIADB`, and `MYSQL`. Drop the driver jar in the lib folder of +the site path if the Jdbc driver of the corresponding Database is not yet in +the class path. + Default is to create H2 database in the db folder of the site path. + @@ -827,8 +828,7 @@ Default is 0. [[cache.name.maxAge]]cache.<name>.maxAge:: + -Maximum age to keep an entry in the cache. Entries are removed from -the cache and refreshed from source data every maxAge interval. +Maximum age to keep an entry in the cache. Values should use common unit suffixes to express their setting: + * s, sec, second, seconds @@ -922,20 +922,6 @@ Default is 128 MiB per cache, except: + If 0 or negative, disk storage for the cache is disabled. -[[cache.name.expireAfterWrite]]cache.<name>.expireAfterWrite:: -+ -Duration after which a cached value will be evicted and not -read anymore. -+ -Values should use common unit suffixes to express their setting: -+ -* ms, milliseconds -* s, sec, second, seconds -* m, min, minute, minutes -* h, hr, hour, hours -+ -Disabled by default. - [[cache.name.refreshAfterWrite]]cache.<name>.refreshAfterWrite:: + Duration after which we asynchronously refresh the cached value. @@ -1004,6 +990,13 @@ or multiple replica nodes. The cache should be flushed whenever NoteDb change metadata in a repository is modified outside of Gerrit. +cache `"changes_by_project"`:: ++ +Ideally, the memorylimit of this cache is large enough to cover all projects. +This should significantly speed up change ref advertisements and git pushes, +especially for projects with lots of changes, and particularly on replicas +where there is no index. + cache `"git_modified_files"`:: + Each item caches the list of git modified files between two git trees @@ -1205,6 +1198,9 @@ branch of each project. If a project record is updated or deleted, this cache should be flushed. Newly inserted projects do not require a cache flush, as they will be read upon first reference. +NOTE: This cache should be disabled or set with a low refreshAfterWrite +in a cluster setup using multiple primary or multiple replica nodes. + cache `"prolog_rules"`:: + Caches parsed `rules.pl` contents for each project. This cache uses the same @@ -1231,6 +1227,9 @@ is per-user, so 1024 items translates to 1024 unique user accounts. As each individual user account may configure multiple SSH keys, the total number of keys may be larger than the item count. +NOTE: This cache should be disabled or set with a low refreshAfterWrite +in a cluster setup using multiple primary or multiple replica nodes. + cache `"web_sessions"`:: + Tracks the live user sessions coming in over HTTP. Flushing this @@ -1247,6 +1246,9 @@ is strongly recommended. + Session storage is relatively inexpensive. The average entry in this cache is approximately 346 bytes. ++ +The `maxAge` configuration is also used for as maximum lifetime +of the HTTP servlet container session. See also link:cmd-flush-caches.html[gerrit flush-caches]. @@ -1654,6 +1656,21 @@ If 0 the update polling is disabled. + Default is 5 minutes. +[[change.skipCurrentRulesEvaluationOnClosedChanges]] ++ +If false, Gerrit will always take latest project configuration to +compute submit labels. This means that, closed changes (either merged +or abandoned) will be evaluated against the latest configuration which +may produce different results. Especially for merged changes, they may +look like they didn't meet the submit requirements. ++ +When true, evaluation will be skipped and Gerrit will show the +exact status of submit labels when change was submitted. Post-review +votes will only be allowed on labels that were configured when change +was closed. ++ +Default it false. + [[changeCleanup]] === Section changeCleanup @@ -1886,6 +1903,13 @@ The maximum time (in seconds) to wait for a gerrit.sh start command to run a new Gerrit daemon successfully. If not set, defaults to 90 seconds. +[[container.shutdownTimeout]]container.shutdownTimeout:: ++ +The maximum time (in seconds) to wait for a gerrit.sh stop command. +This is added to the highest value between either 'sshd.gracefulStopTimeout' +or 'httpd.gracefulStopTimeout'. If not set, defaults to +30 seconds + [[container.user]]container.user:: + Login name (or UID) of the operating system user the Gerrit JVM @@ -2509,6 +2533,10 @@ Used to identify a specific instance within a group of Gerrit instances with the same `serverId` (i.e.: a Gerrit cluster). Unlike `instanceName` this value is not available in the email templates. +The instance ID can also be configured by setting the Java system property +`gerrit.instanceId` on startup. This will override the configuration in the +gerrit.config. + [[gerrit.instanceName]]gerrit.instanceName:: + Short identifier for this Gerrit instance. diff --git a/Documentation/config-project-config.txt b/Documentation/config-project-config.txt index 7b1baba153..25fe9f3001 100644 --- a/Documentation/config-project-config.txt +++ b/Documentation/config-project-config.txt @@ -129,9 +129,10 @@ project, while keeping the old project around for old references. - `Hidden`: + -The project is hidden and only visible to project owners. Other users -are not able to see the project even if they have read permissions -granted on the project. +The project is hidden; It will not appear in any searches and is only visible +to project owners by going directly to the repository admin page. Other users +are not able to see the project even if they have read permissions granted on +the project. [[receive-section]] diff --git a/Documentation/dev-bazel.txt b/Documentation/dev-bazel.txt index 514a4c9418..a70066990c 100644 --- a/Documentation/dev-bazel.txt +++ b/Documentation/dev-bazel.txt @@ -54,7 +54,7 @@ To check the installed version of Java, open a terminal window and run: To build Gerrit with Java 11 language level, run: ``` - $ bazel build --java_toolchain=//tools:error_prone_warnings_toolchain_java11 :release + $ bazel build :release ``` [[java-17]] @@ -225,6 +225,19 @@ The html files will be bundled into `searchfree.zip` in this location: bazel-bin/Documentation/searchfree.zip ---- +To use local fonts with the searchfree target: + +---- + bazel build Documentation:searchfree_safe +---- + +The html files will be bundled into `searchfree.zip` or `searchfree_safe.zip` in this location: + +---- + bazel-bin/Documentation/searchfree.zip + bazel-bin/Documentation/searchfree_safe.zip +---- + To generate HTML files skipping the zip archiving: ---- @@ -650,6 +663,21 @@ bazel test --config=remote \ ``` +== BuildBuddy Remote Build Support + +To utilize the BuildBuddy Remote Build Execution service, please consult the +documentation available at the following link: https://www.buildbuddy.io[BuildBuddy]. + +To use RBE, execute + +``` +bazelisk test --config=remote_bb \ + --remote_instance_name=projects/${PROJECT}/instances/default_instance \ + --remote_header=x-buildbuddy-api-key=YOUR_API_KEY \ + javatests/... +``` + + GERRIT ------ Part of link:index.html[Gerrit Code Review] diff --git a/Documentation/dev-eclipse.txt b/Documentation/dev-eclipse.txt index 735309239d..f4238d1451 100644 --- a/Documentation/dev-eclipse.txt +++ b/Documentation/dev-eclipse.txt @@ -82,6 +82,11 @@ the same way you would when link:dev-build-plugins.html#_bundle_custom_plugin_in_release_war[bundling in release.war] and run `tools/eclipse/project.py`. +If a plugin requires additional test dependencies (not available in the Gerrit), +then in order to execute tests directly from Eclipse, that plugin must be also +added to `CUSTOM_PLUGINS_TEST_DEPS` list in `tools/bzl/plugins.bzl` and Eclipse +project configuration needs to be updated by running `tools/eclipse/project.py`. + == Java Versions Java 11 is supported as a default, but some adjustments must be done for other JDKs: diff --git a/Documentation/metrics.txt b/Documentation/metrics.txt index 93e0eb4aa8..8b21ca2cd5 100644 --- a/Documentation/metrics.txt +++ b/Documentation/metrics.txt @@ -389,12 +389,27 @@ Each queue provides the following metrics: * `git/upload-pack/request_count`: Total number of git-upload-pack requests. ** `operation`: The name of the operation (CLONE, FETCH). +* `git/upload-pack/bitmap_index_misses_count`: Number of bitmap index misses per request. +** `operation`: + The name of the operation (CLONE, FETCH). +* `git/upload-pack/no_bitmap_index`: Total number of requests executed without a bitmap index. +** `operation`: + The name of the operation (CLONE, FETCH). * `git/upload-pack/phase_counting`: Time spent in the 'Counting...' phase. ** `operation`: The name of the operation (CLONE, FETCH). * `git/upload-pack/phase_compressing`: Time spent in the 'Compressing...' phase. ** `operation`: The name of the operation (CLONE, FETCH). +* `git/upload-pack/phase_negotiating`: Time spent in the negotiation phase. +** `operation`: + The name of the operation (CLONE, FETCH). +* `git/upload-pack/phase_searching_for_reuse`: Time spent in the 'Finding sources...' while searching for reuse phase. +** `operation`: + The name of the operation (CLONE, FETCH). +* `git/upload-pack/phase_searching_for_sizes`: Time spent in the 'Finding sources...' while searching for sizes phase. +** `operation`: + The name of the operation (CLONE, FETCH). * `git/upload-pack/phase_writing`: Time spent transferring bytes to client. ** `operation`: The name of the operation (CLONE, FETCH). diff --git a/Documentation/pg-plugin-checks-api.txt b/Documentation/pg-plugin-checks-api.txt index e4cb5d0ed8..968adcc29e 100644 --- a/Documentation/pg-plugin-checks-api.txt +++ b/Documentation/pg-plugin-checks-api.txt @@ -27,7 +27,7 @@ link:https://www.gerritcodereview.com/design-docs/ci-reboot.html[design doc]. Here are some examples of open source plugins that make use of the Checks API: -* link:https://gerrit.googlesource.com/plugins/checks/+/master/gr-checks/plugin.js[Gerrit Checks Plugin] +* link:https://gerrit.googlesource.com/plugins/checks/+/master/web/plugin.ts[Gerrit Checks Plugin] * link:https://chromium.googlesource.com/infra/gerrit-plugins/buildbucket/+/main/web/plugin.ts[Chromium Buildbucket Plugin] * link:https://chromium.googlesource.com/infra/gerrit-plugins/code-coverage/+/main/web/plugin.ts[Chromium Coverage Plugin] @@ -46,3 +46,23 @@ Here are some examples of open source plugins that make use of the Checks API: `checksApi.announceUpdate()` Tells Gerrit to call `provider.fetch()`. + +[[updateResult]] +== updateResult +`checksApi.updateResult(run: CheckRun, result: CheckResult)` + +Updates an individual result. This can be used for lazy laoding detailled +information. For example, if you are using the +link:pg-plugin-endpoints.html#_check_result_expanded[`check-result-expanded` +endpoint], then you can load more result details when the user expands a result +row. + +The parameter `run` is only used to *find* the correct run for updating the +result. It will only be used for comparing `change`, `patchset`, `attempt` and +`checkName`. Its properties other than `results` will not be updated. + +For us being able to identify the result that you want to update you have to +set the `externalId` property. An undefined `externalId` will result in an +error. + +An example usage can be found in link:https://chromium.googlesource.com/infra/gerrit-plugins/buildbucket/+/main/web/checks-result.ts[Chromium Buildbucket Plugin]. diff --git a/Documentation/pg-plugin-endpoints.txt b/Documentation/pg-plugin-endpoints.txt index dd82f27e45..0429f91723 100644 --- a/Documentation/pg-plugin-endpoints.txt +++ b/Documentation/pg-plugin-endpoints.txt @@ -86,6 +86,25 @@ link:rest-api-changes.html#revision-info[RevisionInfo] labels with scores applied to the change, map of the label names to link:rest-api-changes.html#label-info[LabelInfo] entries +=== check-result-expanded +The `check-result-expanded` extension point is attached to a result +of the link:pg-plugin-checks-api.html[ChecksAPI] when it is expanded. This can +be used to attach a Web Component displaying results instead of the +`CheckResult.message` field which is limited to raw unformatted text. + +In addition to default parameters, the following are available: + +* `result` ++ +The `CheckResult` object for the currently expanded result row. + +* `run` ++ +Same as `result`. The `CheckRun` object is not passed to the endpoint. + +The end point contains the `<gr-formatted-text>` element holding the +`CheckResult.message` (if any was set). + === robot-comment-controls The `robot-comment-controls` extension point is located inside each comment rendered on the diff page, and is only visible when the comment is a robot @@ -246,4 +265,4 @@ In addition to default parameters, the following are available: * `accountId` + -the Id of the account that the status icon should correspond to.
\ No newline at end of file +the Id of the account that the status icon should correspond to. diff --git a/Documentation/pgm-reindex.txt b/Documentation/pgm-reindex.txt index b74829dbde..183c13266c 100644 --- a/Documentation/pgm-reindex.txt +++ b/Documentation/pgm-reindex.txt @@ -39,6 +39,12 @@ Rebuilds the secondary index. --show-cache-stats:: Show cache statistics at the end of program. +--build-bloom-filter:: + Whether to build bloom filters for H2 disk caches. When using fully + populated disk caches on large Gerrit sites, it is recommended that + bloom filters are disabled to improve performance. + + == CONTEXT The secondary index must be enabled. See link:config-gerrit.html#index.type[index.type]. diff --git a/Documentation/project-configuration.txt b/Documentation/project-configuration.txt index e583f457e0..3c88c2e400 100644 --- a/Documentation/project-configuration.txt +++ b/Documentation/project-configuration.txt @@ -5,7 +5,7 @@ There are several ways to create a new project in Gerrit: -- in the Web UI under 'Projects' > 'Create Project' +- click 'CREATE NEW' in the Web UI under 'BROWSE' > 'Repositories' - via the link:rest-api-projects.html#create-project[Create Project] REST endpoint - via the link:cmd-create-project.html[create-project] SSH command @@ -58,7 +58,7 @@ See details at link:config-project-config.html#project-section[project section]. There are several ways to create a new branch in a project: -- in the Web UI under 'Projects' > 'List' > <project> > 'Branches' +- in the Web UI under 'BROWSE' > 'Repositories' > <project> > 'Branches' - via the link:rest-api-projects.html#create-branch[Create Branch] REST endpoint - via the link:cmd-create-branch.html[create-branch] SSH command @@ -84,7 +84,7 @@ are not supported. There are several ways to delete a branch: -- in the Web UI under 'Projects' > 'List' > <project> > 'Branches' +- in the Web UI under 'BROWSE' > 'Repositories' > <project> > 'Branches' - via the link:rest-api-projects.html#delete-branch[Delete Branch] REST endpoint - by using a git client @@ -114,10 +114,11 @@ if the project was created with empty branches. For convenience reasons, when the repository is cloned Git creates a local branch for this default branch and checks it out. -Project owners can set `HEAD` +Project owners can set `HEAD` several ways: -- in the Web UI under 'Projects' > 'List' > <project> > 'Branches' or +- in the Web UI under 'BROWSE' > 'Repositories' > <project> > 'Branches' - via the link:rest-api-projects.html#set-head[Set HEAD] REST endpoint +- via the link:cmd-set-head.html[Set HEAD] SSH command GERRIT diff --git a/Documentation/rest-api-changes.txt b/Documentation/rest-api-changes.txt index 4bb4aadfac..92d4030b2c 100644 --- a/Documentation/rest-api-changes.txt +++ b/Documentation/rest-api-changes.txt @@ -379,6 +379,13 @@ link:#approval-info[ApprovalInfo] of the `all` attribute. as link:#tracking-id-info[TrackingIdInfo]. -- +[[star]] +-- +* `STAR`: include the `starred` field in + link:#change-info[ChangeInfo], which indicates if the change is starred + by the current user or not. +-- + .Request ---- GET /changes/?q=97&o=CURRENT_REVISION&o=CURRENT_COMMIT&o=CURRENT_FILES&o=DOWNLOAD_COMMANDS HTTP/1.0 @@ -6958,6 +6965,7 @@ The user who submitted the change, as an link:rest-api-accounts.html#account-info[ AccountInfo] entity. |`starred` |not set if `false`| Whether the calling user has starred this change with the default label. +Only set if link:#star[requested]. |`stars` |optional| A list of star labels that are applied by the calling user to this change. The labels are lexicographically sorted. diff --git a/Documentation/rest-api-projects.txt b/Documentation/rest-api-projects.txt index 9e71df7bbc..675c05424d 100644 --- a/Documentation/rest-api-projects.txt +++ b/Documentation/rest-api-projects.txt @@ -268,7 +268,20 @@ List all projects that match substring `test/`: Tree(t):: Get projects inheritance in a tree-like format. This option does not work together with the branch option. -+ + +[NOTE] +If the calling user does not meet any of the following criteria: + +* The state of the parent project is either "ACTIVE" or "READ ONLY", +and the calling user has READ permission to at least one ref. +* The state of the parent project is "HIDDEN" and the calling user +has READ permission for 'refs/meta/config'. + +Then the 'parent' field will be labeled as '?-N', where N represents the +nesting level within the project's tree structure. In the provided example, +'All-Projects' corresponds to level 1, 'parent-project' to level 2, and +'child-project' to level 3. + Get all the projects with tree option: + .Request diff --git a/Documentation/user-inline-edit.txt b/Documentation/user-inline-edit.txt index e3e0f681e2..7ed87b6c3e 100644 --- a/Documentation/user-inline-edit.txt +++ b/Documentation/user-inline-edit.txt @@ -15,6 +15,9 @@ To learn more, see the link:intro-user.html[Gerrit User's Guide]. [[create-change]] == Creating a Change +[[create_in_web_interface]] +=== In the web interface + To create a change in the Gerrit web interface: . From the link:http://gerrit-review.googlesource.com[Gerrit Code Review,role=external,window=_blank] @@ -64,6 +67,22 @@ change. . Add the files you want to be reviewed. +[[create_from_url]] +=== From URL + +Gerrit supports creating a new change and opening a specific file for edit +in that change from an "Edit URL": +``` +^\/admin\/repos\/edit\/repo\/(.+)\/branch\/(.+)\/file\/(.+)$ +``` +This enables other tools to provide a direct link to edit their configuration +files in Gerrit. + +Ex: +``` +https://gerrit.mycompany.com/admin/repos/edit/repo/my/repo/branch/refs/heads/master/file/Jenkinsfile # Jenkins build file +https://gerrit.mycompany.com/admin/repos/edit/repo/my/repo/branch/refs/heads/master/file/catalog-info.yaml # Backstage catalog-info +``` [[add-files]] == Adding a File to a Change diff --git a/Documentation/user-upload.txt b/Documentation/user-upload.txt index 8c51207584..c6fce2a531 100644 --- a/Documentation/user-upload.txt +++ b/Documentation/user-upload.txt @@ -1,13 +1,15 @@ :linkattrs: = Gerrit Code Review - Uploading Changes -Gerrit supports three methods of uploading changes: +Gerrit supports five methods of uploading changes: * Use `repo upload`, to create changes for review * Use `git push`, to create changes for review +* link:user-inline-edit.html#create_in_web_interface[Create a change for review from the web interface] +* link:user-inline-edit.html#create_from_url[Create a change for review by using an "Edit URL"] * Use `git push`, and bypass code review -All three methods rely on authentication, which must first be configured +All five methods rely on authentication, which must first be configured by the uploading user. Gerrit supports two protocols for uploading changes; SSH and HTTP/HTTPS. These diff --git a/java/com/google/gerrit/acceptance/AbstractDaemonTest.java b/java/com/google/gerrit/acceptance/AbstractDaemonTest.java index e93c152104..f4e7ccea52 100644 --- a/java/com/google/gerrit/acceptance/AbstractDaemonTest.java +++ b/java/com/google/gerrit/acceptance/AbstractDaemonTest.java @@ -51,6 +51,8 @@ import com.google.common.primitives.Chars; import com.google.common.testing.FakeTicker; import com.google.gerrit.acceptance.AcceptanceTestRequestScope.Context; import com.google.gerrit.acceptance.PushOneCommit.Result; +import com.google.gerrit.acceptance.config.ConfigAnnotationParser; +import com.google.gerrit.acceptance.config.GerritSystemProperty; import com.google.gerrit.acceptance.testsuite.account.TestSshKeys; import com.google.gerrit.acceptance.testsuite.project.ProjectOperations; import com.google.gerrit.acceptance.testsuite.request.RequestScopeOperations; @@ -436,6 +438,14 @@ public abstract class AbstractDaemonTest { GerritServer.Description.forTestMethod(description, configName); testMethodDescription = methodDesc; + if (methodDesc.systemProperties() != null) { + ConfigAnnotationParser.parse(methodDesc.systemProperties()); + } + + if (methodDesc.systemProperty() != null) { + ConfigAnnotationParser.parse(methodDesc.systemProperty()); + } + testRequiresSsh = classDesc.useSshAnnotation() || methodDesc.useSshAnnotation(); if (!testRequiresSsh) { baseConfig.setString("sshd", null, "listenAddress", "off"); @@ -690,6 +700,19 @@ public abstract class AbstractDaemonTest { server.close(); server = null; } + + GerritServer.Description methodDesc = + GerritServer.Description.forTestMethod(description, configName); + if (methodDesc.systemProperties() != null) { + for (GerritSystemProperty sysProp : methodDesc.systemProperties().value()) { + System.clearProperty(sysProp.name()); + } + } + + if (methodDesc.systemProperty() != null) { + System.clearProperty(methodDesc.systemProperty().name()); + } + SystemReader.setInstance(oldSystemReader); oldSystemReader = null; // Set useDefaultTicker in afterTest, so the next beforeTest will use the default ticker diff --git a/java/com/google/gerrit/acceptance/GerritServer.java b/java/com/google/gerrit/acceptance/GerritServer.java index e812cb75b5..d4336893cd 100644 --- a/java/com/google/gerrit/acceptance/GerritServer.java +++ b/java/com/google/gerrit/acceptance/GerritServer.java @@ -31,6 +31,8 @@ import com.google.gerrit.acceptance.ReindexProjectsAtStartup.ReindexProjectsAtSt import com.google.gerrit.acceptance.config.ConfigAnnotationParser; import com.google.gerrit.acceptance.config.GerritConfig; import com.google.gerrit.acceptance.config.GerritConfigs; +import com.google.gerrit.acceptance.config.GerritSystemProperties; +import com.google.gerrit.acceptance.config.GerritSystemProperty; import com.google.gerrit.acceptance.config.GlobalPluginConfig; import com.google.gerrit.acceptance.config.GlobalPluginConfigs; import com.google.gerrit.acceptance.testsuite.account.AccountOperations; @@ -61,6 +63,7 @@ import com.google.gerrit.server.config.SitePath; import com.google.gerrit.server.experiments.ConfigExperimentFeatures.ConfigExperimentFeaturesModule; import com.google.gerrit.server.git.receive.AsyncReceiveCommits.AsyncReceiveCommitsModule; import com.google.gerrit.server.git.validators.CommitValidationListener; +import com.google.gerrit.server.index.AbstractIndexModule; import com.google.gerrit.server.index.options.AutoFlush; import com.google.gerrit.server.schema.JdbcAccountPatchReviewStore; import com.google.gerrit.server.ssh.NoSshModule; @@ -135,6 +138,8 @@ public class GerritServer implements AutoCloseable { false, // @UseSystemTime is only valid on methods. get(UseClockStep.class, testDesc.getTestClass()), get(UseTimezone.class, testDesc.getTestClass()), + null, // @GerritSystemProperty is only valid on methods. + null, // @GerritSystemProperties is only valid on methods. null, // @GerritConfig is only valid on methods. null, // @GerritConfigs is only valid on methods. null, // @GlobalPluginConfig is only valid on methods. @@ -177,6 +182,8 @@ public class GerritServer implements AutoCloseable { testDesc.getAnnotation(UseTimezone.class) != null ? testDesc.getAnnotation(UseTimezone.class) : get(UseTimezone.class, testDesc.getTestClass()), + testDesc.getAnnotation(GerritSystemProperty.class), + testDesc.getAnnotation(GerritSystemProperties.class), testDesc.getAnnotation(GerritConfig.class), testDesc.getAnnotation(GerritConfigs.class), testDesc.getAnnotation(GlobalPluginConfig.class), @@ -232,6 +239,12 @@ public class GerritServer implements AutoCloseable { abstract UseTimezone useTimezone(); @Nullable + abstract GerritSystemProperty systemProperty(); + + @Nullable + abstract GerritSystemProperties systemProperties(); + + @Nullable abstract GerritConfig config(); @Nullable @@ -247,6 +260,10 @@ public class GerritServer implements AutoCloseable { if (useClockStep() != null && useSystemTime()) { throw new IllegalStateException("Use either @UseClockStep or @UseSystemTime, not both"); } + if (systemProperties() != null && systemProperty() != null) { + throw new IllegalStateException( + "Use either @GerritSystemProperties or @GerritSystemProperty, not both"); + } if (configs() != null && config() != null) { throw new IllegalStateException("Use either @GerritConfigs or @GerritConfig, not both"); } @@ -316,7 +333,7 @@ public class GerritServer implements AutoCloseable { String configuredIndexBackend = cfg.getString("index", null, "type"); if (configuredIndexBackend == null) { // Propagate index type to pgms that run off of the gerrit.config file on local disk. - IndexType indexType = IndexType.fromEnvironment().orElse(new IndexType("fake")); + IndexType indexType = IndexType.fromEnvironment().orElseGet(() -> new IndexType("fake")); gerritConfig.setString("index", null, "type", indexType.isLucene() ? "lucene" : "fake"); } gerritConfig.save(); @@ -425,7 +442,6 @@ public class GerritServer implements AutoCloseable { if (testSshModule != null) { daemon.addAdditionalSshModuleForTesting(testSshModule); } - daemon.setEnableSshd(desc.useSsh()); daemon.addAdditionalSysModuleForTesting( new AbstractModule() { @Override @@ -461,7 +477,11 @@ public class GerritServer implements AutoCloseable { if (desc.memory()) { checkArgument(additionalArgs.length == 0, "cannot pass args to in-memory server"); - return startInMemory(desc, site, baseConfig, daemon, inMemoryRepoManager); + AbstractIndexModule testIndexModule = + (testSysModule instanceof AbstractIndexModule) + ? (AbstractIndexModule) testSysModule + : null; + return startInMemory(desc, site, baseConfig, daemon, inMemoryRepoManager, testIndexModule); } return startOnDisk(desc, site, daemon, serverStarted, additionalArgs); } @@ -471,7 +491,8 @@ public class GerritServer implements AutoCloseable { Path site, Config baseConfig, Daemon daemon, - @Nullable InMemoryRepositoryManager inMemoryRepoManager) + @Nullable InMemoryRepositoryManager inMemoryRepoManager, + @Nullable AbstractIndexModule testIndexModule) throws Exception { Config cfg = desc.buildConfig(baseConfig); mergeTestConfig(cfg); @@ -487,24 +508,11 @@ public class GerritServer implements AutoCloseable { "accountPatchReviewDb", null, "url", JdbcAccountPatchReviewStore.TEST_IN_MEMORY_URL); String configuredIndexBackend = cfg.getString("index", null, "type"); - IndexType indexType; - if (configuredIndexBackend != null) { - // Explicitly configured index backend from gerrit.config trumps any other ways to configure - // index backends so that Reindex tests can be explicit about the backend they want to test - // against. - indexType = new IndexType(configuredIndexBackend); - } else { - // Allow configuring the index backend based on sys/env variables so that integration tests - // can be run against different index backends. - indexType = IndexType.fromEnvironment().orElse(new IndexType("fake")); - } - if (indexType.isLucene()) { - daemon.setIndexModule( - LuceneIndexModule.singleVersionAllLatest( - 0, ReplicaUtil.isReplica(baseConfig), AutoFlush.ENABLED)); - } else { - daemon.setIndexModule(FakeIndexModule.latestVersion(false)); - } + IndexType indexType = + (configuredIndexBackend != null) + ? new IndexType(configuredIndexBackend) + : IndexType.fromEnvironment().orElseGet(() -> new IndexType("fake")); + daemon.setIndexModule(createIndexModule(indexType, baseConfig, testIndexModule)); daemon.setEnableHttpd(desc.httpd()); daemon.setInMemory(true); @@ -524,6 +532,17 @@ public class GerritServer implements AutoCloseable { return new GerritServer(desc, null, createTestInjector(daemon), daemon, null); } + private static AbstractIndexModule createIndexModule( + IndexType indexType, Config baseConfig, @Nullable AbstractIndexModule testIndexModule) { + if (testIndexModule != null) { + return testIndexModule; + } + return indexType.isLucene() + ? LuceneIndexModule.singleVersionAllLatest( + 0, ReplicaUtil.isReplica(baseConfig), AutoFlush.ENABLED) + : FakeIndexModule.latestVersion(false); + } + private static GerritServer startOnDisk( Description desc, Path site, diff --git a/java/com/google/gerrit/acceptance/config/ConfigAnnotationParser.java b/java/com/google/gerrit/acceptance/config/ConfigAnnotationParser.java index 27ce85755d..fc6be0376a 100644 --- a/java/com/google/gerrit/acceptance/config/ConfigAnnotationParser.java +++ b/java/com/google/gerrit/acceptance/config/ConfigAnnotationParser.java @@ -47,6 +47,16 @@ public class ConfigAnnotationParser { return cfg; } + public static void parse(GerritSystemProperties annotation) { + for (GerritSystemProperty prop : annotation.value()) { + parse(prop); + } + } + + public static void parse(GerritSystemProperty annotation) { + System.setProperty(annotation.name(), annotation.value()); + } + @Nullable public static Map<String, Config> parse(GlobalPluginConfigs annotation) { if (annotation == null || annotation.value().length < 1) { diff --git a/java/com/google/gerrit/acceptance/config/GerritSystemProperties.java b/java/com/google/gerrit/acceptance/config/GerritSystemProperties.java new file mode 100644 index 0000000000..cc6389c47d --- /dev/null +++ b/java/com/google/gerrit/acceptance/config/GerritSystemProperties.java @@ -0,0 +1,27 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.acceptance.config; + +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +@Target({METHOD}) +@Retention(RUNTIME) +public @interface GerritSystemProperties { + GerritSystemProperty[] value(); +} diff --git a/java/com/google/gerrit/acceptance/config/GerritSystemProperty.java b/java/com/google/gerrit/acceptance/config/GerritSystemProperty.java new file mode 100644 index 0000000000..a2bf735dd1 --- /dev/null +++ b/java/com/google/gerrit/acceptance/config/GerritSystemProperty.java @@ -0,0 +1,33 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.acceptance.config; + +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +import java.lang.annotation.Repeatable; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +@Target({METHOD}) +@Retention(RUNTIME) +@Repeatable(GerritSystemProperties.class) +public @interface GerritSystemProperty { + /** System property name. */ + String name(); + + /** Value of the system property. */ + String value() default ""; +} diff --git a/java/com/google/gerrit/acceptance/testsuite/group/GroupOperationsImpl.java b/java/com/google/gerrit/acceptance/testsuite/group/GroupOperationsImpl.java index dcf1158e14..a37c2babbe 100644 --- a/java/com/google/gerrit/acceptance/testsuite/group/GroupOperationsImpl.java +++ b/java/com/google/gerrit/acceptance/testsuite/group/GroupOperationsImpl.java @@ -78,7 +78,7 @@ public class GroupOperationsImpl implements GroupOperations { private InternalGroupCreation toInternalGroupCreation(TestGroupCreation groupCreation) { AccountGroup.Id groupId = AccountGroup.id(seq.nextGroupId()); - String groupName = groupCreation.name().orElse("group-with-id-" + groupId.get()); + String groupName = groupCreation.name().orElseGet(() -> "group-with-id-" + groupId.get()); AccountGroup.UUID groupUuid = GroupUuid.make(groupName, serverIdent); AccountGroup.NameKey nameKey = AccountGroup.nameKey(groupName); return InternalGroupCreation.builder() diff --git a/java/com/google/gerrit/acceptance/testsuite/project/ProjectOperationsImpl.java b/java/com/google/gerrit/acceptance/testsuite/project/ProjectOperationsImpl.java index bd3d65645f..a3112f8f94 100644 --- a/java/com/google/gerrit/acceptance/testsuite/project/ProjectOperationsImpl.java +++ b/java/com/google/gerrit/acceptance/testsuite/project/ProjectOperationsImpl.java @@ -88,7 +88,7 @@ public class ProjectOperationsImpl implements ProjectOperations { } private Project.NameKey createNewProject(TestProjectCreation projectCreation) throws Exception { - String name = projectCreation.name().orElse(RandomStringUtils.randomAlphabetic(8)); + String name = projectCreation.name().orElseGet(() -> RandomStringUtils.randomAlphabetic(8)); CreateProjectArgs args = new CreateProjectArgs(); args.setProjectName(name); diff --git a/java/com/google/gerrit/common/UsedAt.java b/java/com/google/gerrit/common/UsedAt.java index 6a482fb3cc..46b43c6821 100644 --- a/java/com/google/gerrit/common/UsedAt.java +++ b/java/com/google/gerrit/common/UsedAt.java @@ -46,7 +46,8 @@ public @interface UsedAt { PLUGIN_SERVICEUSER, PLUGIN_PULL_REPLICATION, PLUGIN_WEBSESSION_FLATFILE, - MODULE_GIT_REFS_FILTER + MODULE_GIT_REFS_FILTER, + MODULE_VIRTUALHOST } /** Reference to the project that uses the method annotated with this annotation. */ diff --git a/java/com/google/gerrit/entities/Change.java b/java/com/google/gerrit/entities/Change.java index 56fb748e1d..fad3aa84ab 100644 --- a/java/com/google/gerrit/entities/Change.java +++ b/java/com/google/gerrit/entities/Change.java @@ -433,6 +433,9 @@ public final class Change { /** Locally assigned unique identifier of the change */ private Id changeId; + /** ServerId of the Gerrit instance that has created the change */ + private String serverId; + /** Globally assigned unique identifier of the change */ private Key changeKey; @@ -530,6 +533,22 @@ public final class Change { return changeId; } + /** + * Set the serverId of the Gerrit instance that created the change. It can be set to null for + * testing purposes in the protobuf converter tests. + */ + public void setServerId(@Nullable String serverId) { + this.serverId = serverId; + } + + /** + * ServerId of the Gerrit instance that created the change. It could be null when the change is + * not fetched from NoteDb but obtained through protobuf deserialisation. + */ + public @Nullable String getServerId() { + return serverId; + } + /** 32 bit integer identity for a change. */ public int getChangeId() { return changeId.get(); diff --git a/java/com/google/gerrit/extensions/client/ListChangesOption.java b/java/com/google/gerrit/extensions/client/ListChangesOption.java index f1f7831c8a..48a3502df0 100644 --- a/java/com/google/gerrit/extensions/client/ListChangesOption.java +++ b/java/com/google/gerrit/extensions/client/ListChangesOption.java @@ -88,7 +88,10 @@ public enum ListChangesOption implements ListOption { SKIP_DIFFSTAT(23), /** Include the evaluated submit requirements for the caller. */ - SUBMIT_REQUIREMENTS(24); + SUBMIT_REQUIREMENTS(24), + + /** Include the 'starred' field, that is if the change is starred by the current user . */ + STAR(25); private final int value; diff --git a/java/com/google/gerrit/extensions/common/ChangeInfo.java b/java/com/google/gerrit/extensions/common/ChangeInfo.java index dc9bc32ea8..9d812ad40f 100644 --- a/java/com/google/gerrit/extensions/common/ChangeInfo.java +++ b/java/com/google/gerrit/extensions/common/ChangeInfo.java @@ -98,6 +98,7 @@ public class ChangeInfo { public Boolean containsGitConflicts; public Integer _number; + public Integer _virtualIdNumber; public AccountInfo owner; diff --git a/java/com/google/gerrit/httpd/GuiceRequestScopePropagator.java b/java/com/google/gerrit/httpd/GuiceRequestScopePropagator.java index 7c8094a7ea..998b313681 100644 --- a/java/com/google/gerrit/httpd/GuiceRequestScopePropagator.java +++ b/java/com/google/gerrit/httpd/GuiceRequestScopePropagator.java @@ -14,7 +14,6 @@ package com.google.gerrit.httpd; -import com.google.gerrit.common.Nullable; import com.google.gerrit.server.RemotePeer; import com.google.gerrit.server.config.CanonicalWebUrl; import com.google.gerrit.server.util.RequestScopePropagator; @@ -36,20 +35,23 @@ import javax.servlet.http.HttpServletRequest; /** Propagator for Guice's built-in servlet scope. */ public class GuiceRequestScopePropagator extends RequestScopePropagator { - private final String url; + private final HttpCanonicalWebUrlProvider urlProvider; private final SocketAddress peer; private final Provider<HttpServletRequest> request; @Inject GuiceRequestScopePropagator( - @CanonicalWebUrl @Nullable Provider<String> urlProvider, + HttpCanonicalWebUrlProvider urlProvider, @RemotePeer Provider<SocketAddress> remotePeerProvider, ThreadLocalRequestContext local, Provider<HttpServletRequest> request) { super(ServletScopes.REQUEST, local); - this.url = urlProvider != null ? urlProvider.get() : null; + this.urlProvider = urlProvider; this.peer = remotePeerProvider.get(); this.request = request; + // Ensure HttpServletRequest is propagated to HttpCanonicalWebUrlProvider + // so that it won't fallback to the default host name. + urlProvider.setHttpServletRequest(request); } /** @see RequestScopePropagator#wrap(Callable) */ @@ -60,12 +62,15 @@ public class GuiceRequestScopePropagator extends RequestScopePropagator { @Override protected <T> Callable<T> wrapImpl(Callable<T> callable) { Map<Key<?>, Object> seedMap = new HashMap<>(); + String canonicalWebUrl = urlProvider.get(); // Request scopes appear to use specific keys in their map, instead of only // providers. Add bindings for both the key to the instance directly and the // provider to the instance to be safe. - seedMap.put(Key.get(typeOfProvider(String.class), CanonicalWebUrl.class), Providers.of(url)); - seedMap.put(Key.get(String.class, CanonicalWebUrl.class), url); + seedMap.put( + Key.get(typeOfProvider(String.class), CanonicalWebUrl.class), + Providers.of(canonicalWebUrl)); + seedMap.put(Key.get(String.class, CanonicalWebUrl.class), canonicalWebUrl); seedMap.put(Key.get(typeOfProvider(SocketAddress.class), RemotePeer.class), Providers.of(peer)); seedMap.put(Key.get(SocketAddress.class, RemotePeer.class), peer); diff --git a/java/com/google/gerrit/httpd/HttpCanonicalWebUrlProvider.java b/java/com/google/gerrit/httpd/HttpCanonicalWebUrlProvider.java index 6943faa87f..85dc20068c 100644 --- a/java/com/google/gerrit/httpd/HttpCanonicalWebUrlProvider.java +++ b/java/com/google/gerrit/httpd/HttpCanonicalWebUrlProvider.java @@ -14,6 +14,8 @@ package com.google.gerrit.httpd; +import com.google.gerrit.common.UsedAt; +import com.google.gerrit.common.UsedAt.Project; import com.google.gerrit.server.config.CanonicalWebUrlProvider; import com.google.gerrit.server.config.GerritServerConfig; import com.google.inject.Inject; @@ -30,7 +32,8 @@ public class HttpCanonicalWebUrlProvider extends CanonicalWebUrlProvider { private Provider<HttpServletRequest> requestProvider; @Inject - HttpCanonicalWebUrlProvider(@GerritServerConfig Config config) { + @UsedAt(Project.MODULE_VIRTUALHOST) + protected HttpCanonicalWebUrlProvider(@GerritServerConfig Config config) { super(config); } diff --git a/java/com/google/gerrit/httpd/RemoteUserUtil.java b/java/com/google/gerrit/httpd/RemoteUserUtil.java index 6f3e9c45d8..9ec10e2124 100644 --- a/java/com/google/gerrit/httpd/RemoteUserUtil.java +++ b/java/com/google/gerrit/httpd/RemoteUserUtil.java @@ -36,6 +36,7 @@ public class RemoteUserUtil { * @param loginHeader name of header which is used for extracting username. * @return the extracted username or null. */ + @Nullable public static String getRemoteUser(HttpServletRequest req, String loginHeader) { if (AUTHORIZATION.equals(loginHeader)) { String user = emptyToNull(req.getRemoteUser()); diff --git a/java/com/google/gerrit/httpd/auth/container/HttpAuthFilter.java b/java/com/google/gerrit/httpd/auth/container/HttpAuthFilter.java index be833ea468..f0a8b89994 100644 --- a/java/com/google/gerrit/httpd/auth/container/HttpAuthFilter.java +++ b/java/com/google/gerrit/httpd/auth/container/HttpAuthFilter.java @@ -147,8 +147,10 @@ class HttpAuthFilter implements Filter { @Nullable String getRemoteDisplayname(HttpServletRequest req) { if (displaynameHeader != null) { - String raw = req.getHeader(displaynameHeader); - return emptyToNull(new String(raw.getBytes(ISO_8859_1), UTF_8)); + String raw = emptyToNull(req.getHeader(displaynameHeader)); + if (raw != null) { + return new String(raw.getBytes(ISO_8859_1), UTF_8); + } } return null; } diff --git a/java/com/google/gerrit/httpd/init/WebAppInitializer.java b/java/com/google/gerrit/httpd/init/WebAppInitializer.java index 7293f35f49..e3cc0a5ccb 100644 --- a/java/com/google/gerrit/httpd/init/WebAppInitializer.java +++ b/java/com/google/gerrit/httpd/init/WebAppInitializer.java @@ -76,9 +76,9 @@ import com.google.gerrit.server.config.SitePath; import com.google.gerrit.server.config.SysExecutorModule; import com.google.gerrit.server.events.EventBroker.EventBrokerModule; import com.google.gerrit.server.events.StreamEventsApiListener.StreamEventsApiListenerModule; +import com.google.gerrit.server.git.ChangesByProjectCache; import com.google.gerrit.server.git.GarbageCollectionModule; import com.google.gerrit.server.git.GitRepositoryManagerModule; -import com.google.gerrit.server.git.SearchingChangeCacheImpl.SearchingChangeCacheImplModule; import com.google.gerrit.server.git.SystemReaderInstaller; import com.google.gerrit.server.git.WorkQueue.WorkQueueModule; import com.google.gerrit.server.index.IndexModule; @@ -313,7 +313,7 @@ public class WebAppInitializer extends GuiceServletContextListener implements Fi modules.add(new ProjectQueryBuilderModule()); modules.add(new DefaultRefLogIdentityProvider.Module()); modules.add(new PluginApiModule()); - modules.add(new SearchingChangeCacheImplModule()); + modules.add(new ChangesByProjectCache.Module(ChangesByProjectCache.UseIndex.TRUE, config)); modules.add(new InternalAccountDirectoryModule()); modules.add(new DefaultPermissionBackendModule()); modules.add(new DefaultMemoryCacheModule()); diff --git a/java/com/google/gerrit/httpd/raw/IndexPreloadingUtil.java b/java/com/google/gerrit/httpd/raw/IndexPreloadingUtil.java index 402e48a99e..36fa61b239 100644 --- a/java/com/google/gerrit/httpd/raw/IndexPreloadingUtil.java +++ b/java/com/google/gerrit/httpd/raw/IndexPreloadingUtil.java @@ -87,7 +87,8 @@ public class IndexPreloadingUtil { ImmutableSet.of( ListChangesOption.LABELS, ListChangesOption.DETAILED_ACCOUNTS, - ListChangesOption.SUBMIT_REQUIREMENTS); + ListChangesOption.SUBMIT_REQUIREMENTS, + ListChangesOption.STAR); public static final ImmutableSet<ListChangesOption> CHANGE_DETAIL_OPTIONS = ImmutableSet.of( diff --git a/java/com/google/gerrit/httpd/raw/StaticModule.java b/java/com/google/gerrit/httpd/raw/StaticModule.java index 8319d9db21..306973f593 100644 --- a/java/com/google/gerrit/httpd/raw/StaticModule.java +++ b/java/com/google/gerrit/httpd/raw/StaticModule.java @@ -243,7 +243,7 @@ public class StaticModule extends ServletModule { @GerritServerConfig Config cfg, GerritApi gerritApi, ExperimentFeatures experimentFeatures) { - String cdnPath = options.devCdn().orElse(cfg.getString("gerrit", null, "cdnPath")); + String cdnPath = options.devCdn().orElseGet(() -> cfg.getString("gerrit", null, "cdnPath")); String faviconPath = cfg.getString("gerrit", null, "faviconPath"); return new IndexServlet(canonicalUrl, cdnPath, faviconPath, gerritApi, experimentFeatures); } diff --git a/java/com/google/gerrit/index/QueryOptions.java b/java/com/google/gerrit/index/QueryOptions.java index 29ab6d0119..40677a18da 100644 --- a/java/com/google/gerrit/index/QueryOptions.java +++ b/java/com/google/gerrit/index/QueryOptions.java @@ -128,4 +128,8 @@ public abstract class QueryOptions { limit(), filter.apply(this)); } + + public int getLimitBasedOnPaginationType() { + return PaginationType.NONE == config().paginationType() ? limit() : pageSize(); + } } diff --git a/java/com/google/gerrit/index/Schema.java b/java/com/google/gerrit/index/Schema.java index 893f12d40a..974bb74d55 100644 --- a/java/com/google/gerrit/index/Schema.java +++ b/java/com/google/gerrit/index/Schema.java @@ -283,22 +283,19 @@ public class Schema<T> { /** * Build all fields in the schema from an input object. * - * <p>Null values are omitted, as are fields which cause errors, which are logged. + * <p>Null values are omitted, as are fields which cause errors, which are logged. If any of the + * fields cause a StorageException, the whole operation fails and the exception is propagated to + * the caller. * * @param obj input object. * @param skipFields set of field names to skip when indexing the document * @return all non-null field values from the object. */ public final Iterable<Values<T>> buildFields(T obj, ImmutableSet<String> skipFields) { - try { - return schemaFields.values().stream() - .map(f -> fieldValues(obj, f, skipFields)) - .filter(Objects::nonNull) - .collect(toImmutableList()); - - } catch (StorageException e) { - return ImmutableList.of(); - } + return schemaFields.values().stream() + .map(f -> fieldValues(obj, f, skipFields)) + .filter(Objects::nonNull) + .collect(toImmutableList()); } @Override diff --git a/java/com/google/gerrit/index/query/IndexedQuery.java b/java/com/google/gerrit/index/query/IndexedQuery.java index ee25ef96eb..cb98c06a67 100644 --- a/java/com/google/gerrit/index/query/IndexedQuery.java +++ b/java/com/google/gerrit/index/query/IndexedQuery.java @@ -87,6 +87,12 @@ public class IndexedQuery<I, T> extends Predicate<T> implements DataSource<T>, P } @Override + public ResultSet<T> restart(int start) { + opts = opts.withStart(start); + return search(); + } + + @Override public ResultSet<T> restart(int start, int pageSize) { opts = opts.withStart(start).withPageSize(pageSize); return search(); diff --git a/java/com/google/gerrit/index/query/Paginated.java b/java/com/google/gerrit/index/query/Paginated.java index 552199093b..1065019c12 100644 --- a/java/com/google/gerrit/index/query/Paginated.java +++ b/java/com/google/gerrit/index/query/Paginated.java @@ -19,6 +19,8 @@ import com.google.gerrit.index.QueryOptions; public interface Paginated<T> { QueryOptions getOptions(); + ResultSet<T> restart(int start); + ResultSet<T> restart(int start, int pageSize); ResultSet<T> restart(Object searchAfter, int pageSize); diff --git a/java/com/google/gerrit/index/query/PaginatingSource.java b/java/com/google/gerrit/index/query/PaginatingSource.java index 98a0ed33f8..8a2d94e719 100644 --- a/java/com/google/gerrit/index/query/PaginatingSource.java +++ b/java/com/google/gerrit/index/query/PaginatingSource.java @@ -63,42 +63,55 @@ public class PaginatingSource<T> implements DataSource<T> { pageResultSize++; } - if (last != null - && source instanceof Paginated - // TODO: this fix is only for the stable branches and the real refactoring would be to - // restore the logic - // for the filtering in AndSource (L58 - 64) as per - // https://gerrit-review.googlesource.com/c/gerrit/+/345634/9 - && !indexConfig.paginationType().equals(PaginationType.NONE)) { + if (last != null && source instanceof Paginated) { // Restart source and continue if we have not filled the // full limit the caller wants. - // @SuppressWarnings("unchecked") Paginated<T> p = (Paginated<T>) source; QueryOptions opts = p.getOptions(); final int limit = opts.limit(); - int pageSize = opts.pageSize(); - int pageSizeMultiplier = opts.pageSizeMultiplier(); - Object searchAfter = resultSet.searchAfter(); - int nextStart = pageResultSize; - while (pageResultSize == pageSize && r.size() <= limit) { // get 1 more than the limit - pageSize = getNextPageSize(pageSize, pageSizeMultiplier); - ResultSet<T> next = - indexConfig.paginationType().equals(PaginationType.SEARCH_AFTER) - ? p.restart(searchAfter, pageSize) - : p.restart(nextStart, pageSize); - pageResultSize = 0; - for (T data : buffer(next)) { - if (match(data)) { - r.add(data); + + // TODO: this fix is only for the stable branches and the real refactoring would be to + // restore the logic + // for the filtering in AndSource (L58 - 64) as per + // https://gerrit-review.googlesource.com/c/gerrit/+/345634/9 + if (!indexConfig.paginationType().equals(PaginationType.NONE)) { + int pageSize = opts.pageSize(); + int pageSizeMultiplier = opts.pageSizeMultiplier(); + Object searchAfter = resultSet.searchAfter(); + int nextStart = pageResultSize; + while (pageResultSize == pageSize && r.size() <= limit) { // get 1 more than the limit + pageSize = getNextPageSize(pageSize, pageSizeMultiplier); + ResultSet<T> next = + indexConfig.paginationType().equals(PaginationType.SEARCH_AFTER) + ? p.restart(searchAfter, pageSize) + : p.restart(nextStart, pageSize); + pageResultSize = 0; + for (T data : buffer(next)) { + if (match(data)) { + r.add(data); + } + pageResultSize++; + if (r.size() > limit) { + break; + } } - pageResultSize++; - if (r.size() > limit) { - break; + nextStart += pageResultSize; + searchAfter = next.searchAfter(); + } + } else { + int nextStart = pageResultSize; + while (pageResultSize == limit && r.size() < limit) { + ResultSet<T> next = p.restart(nextStart); + pageResultSize = 0; + for (T data : buffer(next)) { + if (match(data)) { + r.add(data); + } + pageResultSize++; } + nextStart += pageResultSize; } - nextStart += pageResultSize; - searchAfter = next.searchAfter(); } } diff --git a/java/com/google/gerrit/index/query/QueryProcessor.java b/java/com/google/gerrit/index/query/QueryProcessor.java index e6077ad4bd..1f8266a3bc 100644 --- a/java/com/google/gerrit/index/query/QueryProcessor.java +++ b/java/com/google/gerrit/index/query/QueryProcessor.java @@ -61,7 +61,7 @@ public abstract class QueryProcessor<T> { protected static class Metrics { final Timer1<String> executionTime; - Metrics(MetricMaker metricMaker) { + protected Metrics(MetricMaker metricMaker) { executionTime = metricMaker.newTimer( "query/query_latency", @@ -95,14 +95,14 @@ public abstract class QueryProcessor<T> { private Set<String> requestedFields; protected QueryProcessor( - MetricMaker metricMaker, + Metrics metrics, SchemaDefinitions<T> schemaDef, IndexConfig indexConfig, IndexCollection<?, T, ? extends Index<?, T>> indexes, IndexRewriter<T> rewriter, String limitField, IntSupplier userQueryLimit) { - this.metrics = new Metrics(metricMaker); + this.metrics = metrics; this.schemaDef = schemaDef; this.indexConfig = indexConfig; this.indexes = indexes; diff --git a/java/com/google/gerrit/index/testing/AbstractFakeIndex.java b/java/com/google/gerrit/index/testing/AbstractFakeIndex.java index 944f9562d9..071c7fbb33 100644 --- a/java/com/google/gerrit/index/testing/AbstractFakeIndex.java +++ b/java/com/google/gerrit/index/testing/AbstractFakeIndex.java @@ -150,10 +150,14 @@ public abstract class AbstractFakeIndex<K, V, D> implements Index<K, V> { .findFirst() .orElse(-1) + 1; - int toIndex = Math.min(fromIndex + opts.pageSize(), valueList.size()); + int toIndex = Math.min(fromIndex + opts.getLimitBasedOnPaginationType(), valueList.size()); results = valueList.subList(fromIndex, toIndex); } else { - results = valueStream.skip(opts.start()).limit(opts.pageSize()).collect(toImmutableList()); + results = + valueStream + .skip(opts.start()) + .limit(opts.getLimitBasedOnPaginationType()) + .collect(toImmutableList()); } queryCount++; resultsSizes.add(results.size()); @@ -238,7 +242,8 @@ public abstract class AbstractFakeIndex<K, V, D> implements Index<K, V> { private final boolean skipMergable; @Inject - FakeChangeIndex( + @VisibleForTesting + protected FakeChangeIndex( SitePaths sitePaths, ChangeData.Factory changeDataFactory, @Assisted Schema<ChangeData> schema, diff --git a/java/com/google/gerrit/lucene/LuceneChangeIndex.java b/java/com/google/gerrit/lucene/LuceneChangeIndex.java index 4c05f70029..57ef441e57 100644 --- a/java/com/google/gerrit/lucene/LuceneChangeIndex.java +++ b/java/com/google/gerrit/lucene/LuceneChangeIndex.java @@ -113,7 +113,7 @@ public class LuceneChangeIndex implements ChangeIndex { private static final String CHANGE_FIELD = ChangeField.CHANGE_SPEC.getName(); static Term idTerm(ChangeData cd) { - return idTerm(cd.getVirtualId()); + return idTerm(cd.virtualId()); } static Term idTerm(Change.Id id) { @@ -400,11 +400,11 @@ public class LuceneChangeIndex implements ChangeIndex { IndexSearcher[] searchers = new IndexSearcher[indexes.size()]; Map<ChangeSubIndex, ScoreDoc> searchAfterBySubIndex = new HashMap<>(); try { - int realPageSize = opts.start() + opts.pageSize(); - if (Integer.MAX_VALUE - opts.pageSize() < opts.start()) { - realPageSize = Integer.MAX_VALUE; + int pageLimit = AbstractLuceneIndex.getLimitBasedOnPaginationType(opts, opts.pageSize()); + int queryLimit = opts.start() + pageLimit; + if (Integer.MAX_VALUE - pageLimit < opts.start()) { + queryLimit = Integer.MAX_VALUE; } - int queryLimit = AbstractLuceneIndex.getLimitBasedOnPaginationType(opts, realPageSize); List<TopFieldDocs> hits = new ArrayList<>(); int searchAfterHitsCount = 0; for (int i = 0; i < indexes.size(); i++) { @@ -412,7 +412,7 @@ public class LuceneChangeIndex implements ChangeIndex { searchers[i] = subIndex.acquire(); if (isSearchAfterPagination) { ScoreDoc searchAfter = getSearchAfter(subIndex); - int maxRemainingHits = realPageSize - searchAfterHitsCount; + int maxRemainingHits = queryLimit - searchAfterHitsCount; if (maxRemainingHits > 0) { TopFieldDocs subIndexHits = searchers[i].searchAfter( @@ -549,7 +549,13 @@ public class LuceneChangeIndex implements ChangeIndex { IndexableField cb = Iterables.getFirst(doc.get(CHANGE_FIELD), null); if (cb != null) { BytesRef proto = cb.binaryValue(); - cd = changeDataFactory.create(parseProtoFrom(proto, ChangeProtoConverter.INSTANCE)); + // pass the id field value (which is the change virtual id for the imported changes) when + // available + IndexableField f = Iterables.getFirst(doc.get(idFieldName), null); + cd = + changeDataFactory.create( + parseProtoFrom(proto, ChangeProtoConverter.INSTANCE), + f != null ? Change.id(Integer.valueOf(f.stringValue())) : null); } else { IndexableField f = Iterables.getFirst(doc.get(idFieldName), null); @@ -560,7 +566,7 @@ public class LuceneChangeIndex implements ChangeIndex { } for (SchemaField<ChangeData, ?> field : getSchema().getSchemaFields().values()) { - if (fields.contains(field.getName()) && doc.get(field.getName()) != null) { + if (fields.contains(field.getName())) { field.setIfPossible(cd, new LuceneStoredValue(doc.get(field.getName()))); } } diff --git a/java/com/google/gerrit/pgm/Daemon.java b/java/com/google/gerrit/pgm/Daemon.java index 7474709353..72c465ddde 100644 --- a/java/com/google/gerrit/pgm/Daemon.java +++ b/java/com/google/gerrit/pgm/Daemon.java @@ -85,8 +85,8 @@ import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.config.SysExecutorModule; import com.google.gerrit.server.events.EventBroker.EventBrokerModule; import com.google.gerrit.server.events.StreamEventsApiListener.StreamEventsApiListenerModule; +import com.google.gerrit.server.git.ChangesByProjectCache; import com.google.gerrit.server.git.GarbageCollectionModule; -import com.google.gerrit.server.git.SearchingChangeCacheImpl.SearchingChangeCacheImplModule; import com.google.gerrit.server.git.WorkQueue.WorkQueueModule; import com.google.gerrit.server.group.PeriodicGroupIndexer.PeriodicGroupIndexerModule; import com.google.gerrit.server.index.AbstractIndexModule; @@ -140,7 +140,6 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import javax.servlet.http.HttpServletRequest; import org.eclipse.jgit.lib.Config; import org.kohsuke.args4j.Option; @@ -467,7 +466,10 @@ public class Daemon extends SiteProgram { modules.add(new DefaultRefLogIdentityProvider.Module()); modules.add(new PluginApiModule()); - modules.add(new SearchingChangeCacheImplModule(replica)); + modules.add( + new ChangesByProjectCache.Module( + replica ? ChangesByProjectCache.UseIndex.FALSE : ChangesByProjectCache.UseIndex.TRUE, + config)); modules.add(new InternalAccountDirectoryModule()); modules.add(new DefaultPermissionBackendModule()); modules.add(new DefaultMemoryCacheModule()); @@ -610,10 +612,6 @@ public class Daemon extends SiteProgram { sysInjector.getInstance(PluginGuiceEnvironment.class).setHttpInjector(webInjector); - sysInjector - .getInstance(HttpCanonicalWebUrlProvider.class) - .setHttpServletRequest(webInjector.getProvider(HttpServletRequest.class)); - httpdInjector = createHttpdInjector(); manager.add(webInjector, httpdInjector); } diff --git a/java/com/google/gerrit/pgm/Reindex.java b/java/com/google/gerrit/pgm/Reindex.java index feae28438e..b4344d749d 100644 --- a/java/com/google/gerrit/pgm/Reindex.java +++ b/java/com/google/gerrit/pgm/Reindex.java @@ -40,6 +40,7 @@ import com.google.gerrit.server.git.WorkQueue.WorkQueueModule; import com.google.gerrit.server.index.IndexModule; import com.google.gerrit.server.index.change.ChangeSchemaDefinitions; import com.google.gerrit.server.index.options.AutoFlush; +import com.google.gerrit.server.index.options.BuildBloomFilter; import com.google.gerrit.server.index.options.IsFirstInsertForEntry; import com.google.gerrit.server.plugins.PluginGuiceEnvironment; import com.google.gerrit.server.util.ReplicaUtil; @@ -90,6 +91,9 @@ public class Reindex extends SiteProgram { @Option(name = "--show-cache-stats", usage = "Show cache statistics at the end.") private boolean showCacheStats; + @Option(name = "--build-bloom-filter", usage = "Build bloom filter for H2 disk caches.") + private boolean buildBloomFilter; + private Injector dbInjector; private Injector sysInjector; private Injector cfgInjector; @@ -207,6 +211,9 @@ public class Reindex extends SiteProgram { OptionalBinder.newOptionalBinder(binder(), IsFirstInsertForEntry.class) .setBinding() .toInstance(IsFirstInsertForEntry.YES); + OptionalBinder.newOptionalBinder(binder(), BuildBloomFilter.class) + .setBinding() + .toInstance(buildBloomFilter ? BuildBloomFilter.TRUE : BuildBloomFilter.FALSE); } }); modules.add(new BatchProgramModule(dbInjector)); diff --git a/java/com/google/gerrit/pgm/http/jetty/JettyServer.java b/java/com/google/gerrit/pgm/http/jetty/JettyServer.java index 6f3514f974..10fe2f374c 100644 --- a/java/com/google/gerrit/pgm/http/jetty/JettyServer.java +++ b/java/com/google/gerrit/pgm/http/jetty/JettyServer.java @@ -14,7 +14,9 @@ package com.google.gerrit.pgm.http.jetty; +import static com.google.gerrit.httpd.CacheBasedWebSession.MAX_AGE_MINUTES; import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.SECONDS; import com.google.common.annotations.VisibleForTesting; @@ -244,6 +246,15 @@ public class JettyServer { } }); + sessionHandler.setMaxInactiveInterval( + (int) + cfg.getTimeUnit( + "cache", + "web_sessions", + "maxAge", + SECONDS.convert(MAX_AGE_MINUTES, MINUTES), + SECONDS)); + Handler app = makeContext(env, cfg, sessionHandler); if (cfg.getBoolean("httpd", "requestLog", !reverseProxy)) { RequestLogHandler handler = new RequestLogHandler(); diff --git a/java/com/google/gerrit/pgm/util/BatchProgramModule.java b/java/com/google/gerrit/pgm/util/BatchProgramModule.java index cae7ca6733..1e41cbc1bf 100644 --- a/java/com/google/gerrit/pgm/util/BatchProgramModule.java +++ b/java/com/google/gerrit/pgm/util/BatchProgramModule.java @@ -62,14 +62,15 @@ import com.google.gerrit.server.config.EnablePeerIPInReflogRecordProvider; import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.config.GitReceivePackGroups; import com.google.gerrit.server.config.GitUploadPackGroups; +import com.google.gerrit.server.config.SkipCurrentRulesEvaluationOnClosedChangesModule; import com.google.gerrit.server.config.SysExecutorModule; import com.google.gerrit.server.extensions.events.AttentionSetObserver; import com.google.gerrit.server.extensions.events.EventUtil; import com.google.gerrit.server.extensions.events.GitReferenceUpdated; import com.google.gerrit.server.extensions.events.RevisionCreated; import com.google.gerrit.server.extensions.events.WorkInProgressStateChanged; +import com.google.gerrit.server.git.ChangesByProjectCache; import com.google.gerrit.server.git.PureRevertCache; -import com.google.gerrit.server.git.SearchingChangeCacheImpl; import com.google.gerrit.server.git.TagCache; import com.google.gerrit.server.notedb.NoteDbModule; import com.google.gerrit.server.patch.DiffExecutorModule; @@ -162,10 +163,6 @@ public class BatchProgramModule extends FactoryModule { factory(PatchSetInserter.Factory.class); factory(RebaseChangeOp.Factory.class); - // As Reindex is a batch program, don't assume the index is available for - // the change cache. - bind(SearchingChangeCacheImpl.class).toProvider(Providers.of(null)); - bind(new TypeLiteral<ImmutableSet<GroupReference>>() {}) .annotatedWith(AdministrateServerGroups.class) .toInstance(ImmutableSet.of()); @@ -177,6 +174,8 @@ public class BatchProgramModule extends FactoryModule { .toInstance(Collections.emptySet()); modules.add(new BatchGitModule()); + modules.add( + new ChangesByProjectCache.Module(ChangesByProjectCache.UseIndex.FALSE, getConfig())); modules.add(new DefaultPermissionBackendModule()); modules.add(new DefaultMemoryCacheModule()); modules.add(new H2CacheModule()); @@ -214,6 +213,7 @@ public class BatchProgramModule extends FactoryModule { modules.add(new PrologModule(getConfig())); modules.add(new DefaultSubmitRuleModule()); modules.add(new IgnoreSelfApprovalRuleModule()); + modules.add(new SkipCurrentRulesEvaluationOnClosedChangesModule()); // Global submit requirements DynamicSet.setOf(binder(), SubmitRequirement.class); diff --git a/java/com/google/gerrit/server/CommentsUtil.java b/java/com/google/gerrit/server/CommentsUtil.java index 285657e860..85ad6ccaf9 100644 --- a/java/com/google/gerrit/server/CommentsUtil.java +++ b/java/com/google/gerrit/server/CommentsUtil.java @@ -46,6 +46,7 @@ import com.google.gerrit.server.patch.DiffNotAvailableException; import com.google.gerrit.server.patch.DiffOperations; import com.google.gerrit.server.patch.DiffOptions; import com.google.gerrit.server.patch.filediff.FileDiffOutput; +import com.google.gerrit.server.query.change.ChangeNumberVirtualIdAlgorithm; import com.google.gerrit.server.update.ChangeContext; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -118,17 +119,20 @@ public class CommentsUtil { private final GitRepositoryManager repoManager; private final AllUsersName allUsers; private final String serverId; + private final ChangeNumberVirtualIdAlgorithm virtualIdAlgorithm; @Inject CommentsUtil( DiffOperations diffOperations, GitRepositoryManager repoManager, AllUsersName allUsers, - @GerritServerId String serverId) { + @GerritServerId String serverId, + @Nullable ChangeNumberVirtualIdAlgorithm virtualIdAlgorithm) { this.diffOperations = diffOperations; this.repoManager = repoManager; this.allUsers = allUsers; this.serverId = serverId; + this.virtualIdAlgorithm = virtualIdAlgorithm; } public HumanComment newHumanComment( @@ -225,7 +229,7 @@ public class CommentsUtil { public List<HumanComment> draftByChange(ChangeNotes notes) { List<HumanComment> comments = new ArrayList<>(); - for (Ref ref : getDraftRefs(notes.getChangeId())) { + for (Ref ref : getDraftRefs(getVirtualId(notes))) { Account.Id account = Account.Id.fromRefSuffix(ref.getName()); if (account != null) { comments.addAll(draftByChangeAuthor(notes, account)); @@ -324,17 +328,19 @@ public class CommentsUtil { public List<HumanComment> draftByPatchSetAuthor( PatchSet.Id psId, Account.Id author, ChangeNotes notes) { - return commentsOnPatchSet(notes.load().getDraftComments(author).values(), psId); + return commentsOnPatchSet( + notes.load().getDraftComments(author, getVirtualId(notes)).values(), psId); } public List<HumanComment> draftByChangeFileAuthor( ChangeNotes notes, String file, Account.Id author) { - return commentsOnFile(notes.load().getDraftComments(author).values(), file); + return commentsOnFile( + notes.load().getDraftComments(author, getVirtualId(notes)).values(), file); } public List<HumanComment> draftByChangeAuthor(ChangeNotes notes, Account.Id author) { List<HumanComment> comments = new ArrayList<>(); - comments.addAll(notes.getDraftComments(author).values()); + comments.addAll(notes.getDraftComments(author, getVirtualId(notes)).values()); return sort(comments); } @@ -478,8 +484,8 @@ public class CommentsUtil { } } - private Collection<Ref> getDraftRefs(Repository repo, Change.Id changeId) throws IOException { - return repo.getRefDatabase().getRefsByPrefix(RefNames.refsDraftCommentsPrefix(changeId)); + private Collection<Ref> getDraftRefs(Repository repo, Change.Id virtualId) throws IOException { + return repo.getRefDatabase().getRefsByPrefix(RefNames.refsDraftCommentsPrefix(virtualId)); } private Collection<Change.Id> getChangesWithDrafts(Repository repo, Account.Id accountId) @@ -502,4 +508,10 @@ public class CommentsUtil { comments.sort(COMMENT_ORDER); return comments; } + + private Change.Id getVirtualId(ChangeNotes notes) { + return virtualIdAlgorithm == null + ? notes.getChangeId() + : virtualIdAlgorithm.apply(notes.getServerId(), notes.getChangeId()); + } } diff --git a/java/com/google/gerrit/server/DeadlineChecker.java b/java/com/google/gerrit/server/DeadlineChecker.java index f41b1e3c3d..9b7ffe6c3f 100644 --- a/java/com/google/gerrit/server/DeadlineChecker.java +++ b/java/com/google/gerrit/server/DeadlineChecker.java @@ -180,12 +180,14 @@ public class DeadlineChecker implements RequestStateProvider { this.timeoutName = clientedProvidedTimeout .map(clientTimeout -> "client.timeout") - .orElse( - serverSideDeadline - .map(serverDeadline -> serverDeadline.id() + ".timeout") - .orElse("timeout")); + .orElseGet( + () -> + serverSideDeadline + .map(serverDeadline -> serverDeadline.id() + ".timeout") + .orElse("timeout")); this.timeout = - clientedProvidedTimeout.orElse(serverSideDeadline.map(ServerDeadline::timeout).orElse(0L)); + clientedProvidedTimeout.orElseGet( + () -> serverSideDeadline.map(ServerDeadline::timeout).orElse(0L)); this.deadline = timeout > 0 ? Optional.of(start + timeout) : Optional.empty(); } diff --git a/java/com/google/gerrit/server/StarredChangesUtil.java b/java/com/google/gerrit/server/StarredChangesUtil.java index 2d1805451f..75729398f7 100644 --- a/java/com/google/gerrit/server/StarredChangesUtil.java +++ b/java/com/google/gerrit/server/StarredChangesUtil.java @@ -49,10 +49,12 @@ import com.google.inject.Singleton; import java.io.IOException; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.NavigableSet; import java.util.Objects; import java.util.Set; import java.util.TreeSet; +import java.util.stream.Collectors; import org.eclipse.jgit.lib.BatchRefUpdate; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.NullProgressMonitor; @@ -176,22 +178,22 @@ public class StarredChangesUtil { this.serverIdent = serverIdent; } - public NavigableSet<String> getLabels(Account.Id accountId, Change.Id changeId) { + public NavigableSet<String> getLabels(Account.Id accountId, Change.Id virtualId) { try (Repository repo = repoManager.openRepository(allUsers)) { - return readLabels(repo, RefNames.refsStarredChanges(changeId, accountId)).labels(); + return readLabels(repo, RefNames.refsStarredChanges(virtualId, accountId)).labels(); } catch (IOException e) { throw new StorageException( String.format( "Reading stars from change %d for account %d failed", - changeId.get(), accountId.get()), + virtualId.get(), accountId.get()), e); } } - public void star(Account.Id accountId, Change.Id changeId, Operation op) + public void star(Account.Id accountId, Change.Id virtualId, Operation op) throws IllegalLabelException { try (Repository repo = repoManager.openRepository(allUsers)) { - String refName = RefNames.refsStarredChanges(changeId, accountId); + String refName = RefNames.refsStarredChanges(virtualId, accountId); StarRef old = readLabels(repo, refName); NavigableSet<String> labels = new TreeSet<>(old.labels()); @@ -211,29 +213,52 @@ public class StarredChangesUtil { } } catch (IOException e) { throw new StorageException( - String.format("Star change %d for account %d failed", changeId.get(), accountId.get()), + String.format("Star change %d for account %d failed", virtualId.get(), accountId.get()), e); } } /** + * Returns a subset of change IDs among the input {@code virtualIds} list that are starred by the + * {@code caller} user. + */ + public Set<Change.Id> areStarred( + Repository allUsersRepo, List<Change.Id> virtualIds, Account.Id caller) { + List<String> starRefs = + virtualIds.stream() + .map(c -> RefNames.refsStarredChanges(c, caller)) + .collect(Collectors.toList()); + try { + return allUsersRepo.getRefDatabase().exactRef(starRefs.toArray(new String[0])).keySet() + .stream() + .map(r -> Change.Id.fromAllUsersRef(r)) + .collect(Collectors.toSet()); + } catch (IOException e) { + logger.atWarning().withCause(e).log( + "Failed getting starred changes for account %d within changes: %s", + caller.get(), Joiner.on(", ").join(virtualIds)); + return ImmutableSet.of(); + } + } + + /** * Unstar the given change for all users. * * <p>Intended for use only when we're about to delete a change. For that reason, the change is * not reindexed. * - * @param changeId change ID. + * @param virtualId change ID. * @throws IOException if an error occurred. */ - public void unstarAllForChangeDeletion(Change.Id changeId) throws IOException { + public void unstarAllForChangeDeletion(Change.Id virtualId) throws IOException { try (Repository repo = repoManager.openRepository(allUsers); RevWalk rw = new RevWalk(repo)) { BatchRefUpdate batchUpdate = repo.getRefDatabase().newBatchUpdate(); batchUpdate.setAllowNonFastForwards(true); batchUpdate.setRefLogIdent(serverIdent.get()); - batchUpdate.setRefLogMessage("Unstar change " + changeId.get(), true); - for (Account.Id accountId : getStars(repo, changeId)) { - String refName = RefNames.refsStarredChanges(changeId, accountId); + batchUpdate.setRefLogMessage("Unstar change " + virtualId.get(), true); + for (Account.Id accountId : getStars(repo, virtualId)) { + String refName = RefNames.refsStarredChanges(virtualId, accountId); Ref ref = repo.getRefDatabase().exactRef(refName); if (ref != null) { batchUpdate.addCommand(new ReceiveCommand(ref.getObjectId(), ObjectId.zeroId(), refName)); @@ -245,7 +270,7 @@ public class StarredChangesUtil { String message = String.format( "Unstar change %d failed, ref %s could not be deleted: %s", - changeId.get(), command.getRefName(), command.getResult()); + virtualId.get(), command.getRefName(), command.getResult()); if (command.getResult() == ReceiveCommand.Result.LOCK_FAILURE) { throw new LockFailureException(message, batchUpdate); } @@ -255,16 +280,16 @@ public class StarredChangesUtil { } } - public ImmutableMap<Account.Id, StarRef> byChange(Change.Id changeId) { + public ImmutableMap<Account.Id, StarRef> byChange(Change.Id virtualId) { try (Repository repo = repoManager.openRepository(allUsers)) { ImmutableMap.Builder<Account.Id, StarRef> builder = ImmutableMap.builder(); - for (Account.Id accountId : getStars(repo, changeId)) { - builder.put(accountId, readLabels(repo, RefNames.refsStarredChanges(changeId, accountId))); + for (Account.Id accountId : getStars(repo, virtualId)) { + builder.put(accountId, readLabels(repo, RefNames.refsStarredChanges(virtualId, accountId))); } return builder.build(); } catch (IOException e) { throw new StorageException( - String.format("Get accounts that starred change %d failed", changeId.get()), e); + String.format("Get accounts that starred change %d failed", virtualId.get()), e); } } @@ -297,9 +322,9 @@ public class StarredChangesUtil { } } - private static Set<Account.Id> getStars(Repository allUsers, Change.Id changeId) + private static Set<Account.Id> getStars(Repository allUsers, Change.Id virtualId) throws IOException { - String prefix = RefNames.refsStarredChangesPrefix(changeId); + String prefix = RefNames.refsStarredChangesPrefix(virtualId); RefDatabase refDb = allUsers.getRefDatabase(); return refDb.getRefsByPrefix(prefix).stream() .map(r -> r.getName().substring(prefix.length())) @@ -309,14 +334,14 @@ public class StarredChangesUtil { .collect(toSet()); } - public ObjectId getObjectId(Account.Id accountId, Change.Id changeId) { + public ObjectId getObjectId(Account.Id accountId, Change.Id virtualId) { try (Repository repo = repoManager.openRepository(allUsers)) { - Ref ref = repo.exactRef(RefNames.refsStarredChanges(changeId, accountId)); + Ref ref = repo.exactRef(RefNames.refsStarredChanges(virtualId, accountId)); return ref != null ? ref.getObjectId() : ObjectId.zeroId(); } catch (IOException e) { logger.atSevere().withCause(e).log( "Getting star object ID for account %d on change %d failed", - accountId.get(), changeId.get()); + accountId.get(), virtualId.get()); return ObjectId.zeroId(); } } diff --git a/java/com/google/gerrit/server/account/AccountCacheImpl.java b/java/com/google/gerrit/server/account/AccountCacheImpl.java index 66a36f6595..d306ad0e92 100644 --- a/java/com/google/gerrit/server/account/AccountCacheImpl.java +++ b/java/com/google/gerrit/server/account/AccountCacheImpl.java @@ -98,7 +98,7 @@ public class AccountCacheImpl implements AccountCache { @Override public AccountState getEvenIfMissing(Account.Id accountId) { - return get(accountId).orElse(missing(accountId)); + return get(accountId).orElseGet(() -> missing(accountId)); } @Override diff --git a/java/com/google/gerrit/server/account/AccountManager.java b/java/com/google/gerrit/server/account/AccountManager.java index edec52c1f7..f5e9dcc12c 100644 --- a/java/com/google/gerrit/server/account/AccountManager.java +++ b/java/com/google/gerrit/server/account/AccountManager.java @@ -234,7 +234,7 @@ public class AccountManager { "Unable to deactivate account %s", authRequest .getUserName() - .orElse(" for external ID key " + authRequest.getExternalIdKey().get())); + .orElseGet(() -> " for external ID key " + authRequest.getExternalIdKey().get())); } } diff --git a/java/com/google/gerrit/server/account/AccountProperties.java b/java/com/google/gerrit/server/account/AccountProperties.java index 928d851729..5f56aa3485 100644 --- a/java/com/google/gerrit/server/account/AccountProperties.java +++ b/java/com/google/gerrit/server/account/AccountProperties.java @@ -124,6 +124,7 @@ public class AccountProperties { * @param key the key * @return the value, {@code null} if key was not set or key was set to empty string */ + @Nullable private static String get(Config cfg, String key) { return Strings.emptyToNull(cfg.getString(ACCOUNT, null, key)); } diff --git a/java/com/google/gerrit/server/account/AccountResource.java b/java/com/google/gerrit/server/account/AccountResource.java index 9629809eca..14b363b2f7 100644 --- a/java/com/google/gerrit/server/account/AccountResource.java +++ b/java/com/google/gerrit/server/account/AccountResource.java @@ -99,6 +99,10 @@ public class AccountResource implements RestResource { public Change getChange() { return change.getChange(); } + + public Change.Id getVirtualId() { + return change.getVirtualId(); + } } public static class Star implements RestResource { diff --git a/java/com/google/gerrit/server/account/GroupCacheImpl.java b/java/com/google/gerrit/server/account/GroupCacheImpl.java index 6f4fce9434..fac2fd5135 100644 --- a/java/com/google/gerrit/server/account/GroupCacheImpl.java +++ b/java/com/google/gerrit/server/account/GroupCacheImpl.java @@ -304,7 +304,7 @@ public class GroupCacheImpl implements GroupCache { List<Cache.GroupKeyProto> keyList = new ArrayList<>(); try (TraceTimer ignored = TraceContext.newTimer( - "Loading group from serialized cache", + "Building keys to load group(s) from serialized cache", Metadata.builder().cacheName(BYUUID_NAME_PERSISTED).build()); Repository allUsers = repoManager.openRepository(allUsersName)) { while (uuidIterator.hasNext()) { @@ -323,8 +323,13 @@ public class GroupCacheImpl implements GroupCache { keyList.add(key); } } - persistedCache.getAll(keyList).entrySet().stream() - .forEach(g -> toReturn.put(g.getKey().getUuid(), Optional.of(g.getValue()))); + try (TraceTimer ignored = + TraceContext.newTimer( + "Loading group(s) from serialized cache", + Metadata.builder().cacheName(BYUUID_NAME_PERSISTED).build())) { + persistedCache.getAll(keyList).entrySet().stream() + .forEach(g -> toReturn.put(g.getKey().getUuid(), Optional.of(g.getValue()))); + } return toReturn; } } diff --git a/java/com/google/gerrit/server/approval/ApprovalsUtil.java b/java/com/google/gerrit/server/approval/ApprovalsUtil.java index 09820b13f9..8fae13a34f 100644 --- a/java/com/google/gerrit/server/approval/ApprovalsUtil.java +++ b/java/com/google/gerrit/server/approval/ApprovalsUtil.java @@ -377,6 +377,11 @@ public class ApprovalsUtil { return notes.load().getApprovals().onlyNonCopied(); } + public ListMultimap<PatchSet.Id, PatchSetApproval> byChangeIncludingCopiedApprovals( + ChangeNotes notes) { + return notes.load().getApprovals().all(); + } + /** * Copies approvals to a new patch set. * diff --git a/java/com/google/gerrit/server/cache/CacheInfo.java b/java/com/google/gerrit/server/cache/CacheInfo.java index 94a9e05a61..76756c2ed9 100644 --- a/java/com/google/gerrit/server/cache/CacheInfo.java +++ b/java/com/google/gerrit/server/cache/CacheInfo.java @@ -92,7 +92,7 @@ public class CacheInfo { space = bytes(value); } - private static String bytes(double value) { + public static String bytes(double value) { value /= 1024; String suffix = "k"; diff --git a/java/com/google/gerrit/server/cache/h2/H2CacheFactory.java b/java/com/google/gerrit/server/cache/h2/H2CacheFactory.java index 445d8a0178..fdd55ac1df 100644 --- a/java/com/google/gerrit/server/cache/h2/H2CacheFactory.java +++ b/java/com/google/gerrit/server/cache/h2/H2CacheFactory.java @@ -14,11 +14,15 @@ package com.google.gerrit.server.cache.h2; +import static java.util.concurrent.TimeUnit.SECONDS; + +import com.google.common.base.Strings; import com.google.common.cache.Cache; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.flogger.FluentLogger; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.gerrit.common.Nullable; import com.google.gerrit.extensions.events.LifecycleListener; import com.google.gerrit.extensions.registration.DynamicMap; import com.google.gerrit.server.cache.MemoryCacheFactory; @@ -26,13 +30,17 @@ import com.google.gerrit.server.cache.PersistentCacheBaseFactory; import com.google.gerrit.server.cache.PersistentCacheDef; import com.google.gerrit.server.cache.h2.H2CacheImpl.SqlStore; import com.google.gerrit.server.cache.h2.H2CacheImpl.ValueHolder; +import com.google.gerrit.server.config.ConfigUtil; import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.index.options.BuildBloomFilter; +import com.google.gerrit.server.index.options.IsFirstInsertForEntry; import com.google.gerrit.server.logging.LoggingContextAwareExecutorService; import com.google.gerrit.server.logging.LoggingContextAwareScheduledExecutorService; import com.google.inject.Inject; import com.google.inject.Provider; import com.google.inject.Singleton; +import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -57,32 +65,43 @@ class H2CacheFactory extends PersistentCacheBaseFactory implements LifecycleList private final ScheduledExecutorService cleanup; private final long h2CacheSize; private final boolean h2AutoServer; + private final boolean isOfflineReindex; + private final boolean buildBloomFilter; @Inject H2CacheFactory( MemoryCacheFactory memCacheFactory, @GerritServerConfig Config cfg, SitePaths site, - DynamicMap<Cache<?, ?>> cacheMap) { + DynamicMap<Cache<?, ?>> cacheMap, + @Nullable IsFirstInsertForEntry isFirstInsertForEntry, + @Nullable BuildBloomFilter buildBloomFilter) { super(memCacheFactory, cfg, site); h2CacheSize = cfg.getLong("cache", null, "h2CacheSize", -1); h2AutoServer = cfg.getBoolean("cache", null, "h2AutoServer", false); caches = new ArrayList<>(); this.cacheMap = cacheMap; + this.isOfflineReindex = + isFirstInsertForEntry != null && isFirstInsertForEntry.equals(IsFirstInsertForEntry.YES); + this.buildBloomFilter = + !(buildBloomFilter != null && buildBloomFilter.equals(BuildBloomFilter.FALSE)); if (diskEnabled) { executor = new LoggingContextAwareExecutorService( Executors.newFixedThreadPool( 1, new ThreadFactoryBuilder().setNameFormat("DiskCache-Store-%d").build())); + cleanup = - new LoggingContextAwareScheduledExecutorService( - Executors.newScheduledThreadPool( - 1, - new ThreadFactoryBuilder() - .setNameFormat("DiskCache-Prune-%d") - .setDaemon(true) - .build())); + isOfflineReindex + ? null + : new LoggingContextAwareScheduledExecutorService( + Executors.newScheduledThreadPool( + 1, + new ThreadFactoryBuilder() + .setNameFormat("DiskCache-Prune-%d") + .setDaemon(true) + .build())); } else { executor = null; cleanup = null; @@ -94,9 +113,11 @@ class H2CacheFactory extends PersistentCacheBaseFactory implements LifecycleList if (executor != null) { for (H2CacheImpl<?, ?> cache : caches) { executor.execute(cache::start); - @SuppressWarnings("unused") - Future<?> possiblyIgnoredError = - cleanup.schedule(() -> cache.prune(cleanup), 30, TimeUnit.SECONDS); + if (cleanup != null) { + @SuppressWarnings("unused") + Future<?> possiblyIgnoredError = + cleanup.schedule(() -> cache.prune(cleanup), 30, TimeUnit.SECONDS); + } } } } @@ -105,7 +126,9 @@ class H2CacheFactory extends PersistentCacheBaseFactory implements LifecycleList public void stop() { if (executor != null) { try { - cleanup.shutdownNow(); + if (cleanup != null) { + cleanup.shutdownNow(); + } List<Runnable> pending = executor.shutdownNow(); if (executor.awaitTermination(15, TimeUnit.MINUTES)) { @@ -183,6 +206,22 @@ class H2CacheFactory extends PersistentCacheBaseFactory implements LifecycleList if (h2AutoServer) { url.append(";AUTO_SERVER=TRUE"); } + Duration refreshAfterWrite = def.refreshAfterWrite(); + if (has(def.configKey(), "refreshAfterWrite")) { + long refreshAfterWriteInSec = + ConfigUtil.getTimeUnit(config, "cache", def.configKey(), "refreshAfterWrite", 0, SECONDS); + if (refreshAfterWriteInSec != 0) { + refreshAfterWrite = Duration.ofSeconds(refreshAfterWriteInSec); + } + } + Duration expireAfterWrite = def.expireAfterWrite(); + if (has(def.configKey(), "maxAge")) { + long expireAfterWriteInsec = + ConfigUtil.getTimeUnit(config, "cache", def.configKey(), "maxAge", 0, SECONDS); + if (expireAfterWriteInsec != 0) { + expireAfterWrite = Duration.ofSeconds(expireAfterWriteInsec); + } + } return new SqlStore<>( url.toString(), def.keyType(), @@ -190,7 +229,12 @@ class H2CacheFactory extends PersistentCacheBaseFactory implements LifecycleList def.valueSerializer(), def.version(), maxSize, - def.expireAfterWrite(), - def.expireFromMemoryAfterAccess()); + expireAfterWrite, + refreshAfterWrite, + buildBloomFilter); + } + + private boolean has(String name, String var) { + return !Strings.isNullOrEmpty(config.getString("cache", name, var)); } } diff --git a/java/com/google/gerrit/server/cache/h2/H2CacheImpl.java b/java/com/google/gerrit/server/cache/h2/H2CacheImpl.java index 8327b888a1..27a09ed70d 100644 --- a/java/com/google/gerrit/server/cache/h2/H2CacheImpl.java +++ b/java/com/google/gerrit/server/cache/h2/H2CacheImpl.java @@ -28,6 +28,7 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.gerrit.common.Nullable; +import com.google.gerrit.server.cache.CacheInfo; import com.google.gerrit.server.cache.PersistentCache; import com.google.gerrit.server.cache.serialize.CacheSerializer; import com.google.gerrit.server.logging.Metadata; @@ -365,6 +366,7 @@ public class H2CacheImpl<K, V> extends AbstractLoadingCache<K, V> implements Per private final AtomicLong missCount = new AtomicLong(); private volatile BloomFilter<K> bloomFilter; private int estimatedSize; + private boolean buildBloomFilter; SqlStore( String jdbcUrl, @@ -374,7 +376,8 @@ public class H2CacheImpl<K, V> extends AbstractLoadingCache<K, V> implements Per int version, long maxSize, @Nullable Duration expireAfterWrite, - @Nullable Duration refreshAfterWrite) { + @Nullable Duration refreshAfterWrite, + boolean buildBloomFilter) { this.url = jdbcUrl; this.keyType = createKeyType(keyType, keySerializer); this.valueSerializer = valueSerializer; @@ -382,6 +385,7 @@ public class H2CacheImpl<K, V> extends AbstractLoadingCache<K, V> implements Per this.maxSize = maxSize; this.expireAfterWrite = expireAfterWrite; this.refreshAfterWrite = refreshAfterWrite; + this.buildBloomFilter = buildBloomFilter; int cores = Runtime.getRuntime().availableProcessors(); int keep = Math.min(cores, 16); @@ -398,7 +402,7 @@ public class H2CacheImpl<K, V> extends AbstractLoadingCache<K, V> implements Per } synchronized void open() { - if (bloomFilter == null) { + if (buildBloomFilter && bloomFilter == null) { bloomFilter = buildBloomFilter(); } } @@ -412,7 +416,7 @@ public class H2CacheImpl<K, V> extends AbstractLoadingCache<K, V> implements Per boolean mightContain(K key) { BloomFilter<K> b = bloomFilter; - if (b == null) { + if (buildBloomFilter && b == null) { synchronized (this) { b = bloomFilter; if (b == null) { @@ -660,12 +664,19 @@ public class H2CacheImpl<K, V> extends AbstractLoadingCache<K, V> implements Per try (ResultSet r = s.executeQuery("SELECT SUM(space) FROM data")) { used = r.next() ? r.getLong(1) : 0; } + String formattedMaxSize = CacheInfo.EntriesInfo.bytes(maxSize); if (used <= maxSize) { + logger.atFine().log( + "Cache %s size (%s) is less than maxSize (%s), not pruning", + url, CacheInfo.EntriesInfo.bytes(used), formattedMaxSize); return; } try (ResultSet r = s.executeQuery("SELECT k, space, created FROM data ORDER BY accessed")) { + logger.atInfo().log( + "Cache %s size (%s) is greater than maxSize (%s), pruning", + url, CacheInfo.EntriesInfo.bytes(used), formattedMaxSize); while (maxSize < used && r.next()) { K key = keyType.get(r, 1); Timestamp created = r.getTimestamp(3); @@ -676,6 +687,9 @@ public class H2CacheImpl<K, V> extends AbstractLoadingCache<K, V> implements Per used -= r.getLong(2); } } + logger.atInfo().log( + "Done pruning cache %s, size (%s) is now less than maxSize (%s)", + url, CacheInfo.EntriesInfo.bytes(used), formattedMaxSize); } } } catch (IOException | SQLException e) { diff --git a/java/com/google/gerrit/server/change/ChangeJson.java b/java/com/google/gerrit/server/change/ChangeJson.java index f733a7b2e6..da05f29b6c 100644 --- a/java/com/google/gerrit/server/change/ChangeJson.java +++ b/java/com/google/gerrit/server/change/ChangeJson.java @@ -30,6 +30,7 @@ import static com.google.gerrit.extensions.client.ListChangesOption.MESSAGES; import static com.google.gerrit.extensions.client.ListChangesOption.REVIEWED; import static com.google.gerrit.extensions.client.ListChangesOption.REVIEWER_UPDATES; import static com.google.gerrit.extensions.client.ListChangesOption.SKIP_DIFFSTAT; +import static com.google.gerrit.extensions.client.ListChangesOption.STAR; import static com.google.gerrit.extensions.client.ListChangesOption.SUBMITTABLE; import static com.google.gerrit.extensions.client.ListChangesOption.SUBMIT_REQUIREMENTS; import static com.google.gerrit.extensions.client.ListChangesOption.TRACKING_IDS; @@ -99,8 +100,10 @@ import com.google.gerrit.server.StarredChangesUtil; import com.google.gerrit.server.account.AccountInfoComparator; import com.google.gerrit.server.account.AccountLoader; import com.google.gerrit.server.cancellation.RequestCancelledException; +import com.google.gerrit.server.config.AllUsersName; import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.config.TrackingFooters; +import com.google.gerrit.server.git.GitRepositoryManager; import com.google.gerrit.server.index.change.ChangeField; import com.google.gerrit.server.notedb.ChangeNotes; import com.google.gerrit.server.notedb.ReviewerStateInternal; @@ -130,6 +133,7 @@ import java.util.Set; import java.util.stream.Collectors; import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Repository; /** * Produces {@link ChangeInfo} (which is serialized to JSON afterwards) from {@link ChangeData}. @@ -219,12 +223,15 @@ public class ChangeJson { } } + private final GitRepositoryManager repoManager; + private final AllUsersName allUsers; private final Provider<CurrentUser> userProvider; private final PermissionBackend permissionBackend; private final ChangeData.Factory changeDataFactory; private final AccountLoader.Factory accountLoaderFactory; private final ImmutableSet<ListChangesOption> options; private final ChangeMessagesUtil cmUtil; + private final StarredChangesUtil starredChangesUtil; private final Provider<ConsistencyChecker> checkerProvider; private final ActionJson actionJson; private final ChangeNotes.Factory notesFactory; @@ -243,11 +250,14 @@ public class ChangeJson { @Inject ChangeJson( + GitRepositoryManager repoManager, + AllUsersName allUsers, Provider<CurrentUser> user, PermissionBackend permissionBackend, ChangeData.Factory cdf, AccountLoader.Factory ailf, ChangeMessagesUtil cmUtil, + StarredChangesUtil starredChangesUtil, Provider<ConsistencyChecker> checkerProvider, ActionJson actionJson, ChangeNotes.Factory notesFactory, @@ -259,11 +269,14 @@ public class ChangeJson { @GerritServerConfig Config cfg, @Assisted Iterable<ListChangesOption> options, @Assisted Optional<PluginDefinedInfosFactory> pluginDefinedInfosFactory) { + this.repoManager = repoManager; + this.allUsers = allUsers; this.userProvider = user; this.changeDataFactory = cdf; this.permissionBackend = permissionBackend; this.accountLoaderFactory = ailf; this.cmUtil = cmUtil; + this.starredChangesUtil = starredChangesUtil; this.checkerProvider = checkerProvider; this.actionJson = actionJson; this.notesFactory = notesFactory; @@ -474,6 +487,9 @@ public class ChangeJson { if (has(REVIEWED) && userProvider.get().isIdentifiedUser()) { ChangeData.ensureReviewedByLoadedForOpenChanges(all); } + if (has(STAR) && userProvider.get().isIdentifiedUser()) { + ChangeData.ensureChangeServerId(all); + } ChangeData.ensureCurrentApprovalsLoaded(all); } else { for (ChangeData cd : all) { @@ -526,6 +542,9 @@ public class ChangeJson { "Omitting corrupt change %s from results", cd.getId()); } } + if (has(STAR) && userProvider.get().isIdentifiedUser()) { + populateStarField(changeInfos); + } return changeInfos; } } @@ -755,6 +774,8 @@ public class ChangeJson { .collect(toList()); } + out._virtualIdNumber = cd.virtualId().get(); + return out; } @@ -940,6 +961,26 @@ public class ChangeJson { return map.build(); } + /** Populate the 'starred' field. */ + private void populateStarField(List<ChangeInfo> changeInfos) { + // We populate the 'starred' field for all change infos together so that we open the All-Users + // repository only once + try (Repository allUsersRepo = repoManager.openRepository(allUsers)) { + List<Change.Id> changeIds = + changeInfos.stream().map(c -> Change.id(c._virtualIdNumber)).collect(Collectors.toList()); + Set<Change.Id> starredChanges = + starredChangesUtil.areStarred( + allUsersRepo, changeIds, userProvider.get().asIdentifiedUser().getAccountId()); + if (starredChanges.isEmpty()) { + return; + } + changeInfos.stream() + .forEach(c -> c.starred = starredChanges.contains(Change.id(c._virtualIdNumber))); + } catch (IOException e) { + logger.atWarning().withCause(e).log("Failed to open All-Users repo."); + } + } + private List<PluginDefinedInfo> getPluginInfos(ChangeData cd) { return getPluginInfos(Collections.singleton(cd)).get(cd.getId()); } diff --git a/java/com/google/gerrit/server/change/ChangeResource.java b/java/com/google/gerrit/server/change/ChangeResource.java index c5c0be0814..ee48c7f45a 100644 --- a/java/com/google/gerrit/server/change/ChangeResource.java +++ b/java/com/google/gerrit/server/change/ChangeResource.java @@ -161,6 +161,10 @@ public class ChangeResource implements RestResource, HasETag { return changeData; } + public Change.Id getVirtualId() { + return getChangeData().virtualId(); + } + // This includes all information relevant for ETag computation // unrelated to the UI. public void prepareETag(Hasher h, CurrentUser user) { @@ -237,7 +241,8 @@ public class ChangeResource implements RestResource, HasETag { .build())) { Hasher h = Hashing.murmur3_128().newHasher(); if (user.isIdentifiedUser()) { - h.putString(starredChangesUtil.getObjectId(user.getAccountId(), getId()).name(), UTF_8); + h.putString( + starredChangesUtil.getObjectId(user.getAccountId(), getVirtualId()).name(), UTF_8); } prepareETag(h, user); return h.hash().toString(); diff --git a/java/com/google/gerrit/server/change/DeleteChangeOp.java b/java/com/google/gerrit/server/change/DeleteChangeOp.java index c7ddf199e4..4ac27c1376 100644 --- a/java/com/google/gerrit/server/change/DeleteChangeOp.java +++ b/java/com/google/gerrit/server/change/DeleteChangeOp.java @@ -80,7 +80,8 @@ public class DeleteChangeOp implements BatchUpdateOp { ensureDeletable(ctx, id, patchSets); // Cleaning up is only possible as long as the change and its elements are // still part of the database. - cleanUpReferences(id); + ChangeData cd = changeDataFactory.create(ctx.getChange()); + cleanUpReferences(cd); logger.atFine().log( "Deleting change %s, current patch set %d is commit %s", @@ -94,7 +95,7 @@ public class DeleteChangeOp implements BatchUpdateOp { .map(p -> p.commitId().name()) .orElse("n/a"))); ctx.deleteChange(); - changeDeleted.fire(changeDataFactory.create(ctx.getChange()), ctx.getAccount(), ctx.getWhen()); + changeDeleted.fire(cd, ctx.getAccount(), ctx.getWhen()); return true; } @@ -123,11 +124,11 @@ public class DeleteChangeOp implements BatchUpdateOp { revWalk.parseCommit(patchSet.commitId()), revWalk.parseCommit(destId.get())); } - private void cleanUpReferences(Change.Id id) throws IOException { - accountPatchReviewStore.run(s -> s.clearReviewed(id)); + private void cleanUpReferences(ChangeData cd) throws IOException { + accountPatchReviewStore.run(s -> s.clearReviewed(cd.virtualId())); // Non-atomic operation on All-Users refs; not much we can do to make it atomic. - starredChangesUtil.unstarAllForChangeDeletion(id); + starredChangesUtil.unstarAllForChangeDeletion(cd.virtualId()); } @Override diff --git a/java/com/google/gerrit/server/change/DeleteReviewerOp.java b/java/com/google/gerrit/server/change/DeleteReviewerOp.java index fc07592a9f..785f9e1679 100644 --- a/java/com/google/gerrit/server/change/DeleteReviewerOp.java +++ b/java/com/google/gerrit/server/change/DeleteReviewerOp.java @@ -212,7 +212,7 @@ public class DeleteReviewerOp extends ReviewerOp { reviewerDeleted.fire( ctx.getChangeData(currChange), patchSet, - accountCache.get(reviewer.id()).orElse(AccountState.forAccount(reviewer)), + accountCache.get(reviewer.id()).orElseGet(() -> AccountState.forAccount(reviewer)), ctx.getAccount(), mailMessage, newApprovals, diff --git a/java/com/google/gerrit/server/config/GerritGlobalModule.java b/java/com/google/gerrit/server/config/GerritGlobalModule.java index eee1c835bd..4325ec42c5 100644 --- a/java/com/google/gerrit/server/config/GerritGlobalModule.java +++ b/java/com/google/gerrit/server/config/GerritGlobalModule.java @@ -293,6 +293,7 @@ public class GerritGlobalModule extends FactoryModule { install(ThreadLocalRequestContext.module()); install(new ApprovalModule()); install(new MailSoySauceModule()); + install(new SkipCurrentRulesEvaluationOnClosedChangesModule()); factory(CapabilityCollection.Factory.class); factory(ChangeData.AssistedFactory.class); diff --git a/java/com/google/gerrit/server/config/GerritInstanceIdProvider.java b/java/com/google/gerrit/server/config/GerritInstanceIdProvider.java index 891ca7668f..6523f181a3 100644 --- a/java/com/google/gerrit/server/config/GerritInstanceIdProvider.java +++ b/java/com/google/gerrit/server/config/GerritInstanceIdProvider.java @@ -22,11 +22,14 @@ import org.eclipse.jgit.lib.Config; /** Provides {@link GerritInstanceId} from {@code gerrit.instanceId}. */ @Singleton public class GerritInstanceIdProvider implements Provider<String> { + public static final String INSTANCE_ID_SYSTEM_PROPERTY = "gerrit.instanceId"; private final String instanceId; @Inject public GerritInstanceIdProvider(@GerritServerConfig Config cfg) { - instanceId = cfg.getString("gerrit", null, "instanceId"); + instanceId = + System.getProperty( + INSTANCE_ID_SYSTEM_PROPERTY, cfg.getString("gerrit", null, "instanceId")); } @Override diff --git a/java/com/google/gerrit/server/config/SkipCurrentRulesEvaluationOnClosedChanges.java b/java/com/google/gerrit/server/config/SkipCurrentRulesEvaluationOnClosedChanges.java new file mode 100644 index 0000000000..b812710ca1 --- /dev/null +++ b/java/com/google/gerrit/server/config/SkipCurrentRulesEvaluationOnClosedChanges.java @@ -0,0 +1,25 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.config; + +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +import com.google.inject.BindingAnnotation; +import java.lang.annotation.Retention; + +/** Marker on a {@link Boolean} holding the evaluation of current Prolog rules on closed changes. */ +@Retention(RUNTIME) +@BindingAnnotation +public @interface SkipCurrentRulesEvaluationOnClosedChanges {} diff --git a/java/com/google/gerrit/server/config/SkipCurrentRulesEvaluationOnClosedChangesModule.java b/java/com/google/gerrit/server/config/SkipCurrentRulesEvaluationOnClosedChangesModule.java new file mode 100644 index 0000000000..9eaae021e0 --- /dev/null +++ b/java/com/google/gerrit/server/config/SkipCurrentRulesEvaluationOnClosedChangesModule.java @@ -0,0 +1,27 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.config; + +import com.google.inject.AbstractModule; + +public class SkipCurrentRulesEvaluationOnClosedChangesModule extends AbstractModule { + + @Override + protected void configure() { + bind(Boolean.class) + .annotatedWith(SkipCurrentRulesEvaluationOnClosedChanges.class) + .toProvider(SkipCurrentRulesEvaluationOnClosedChangesProvider.class); + } +} diff --git a/java/com/google/gerrit/server/config/SkipCurrentRulesEvaluationOnClosedChangesProvider.java b/java/com/google/gerrit/server/config/SkipCurrentRulesEvaluationOnClosedChangesProvider.java new file mode 100644 index 0000000000..bc25a3353f --- /dev/null +++ b/java/com/google/gerrit/server/config/SkipCurrentRulesEvaluationOnClosedChangesProvider.java @@ -0,0 +1,35 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.config; + +import com.google.inject.Inject; +import com.google.inject.Provider; +import com.google.inject.Singleton; +import org.eclipse.jgit.lib.Config; + +@Singleton +public class SkipCurrentRulesEvaluationOnClosedChangesProvider implements Provider<Boolean> { + private final Boolean value; + + @Inject + SkipCurrentRulesEvaluationOnClosedChangesProvider(@GerritServerConfig Config config) { + value = config.getBoolean("change", null, "skipCurrentRulesEvaluationOnClosedChanges", false); + } + + @Override + public Boolean get() { + return value; + } +} diff --git a/java/com/google/gerrit/server/edit/ChangeEditUtil.java b/java/com/google/gerrit/server/edit/ChangeEditUtil.java index 4d3f2a56d9..e7de3227fb 100644 --- a/java/com/google/gerrit/server/edit/ChangeEditUtil.java +++ b/java/com/google/gerrit/server/edit/ChangeEditUtil.java @@ -72,7 +72,6 @@ public class ChangeEditUtil { private final Provider<CurrentUser> userProvider; private final ChangeKindCache changeKindCache; private final PatchSetUtil psUtil; - private final GitReferenceUpdated gitReferenceUpdated; @Inject diff --git a/java/com/google/gerrit/server/events/EventTypes.java b/java/com/google/gerrit/server/events/EventTypes.java index e24bbd2de2..c2c057ce64 100644 --- a/java/com/google/gerrit/server/events/EventTypes.java +++ b/java/com/google/gerrit/server/events/EventTypes.java @@ -33,6 +33,7 @@ public class EventTypes { register(PatchSetCreatedEvent.TYPE, PatchSetCreatedEvent.class); register(PrivateStateChangedEvent.TYPE, PrivateStateChangedEvent.class); register(ProjectCreatedEvent.TYPE, ProjectCreatedEvent.class); + register(ProjectHeadUpdatedEvent.TYPE, ProjectHeadUpdatedEvent.class); register(RefReceivedEvent.TYPE, RefReceivedEvent.class); register(RefUpdatedEvent.TYPE, RefUpdatedEvent.class); register(ReviewerAddedEvent.TYPE, ReviewerAddedEvent.class); diff --git a/java/com/google/gerrit/server/git/ChangesByProjectCache.java b/java/com/google/gerrit/server/git/ChangesByProjectCache.java new file mode 100644 index 0000000000..df91891544 --- /dev/null +++ b/java/com/google/gerrit/server/git/ChangesByProjectCache.java @@ -0,0 +1,63 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.git; + +import com.google.gerrit.entities.Project; +import com.google.gerrit.server.config.GerritServerConfig; +import com.google.gerrit.server.query.change.ChangeData; +import com.google.inject.AbstractModule; +import java.io.IOException; +import java.util.stream.Stream; +import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.lib.Repository; + +public interface ChangesByProjectCache { + public enum UseIndex { + TRUE, + FALSE; + } + + public static class Module extends AbstractModule { + private UseIndex useIndex; + private @GerritServerConfig Config config; + + public Module(UseIndex useIndex, @GerritServerConfig Config config) { + this.useIndex = useIndex; + this.config = config; + } + + @Override + protected void configure() { + boolean searchingCacheEnabled = + config.getLong("cache", SearchingChangeCacheImpl.ID_CACHE, "memoryLimit", 0) > 0; + if (searchingCacheEnabled && UseIndex.TRUE.equals(useIndex)) { + install(new SearchingChangeCacheImpl.SearchingChangeCacheImplModule()); + } else { + bind(UseIndex.class).toInstance(useIndex); + install(new ChangesByProjectCacheImpl.Module()); + } + } + } + + /** + * Stream changeDatas for the project + * + * @param project project to read. + * @param repository repository for the project to read. + * @return Stream of known changes; empty if no changes. + */ + Stream<ChangeData> streamChangeDatas(Project.NameKey project, Repository repository) + throws IOException; +} diff --git a/java/com/google/gerrit/server/git/ChangesByProjectCacheImpl.java b/java/com/google/gerrit/server/git/ChangesByProjectCacheImpl.java new file mode 100644 index 0000000000..094287b7be --- /dev/null +++ b/java/com/google/gerrit/server/git/ChangesByProjectCacheImpl.java @@ -0,0 +1,360 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.git; + +import com.google.auto.value.AutoValue; +import com.google.common.cache.Cache; +import com.google.common.cache.Weigher; +import com.google.common.flogger.FluentLogger; +import com.google.gerrit.common.Nullable; +import com.google.gerrit.entities.BranchNameKey; +import com.google.gerrit.entities.Change; +import com.google.gerrit.entities.Project; +import com.google.gerrit.server.ReviewerSet; +import com.google.gerrit.server.cache.CacheModule; +import com.google.gerrit.server.git.ChangesByProjectCache.UseIndex; +import com.google.gerrit.server.index.change.ChangeField; +import com.google.gerrit.server.logging.Metadata; +import com.google.gerrit.server.logging.TraceContext; +import com.google.gerrit.server.logging.TraceContext.TraceTimer; +import com.google.gerrit.server.notedb.ChangeNotes; +import com.google.gerrit.server.query.change.ChangeData; +import com.google.gerrit.server.query.change.InternalChangeQuery; +import com.google.inject.Inject; +import com.google.inject.Provider; +import com.google.inject.Singleton; +import com.google.inject.name.Named; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.stream.Stream; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Repository; + +/** + * Lightweight cache of changes in each project. + * + * <p>This cache is intended to be used when filtering references and stores only the minimal fields + * required for a read permission check. + */ +@Singleton +public class ChangesByProjectCacheImpl implements ChangesByProjectCache { + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + private static final String CACHE_NAME = "changes_by_project"; + + public static class Module extends CacheModule { + @Override + protected void configure() { + cache(CACHE_NAME, Project.NameKey.class, CachedProjectChanges.class) + .weigher(ChangesByProjetCacheWeigher.class); + bind(ChangesByProjectCache.class).to(ChangesByProjectCacheImpl.class); + } + } + + private final Cache<Project.NameKey, CachedProjectChanges> cache; + private final ChangeData.Factory cdFactory; + private final UseIndex useIndex; + private final Provider<InternalChangeQuery> queryProvider; + + @Inject + ChangesByProjectCacheImpl( + @Named(CACHE_NAME) Cache<Project.NameKey, CachedProjectChanges> cache, + ChangeData.Factory cdFactory, + UseIndex useIndex, + Provider<InternalChangeQuery> queryProvider) { + this.cache = cache; + this.cdFactory = cdFactory; + this.useIndex = useIndex; + this.queryProvider = queryProvider; + } + + /** {@inheritDoc} */ + @Override + public Stream<ChangeData> streamChangeDatas(Project.NameKey project, Repository repo) + throws IOException { + CachedProjectChanges projectChanges = cache.getIfPresent(project); + if (projectChanges != null) { + return projectChanges + .getUpdatedChangeDatas( + project, repo, cdFactory, ChangeNotes.Factory.scanChangeIds(repo), "Updating") + .stream(); + } + if (UseIndex.TRUE.equals(useIndex)) { + return queryChangeDatasAndLoad(project).stream(); + } + return scanChangeDatasAndLoad(project, repo).stream(); + } + + private Collection<ChangeData> scanChangeDatasAndLoad(Project.NameKey project, Repository repo) + throws IOException { + CachedProjectChanges ours = new CachedProjectChanges(); + CachedProjectChanges projectChanges = ours; + try { + projectChanges = cache.get(project, () -> ours); + } catch (ExecutionException e) { + logger.atWarning().withCause(e).log("Cannot load %s for %s", CACHE_NAME, project.get()); + } + return projectChanges.getUpdatedChangeDatas( + project, + repo, + cdFactory, + ChangeNotes.Factory.scanChangeIds(repo), + ours == projectChanges ? "Scanning" : "Updating"); + } + + private Collection<ChangeData> queryChangeDatasAndLoad(Project.NameKey project) { + Collection<ChangeData> cds = queryChangeDatas(project); + cache.put(project, new CachedProjectChanges(cds)); + return cds; + } + + private Collection<ChangeData> queryChangeDatas(Project.NameKey project) { + try (TraceTimer timer = + TraceContext.newTimer( + "Querying changes of project", Metadata.builder().projectName(project.get()).build())) { + return queryProvider + .get() + .setRequestedFields( + ChangeField.CHANGE_SPEC, ChangeField.REVIEWER_SPEC, ChangeField.REF_STATE_SPEC) + .byProject(project); + } + } + + private static class CachedProjectChanges { + Map<String, Map<Change.Id, ObjectId>> metaObjectIdByNonPrivateChangeByBranch = + new ConcurrentHashMap<>(); // BranchNameKey "normalized" to a String to dedup project + Map<Change.Id, PrivateChange> privateChangeById = new ConcurrentHashMap<>(); + + public CachedProjectChanges() {} + + public CachedProjectChanges(Collection<ChangeData> cds) { + cds.stream().forEach(cd -> insert(cd)); + } + + public Collection<ChangeData> getUpdatedChangeDatas( + Project.NameKey project, + Repository repo, + ChangeData.Factory cdFactory, + Map<Change.Id, ObjectId> metaObjectIdByChange, + String operation) { + try (TraceTimer timer = + TraceContext.newTimer( + operation + " changes of project", + Metadata.builder().projectName(project.get()).build())) { + Map<Change.Id, ChangeData> cachedCdByChange = getChangeDataByChange(project, cdFactory); + List<ChangeData> cds = new ArrayList<>(); + for (Map.Entry<Change.Id, ObjectId> e : metaObjectIdByChange.entrySet()) { + Change.Id id = e.getKey(); + ChangeData cached = cachedCdByChange.get(id); + ChangeData cd = cached; + try { + if (cd == null || !cached.metaRevisionOrThrow().equals(e.getValue())) { + cd = cdFactory.create(project, id); + update(cached, cd); + } + } catch (Exception ex) { + // Do not let a bad change prevent other changes from being available. + logger.atFinest().withCause(ex).log("Can't load changeData for %s", id); + } + cds.add(cd); + } + return cds; + } + } + + public CachedProjectChanges update(ChangeData old, ChangeData updated) { + if (old != null) { + if (old.isPrivateOrThrow()) { + privateChangeById.remove(old.getId()); + } else { + Map<Change.Id, ObjectId> metaObjectIdByNonPrivateChange = + metaObjectIdByNonPrivateChangeByBranch.get(old.branchOrThrow().branch()); + if (metaObjectIdByNonPrivateChange != null) { + metaObjectIdByNonPrivateChange.remove(old.getId()); + } + } + } + return insert(updated); + } + + public CachedProjectChanges insert(ChangeData cd) { + if (cd.isPrivateOrThrow()) { + privateChangeById.put( + cd.getId(), + new AutoValue_ChangesByProjectCacheImpl_PrivateChange( + cd.change(), cd.reviewers(), cd.metaRevisionOrThrow())); + } else { + metaObjectIdByNonPrivateChangeByBranch + .computeIfAbsent(cd.branchOrThrow().branch(), b -> new ConcurrentHashMap<>()) + .put(cd.getId(), cd.metaRevisionOrThrow()); + } + return this; + } + + public Map<Change.Id, ChangeData> getChangeDataByChange( + Project.NameKey project, ChangeData.Factory cdFactory) { + Map<Change.Id, ChangeData> cdByChange = new HashMap<>(privateChangeById.size()); + for (PrivateChange pc : privateChangeById.values()) { + try { + ChangeData cd = cdFactory.create(pc.change()); + cd.setReviewers(pc.reviewers()); + cd.setMetaRevision(pc.metaRevision()); + cdByChange.put(cd.getId(), cd); + } catch (Exception ex) { + // Do not let a bad change prevent other changes from being available. + logger.atFinest().withCause(ex).log("Can't load changeData for %s", pc.change().getId()); + } + } + + for (Map.Entry<String, Map<Change.Id, ObjectId>> e : + metaObjectIdByNonPrivateChangeByBranch.entrySet()) { + BranchNameKey branch = BranchNameKey.create(project, e.getKey()); + for (Map.Entry<Change.Id, ObjectId> e2 : e.getValue().entrySet()) { + Change.Id id = e2.getKey(); + try { + cdByChange.put(id, cdFactory.createNonPrivate(branch, id, e2.getValue())); + } catch (Exception ex) { + // Do not let a bad change prevent other changes from being available. + logger.atFinest().withCause(ex).log("Can't load changeData for %s", id); + } + } + } + return cdByChange; + } + + public int weigh() { + int size = 0; + size += 24 * 2; // guess at basic ConcurrentHashMap overhead * 2 + for (Map.Entry<String, Map<Change.Id, ObjectId>> e : + metaObjectIdByNonPrivateChangeByBranch.entrySet()) { + size += JavaWeights.REFERENCE + e.getKey().length(); + size += + e.getValue().size() + * (JavaWeights.REFERENCE + + JavaWeights.OBJECT // Map.Entry + + JavaWeights.REFERENCE + + GerritWeights.CHANGE_NUM + + JavaWeights.REFERENCE + + GerritWeights.OBJECTID); + } + for (Map.Entry<Change.Id, PrivateChange> e : privateChangeById.entrySet()) { + size += JavaWeights.REFERENCE + GerritWeights.CHANGE_NUM; + size += JavaWeights.REFERENCE + e.getValue().weigh(); + } + return size; + } + } + + @AutoValue + abstract static class PrivateChange { + // Fields needed to serve permission checks on private Changes + abstract Change change(); + + @Nullable + abstract ReviewerSet reviewers(); + + abstract ObjectId metaRevision(); // Needed to confirm whether up-to-date + + public int weigh() { + int size = 0; + size += JavaWeights.OBJECT; // this + size += JavaWeights.REFERENCE + weigh(change()); + size += JavaWeights.REFERENCE + weigh(reviewers()); + size += JavaWeights.REFERENCE + GerritWeights.OBJECTID; // metaRevision + return size; + } + + private static int weigh(Change c) { + int size = 0; + size += JavaWeights.OBJECT; // change + size += JavaWeights.REFERENCE + GerritWeights.KEY_INT; // changeId + size += JavaWeights.REFERENCE + JavaWeights.OBJECT + 40; // changeKey; + size += JavaWeights.REFERENCE + GerritWeights.TIMESTAMP; // createdOn; + size += JavaWeights.REFERENCE + GerritWeights.TIMESTAMP; // lastUpdatedOn; + size += JavaWeights.REFERENCE + GerritWeights.ACCOUNT_ID; // owner; + size += + JavaWeights.REFERENCE + + c.getDest().project().get().length() + + c.getDest().branch().length(); + size += JavaWeights.CHAR; // status; + size += JavaWeights.INT; // currentPatchSetId; + size += JavaWeights.REFERENCE + c.getSubject().length(); + size += JavaWeights.REFERENCE + (c.getTopic() == null ? 0 : c.getTopic().length()); + size += + JavaWeights.REFERENCE + + (c.getOriginalSubject().equals(c.getSubject()) ? 0 : c.getSubject().length()); + size += + JavaWeights.REFERENCE + (c.getSubmissionId() == null ? 0 : c.getSubmissionId().length()); + size += JavaWeights.REFERENCE + GerritWeights.ACCOUNT_ID; // assignee; + size += JavaWeights.REFERENCE + JavaWeights.BOOLEAN; // isPrivate; + size += JavaWeights.REFERENCE + JavaWeights.BOOLEAN; // workInProgress; + size += JavaWeights.REFERENCE + JavaWeights.BOOLEAN; // reviewStarted; + size += JavaWeights.REFERENCE + GerritWeights.CHANGE_NUM; // revertOf; + size += JavaWeights.REFERENCE + GerritWeights.PACTCH_SET_ID; // cherryPickOf; + return size; + } + + private static int weigh(ReviewerSet rs) { + int size = 0; + size += JavaWeights.OBJECT; // ReviewerSet + size += JavaWeights.REFERENCE; // table + size += + rs.asTable().cellSet().size() + * (JavaWeights.OBJECT // cell (at least one object) + + JavaWeights.REFERENCE // ReviewerStateInternal + + (JavaWeights.REFERENCE + GerritWeights.ACCOUNT_ID) + + (JavaWeights.REFERENCE + GerritWeights.TIMESTAMP)); + size += JavaWeights.REFERENCE; // accounts + return size; + } + } + + private static class ChangesByProjetCacheWeigher + implements Weigher<Project.NameKey, CachedProjectChanges> { + @Override + public int weigh(Project.NameKey project, CachedProjectChanges changes) { + int size = 0; + size += project.get().length(); + size += changes.weigh(); + return size; + } + } + + private static class GerritWeights { + public static final int KEY_INT = JavaWeights.OBJECT + JavaWeights.INT; // IntKey + public static final int CHANGE_NUM = KEY_INT; + public static final int ACCOUNT_ID = KEY_INT; + public static final int PACTCH_SET_ID = + JavaWeights.OBJECT + + (JavaWeights.REFERENCE + GerritWeights.CHANGE_NUM) // PatchSet.Id.changeId + + JavaWeights.INT; // PatchSet.Id patch_num; + public static final int TIMESTAMP = JavaWeights.OBJECT + 8; // Timestamp + public static final int OBJECTID = JavaWeights.OBJECT + (5 * JavaWeights.INT); // (w1-w5) + } + + private static class JavaWeights { + public static final int BOOLEAN = 1; + public static final int CHAR = 1; + public static final int INT = 4; + public static final int OBJECT = 16; + public static final int REFERENCE = 8; + } +} diff --git a/java/com/google/gerrit/server/git/DefaultChangeReportFormatter.java b/java/com/google/gerrit/server/git/DefaultChangeReportFormatter.java index 4f6094e685..58c3eb10b9 100644 --- a/java/com/google/gerrit/server/git/DefaultChangeReportFormatter.java +++ b/java/com/google/gerrit/server/git/DefaultChangeReportFormatter.java @@ -54,7 +54,7 @@ public class DefaultChangeReportFormatter implements ChangeReportFormatter { urlFormatter .get() .getChangeViewUrl(c.getProject(), c.getId()) - .orElse(c.getId().toString())); + .orElseGet(() -> c.getId().toString())); } protected String cropSubject(String subject) { diff --git a/java/com/google/gerrit/server/git/MergeUtil.java b/java/com/google/gerrit/server/git/MergeUtil.java index 6922efbfba..7e284de2d0 100644 --- a/java/com/google/gerrit/server/git/MergeUtil.java +++ b/java/com/google/gerrit/server/git/MergeUtil.java @@ -673,6 +673,10 @@ public class MergeUtil { return false; } + return canMerge(mergeTip, repo, toMerge); + } + + private boolean canMerge(CodeReviewCommit mergeTip, Repository repo, CodeReviewCommit toMerge) { try (ObjectInserter ins = new InMemoryInserter(repo)) { return newThreeWayMerger(ins, repo.getConfig()).merge(mergeTip, toMerge); } catch (LargeObjectException e) { @@ -694,6 +698,11 @@ public class MergeUtil { return false; } + return canFastForward(mergeTip, rw, toMerge); + } + + private boolean canFastForward( + CodeReviewCommit mergeTip, CodeReviewRevWalk rw, CodeReviewCommit toMerge) { try { return mergeTip == null || rw.isMergedInto(mergeTip, toMerge) @@ -703,6 +712,19 @@ public class MergeUtil { } } + public boolean canFastForwardOrMerge( + MergeSorter mergeSorter, + CodeReviewCommit mergeTip, + CodeReviewRevWalk rw, + Repository repo, + CodeReviewCommit toMerge) { + if (hasMissingDependencies(mergeSorter, toMerge)) { + return false; + } + + return canFastForward(mergeTip, rw, toMerge) || canMerge(mergeTip, repo, toMerge); + } + public boolean canCherryPick( MergeSorter mergeSorter, Repository repo, @@ -745,8 +767,7 @@ public class MergeUtil { // by an equivalent merge with a different first parent. So // instead behave as though MERGE_IF_NECESSARY was configured. // - return canFastForward(mergeSorter, mergeTip, rw, toMerge) - || canMerge(mergeSorter, repo, mergeTip, toMerge); + return canFastForwardOrMerge(mergeSorter, mergeTip, rw, repo, toMerge); } public boolean hasMissingDependencies(MergeSorter mergeSorter, CodeReviewCommit toMerge) { diff --git a/java/com/google/gerrit/server/git/SearchingChangeCacheImpl.java b/java/com/google/gerrit/server/git/SearchingChangeCacheImpl.java index cfeec70fdd..83024e3f45 100644 --- a/java/com/google/gerrit/server/git/SearchingChangeCacheImpl.java +++ b/java/com/google/gerrit/server/git/SearchingChangeCacheImpl.java @@ -40,12 +40,12 @@ import com.google.inject.Provider; import com.google.inject.Singleton; import com.google.inject.TypeLiteral; import com.google.inject.name.Named; -import com.google.inject.util.Providers; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.stream.Stream; +import org.eclipse.jgit.lib.Repository; /** * Cache based on an index query of the most recent changes. The number of cached items depends on @@ -55,35 +55,22 @@ import java.util.stream.Stream; * fraction of all changes. These are the changes that were modified last. */ @Singleton -public class SearchingChangeCacheImpl implements GitReferenceUpdatedListener { +public class SearchingChangeCacheImpl + implements ChangesByProjectCache, GitReferenceUpdatedListener { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); static final String ID_CACHE = "changes"; public static class SearchingChangeCacheImplModule extends CacheModule { - private final boolean slave; - - public SearchingChangeCacheImplModule() { - this(false); - } - - public SearchingChangeCacheImplModule(boolean slave) { - this.slave = slave; - } - @Override protected void configure() { - if (slave) { - bind(SearchingChangeCacheImpl.class).toProvider(Providers.of(null)); - } else { - cache(ID_CACHE, Project.NameKey.class, new TypeLiteral<List<CachedChange>>() {}) - .maximumWeight(0) - .loader(Loader.class); + cache(ID_CACHE, Project.NameKey.class, new TypeLiteral<List<CachedChange>>() {}) + .maximumWeight(0) + .loader(Loader.class); - bind(SearchingChangeCacheImpl.class); - DynamicSet.bind(binder(), GitReferenceUpdatedListener.class) - .to(SearchingChangeCacheImpl.class); - } + bind(ChangesByProjectCache.class).to(SearchingChangeCacheImpl.class); + DynamicSet.bind(binder(), GitReferenceUpdatedListener.class) + .to(SearchingChangeCacheImpl.class); } } @@ -117,9 +104,11 @@ public class SearchingChangeCacheImpl implements GitReferenceUpdatedListener { * Additional stored fields are not loaded from the index. * * @param project project to read. + * @param unusedrepo repository for the project to read. * @return stream of known changes; empty if no changes. */ - public Stream<ChangeData> getChangeData(Project.NameKey project) { + @Override + public Stream<ChangeData> streamChangeDatas(Project.NameKey project, Repository unusedrepo) { List<CachedChange> cached; try { cached = cache.get(project); diff --git a/java/com/google/gerrit/server/git/UploadPackMetricsHook.java b/java/com/google/gerrit/server/git/UploadPackMetricsHook.java index 1619addd75..0e981f2307 100644 --- a/java/com/google/gerrit/server/git/UploadPackMetricsHook.java +++ b/java/com/google/gerrit/server/git/UploadPackMetricsHook.java @@ -38,7 +38,12 @@ public class UploadPackMetricsHook implements PostUploadHook { private final Counter1<Operation> requestCount; private final Timer1<Operation> counting; + private final Histogram1<Operation> bitmapIndexMissesCount; + private final Counter1<Operation> noBitmapIndex; private final Timer1<Operation> compressing; + private final Timer1<Operation> negotiating; + private final Timer1<Operation> searchingForReuse; + private final Timer1<Operation> searchingForSizes; private final Timer1<Operation> writing; private final Histogram1<Operation> packBytes; @@ -64,6 +69,22 @@ public class UploadPackMetricsHook implements PostUploadHook { .setUnit(Units.MILLISECONDS), operationField); + bitmapIndexMissesCount = + metricMaker.newHistogram( + "git/upload-pack/bitmap_index_misses_count", + new Description("Number of bitmap index misses per request") + .setCumulative() + .setUnit("misses"), + operationField); + + noBitmapIndex = + metricMaker.newCounter( + "git/upload-pack/no_bitmap_index", + new Description("Total number of requests executed without a bitmap index") + .setRate() + .setUnit("requests"), + operationField); + compressing = metricMaker.newTimer( "git/upload-pack/phase_compressing", @@ -72,6 +93,32 @@ public class UploadPackMetricsHook implements PostUploadHook { .setUnit(Units.MILLISECONDS), operationField); + negotiating = + metricMaker.newTimer( + "git/upload-pack/phase_negotiating", + new Description("Time spent in the negotiation phase") + .setCumulative() + .setUnit(Units.MILLISECONDS), + operationField); + + searchingForReuse = + metricMaker.newTimer( + "git/upload-pack/phase_searching_for_reuse", + new Description( + "Time spent in the 'Finding sources...' while searching for reuse phase") + .setCumulative() + .setUnit(Units.MILLISECONDS), + operationField); + + searchingForSizes = + metricMaker.newTimer( + "git/upload-pack/phase_searching_for_sizes", + new Description( + "Time spent in the 'Finding sources...' while searching for sizes phase") + .setCumulative() + .setUnit(Units.MILLISECONDS), + operationField); + writing = metricMaker.newTimer( "git/upload-pack/phase_writing", @@ -98,7 +145,16 @@ public class UploadPackMetricsHook implements PostUploadHook { requestCount.increment(op); counting.record(op, stats.getTimeCounting(), MILLISECONDS); + long bitmapIndexMisses = stats.getBitmapIndexMisses(); + if (bitmapIndexMisses < 0) { + noBitmapIndex.increment(op); + } else { + bitmapIndexMissesCount.record(op, bitmapIndexMisses); + } compressing.record(op, stats.getTimeCompressing(), MILLISECONDS); + negotiating.record(op, stats.getTimeNegotiating(), MILLISECONDS); + searchingForReuse.record(op, stats.getTimeSearchingForReuse(), MILLISECONDS); + searchingForSizes.record(op, stats.getTimeSearchingForSizes(), MILLISECONDS); writing.record(op, stats.getTimeWriting(), MILLISECONDS); packBytes.record(op, stats.getTotalBytes()); } diff --git a/java/com/google/gerrit/server/git/WorkQueue.java b/java/com/google/gerrit/server/git/WorkQueue.java index e8b7c62f15..86d6c7c7d2 100644 --- a/java/com/google/gerrit/server/git/WorkQueue.java +++ b/java/com/google/gerrit/server/git/WorkQueue.java @@ -14,6 +14,7 @@ package com.google.gerrit.server.git; +import static com.google.common.base.MoreObjects.firstNonNull; import static java.util.stream.Collectors.toList; import com.google.common.base.CaseFormat; @@ -286,6 +287,7 @@ public class WorkQueue { /** An isolated queue. */ private class Executor extends ScheduledThreadPoolExecutor { private final ConcurrentHashMap<Integer, Task<?>> all; + private final ConcurrentHashMap<Runnable, Long> nanosPeriodByRunnable; private final String queueName; Executor(int corePoolSize, final String queueName) { @@ -310,6 +312,7 @@ public class WorkQueue { 0.75f, // load factor corePoolSize + 4 // concurrency level ); + nanosPeriodByRunnable = new ConcurrentHashMap<>(1, 0.75f, 1); this.queueName = queueName; } @@ -373,12 +376,14 @@ public class WorkQueue { @Override public ScheduledFuture<?> scheduleAtFixedRate( Runnable command, long initialDelay, long period, TimeUnit unit) { + nanosPeriodByRunnable.put(command, unit.toNanos(period)); return super.scheduleAtFixedRate(LoggingContext.copy(command), initialDelay, period, unit); } @Override public ScheduledFuture<?> scheduleWithFixedDelay( Runnable command, long initialDelay, long delay, TimeUnit unit) { + nanosPeriodByRunnable.put(command, unit.toNanos(delay)); return super.scheduleWithFixedDelay(LoggingContext.copy(command), initialDelay, delay, unit); } @@ -440,6 +445,18 @@ public class WorkQueue { protected <V> RunnableScheduledFuture<V> decorateTask( Runnable runnable, RunnableScheduledFuture<V> r) { r = super.decorateTask(runnable, r); + + // Periodic Tasks may get rescheduled if the previous run has yet to fully complete (and thus + // passed to decorateTask() more than once), and there is no need to redecorate them if they + // are already decorated. + if (runnable instanceof LoggingContextAwareRunnable) { + Runnable unwrappedTask = ((LoggingContextAwareRunnable) runnable).unwrap(); + if (unwrappedTask instanceof Task<?>) { + return r; + } + } + + long nanosPeriod = firstNonNull(nanosPeriodByRunnable.remove(runnable), 0L); for (; ; ) { final int id = idGenerator.next(); @@ -450,9 +467,9 @@ public class WorkQueue { } if (runnable instanceof ProjectRunnable) { - task = new ProjectTask<>((ProjectRunnable) runnable, r, this, id); + task = new ProjectTask<>((ProjectRunnable) runnable, r, nanosPeriod, this, id); } else { - task = new Task<>(runnable, r, this, id); + task = new Task<>(runnable, r, nanosPeriod, this, id); } if (all.putIfAbsent(task.getTaskId(), task) == null) { @@ -553,13 +570,20 @@ public class WorkQueue { private final Executor executor; private final int taskId; private final Instant startTime; + private final long nanosPeriod; // runningState is non-null when listener or task code is running in an executor thread private final AtomicReference<State> runningState = new AtomicReference<>(); - Task(Runnable runnable, RunnableScheduledFuture<V> task, Executor executor, int taskId) { + Task( + Runnable runnable, + RunnableScheduledFuture<V> task, + long nanosPeriod, + Executor executor, + int taskId) { this.runnable = runnable; this.task = task; + this.nanosPeriod = nanosPeriod; this.executor = executor; this.taskId = taskId; this.startTime = Instant.now(); @@ -684,6 +708,8 @@ public class WorkQueue { executor.remove(this); } } + } else { + Future<?> unusedFuture = executor.schedule(this, nanosPeriod / 3, TimeUnit.NANOSECONDS); } } @@ -731,8 +757,12 @@ public class WorkQueue { private final ProjectRunnable runnable; ProjectTask( - ProjectRunnable runnable, RunnableScheduledFuture<V> task, Executor executor, int taskId) { - super(runnable, task, executor, taskId); + ProjectRunnable runnable, + RunnableScheduledFuture<V> task, + long nanosPeriod, + Executor executor, + int taskId) { + super(runnable, task, nanosPeriod, executor, taskId); this.runnable = runnable; } diff --git a/java/com/google/gerrit/server/git/receive/ReceiveCommits.java b/java/com/google/gerrit/server/git/receive/ReceiveCommits.java index 2baca53b1c..8e034211de 100644 --- a/java/com/google/gerrit/server/git/receive/ReceiveCommits.java +++ b/java/com/google/gerrit/server/git/receive/ReceiveCommits.java @@ -2033,10 +2033,10 @@ class ReceiveCommits { magicBranch.dest = BranchNameKey.create(project.getNameKey(), ref); magicBranch.perm = permissions.ref(ref); - Optional<AuthException> err = - checkRefPermission(magicBranch.perm, RefPermission.READ) - .map(Optional::of) - .orElse(checkRefPermission(magicBranch.perm, RefPermission.CREATE_CHANGE)); + Optional<AuthException> err = checkRefPermission(magicBranch.perm, RefPermission.READ); + if (err.isEmpty()) { + err = checkRefPermission(magicBranch.perm, RefPermission.CREATE_CHANGE); + } if (err.isPresent()) { rejectProhibited(cmd, err.get()); return; diff --git a/java/com/google/gerrit/server/git/validators/MergeValidators.java b/java/com/google/gerrit/server/git/validators/MergeValidators.java index 40ce671a36..811e960a34 100644 --- a/java/com/google/gerrit/server/git/validators/MergeValidators.java +++ b/java/com/google/gerrit/server/git/validators/MergeValidators.java @@ -36,6 +36,7 @@ import com.google.gerrit.server.config.PluginConfig; import com.google.gerrit.server.config.ProjectConfigEntry; import com.google.gerrit.server.git.CodeReviewCommit; import com.google.gerrit.server.git.CodeReviewCommit.CodeReviewRevWalk; +import com.google.gerrit.server.group.db.GroupConfig; import com.google.gerrit.server.permissions.GlobalPermission; import com.google.gerrit.server.permissions.PermissionBackend; import com.google.gerrit.server.permissions.PermissionBackendException; @@ -343,10 +344,12 @@ public class MergeValidators { } private final AllUsersName allUsersName; + private final ChangeData.Factory changeDataFactory; @Inject - public GroupMergeValidator(AllUsersName allUsersName) { + public GroupMergeValidator(AllUsersName allUsersName, ChangeData.Factory changeDataFactory) { this.allUsersName = allUsersName; + this.changeDataFactory = changeDataFactory; } @Override @@ -365,7 +368,30 @@ public class MergeValidators { return; } - throw new MergeValidationException("group update not allowed"); + // Update to group files is not supported because there are no validations + // on the changes being done to these files, without which the group data + // might get corrupted. Thus don't allow merges into All-Users group refs + // which updates group files (i.e., group.config, members and subgroups). + // But it is still useful to allow users to update files apart from group + // files. For example, users can maintain task config in group refs which + // allows users to collaborate and review changes on group specific task configs. + ChangeData cd = + changeDataFactory.create(destProject.getProject().getNameKey(), patchSetId.changeId()); + try { + if (cd.currentFilePaths().contains(GroupConfig.GROUP_CONFIG_FILE) + || cd.currentFilePaths().contains(GroupConfig.MEMBERS_FILE) + || cd.currentFilePaths().contains(GroupConfig.SUBGROUPS_FILE)) { + throw new MergeValidationException( + String.format( + "update to group files (%s, %s, %s) not allowed", + GroupConfig.GROUP_CONFIG_FILE, + GroupConfig.MEMBERS_FILE, + GroupConfig.SUBGROUPS_FILE)); + } + } catch (StorageException e) { + logger.atSevere().withCause(e).log("Cannot validate group update"); + throw new MergeValidationException("group validation unavailable", e); + } } } diff --git a/java/com/google/gerrit/server/group/db/AuditLogFormatter.java b/java/com/google/gerrit/server/group/db/AuditLogFormatter.java index 235ca4f613..3ba087e315 100644 --- a/java/com/google/gerrit/server/group/db/AuditLogFormatter.java +++ b/java/com/google/gerrit/server/group/db/AuditLogFormatter.java @@ -167,7 +167,7 @@ public class AuditLogFormatter { .map(Account::getName) // Historically, the database did not enforce relational integrity, so it is // possible for groups to have non-existing members. - .orElse("No Account for Id #" + accountId); + .orElseGet(() -> "No Account for Id #" + accountId); } private PersonIdent getParsableAuthorIdent( diff --git a/java/com/google/gerrit/server/group/db/GroupConfig.java b/java/com/google/gerrit/server/group/db/GroupConfig.java index 4f2c04972b..682fd15f27 100644 --- a/java/com/google/gerrit/server/group/db/GroupConfig.java +++ b/java/com/google/gerrit/server/group/db/GroupConfig.java @@ -19,7 +19,6 @@ import static com.google.common.collect.ImmutableSet.toImmutableSet; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.joining; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Splitter; import com.google.common.base.Strings; import com.google.common.collect.ImmutableSet; @@ -89,9 +88,9 @@ import org.eclipse.jgit.revwalk.RevSort; * doesn't have any members or subgroups. */ public class GroupConfig extends VersionedMetaData { - @VisibleForTesting public static final String GROUP_CONFIG_FILE = "group.config"; - @VisibleForTesting static final String MEMBERS_FILE = "members"; - @VisibleForTesting static final String SUBGROUPS_FILE = "subgroups"; + public static final String GROUP_CONFIG_FILE = "group.config"; + public static final String MEMBERS_FILE = "members"; + public static final String SUBGROUPS_FILE = "subgroups"; private static final Pattern LINE_SEPARATOR_PATTERN = Pattern.compile("\\R"); /** diff --git a/java/com/google/gerrit/server/index/IndexModule.java b/java/com/google/gerrit/server/index/IndexModule.java index 9ad7cdb12e..a6aeb6b7aa 100644 --- a/java/com/google/gerrit/server/index/IndexModule.java +++ b/java/com/google/gerrit/server/index/IndexModule.java @@ -53,6 +53,7 @@ import com.google.gerrit.server.index.group.GroupIndexRewriter; import com.google.gerrit.server.index.group.GroupIndexer; import com.google.gerrit.server.index.group.GroupIndexerImpl; import com.google.gerrit.server.index.group.GroupSchemaDefinitions; +import com.google.gerrit.server.index.options.BuildBloomFilter; import com.google.gerrit.server.index.options.IsFirstInsertForEntry; import com.google.gerrit.server.index.project.ProjectIndexDefinition; import com.google.gerrit.server.index.project.ProjectIndexerImpl; @@ -154,6 +155,9 @@ public class IndexModule extends LifecycleModule { OptionalBinder.newOptionalBinder(binder(), IsFirstInsertForEntry.class) .setDefault() .toInstance(IsFirstInsertForEntry.NO); + OptionalBinder.newOptionalBinder(binder(), BuildBloomFilter.class) + .setDefault() + .toInstance(BuildBloomFilter.TRUE); } @Provides diff --git a/java/com/google/gerrit/server/index/change/ChangeField.java b/java/com/google/gerrit/server/index/change/ChangeField.java index 7057ff7351..315f9bf110 100644 --- a/java/com/google/gerrit/server/index/change/ChangeField.java +++ b/java/com/google/gerrit/server/index/change/ChangeField.java @@ -65,7 +65,6 @@ import com.google.gerrit.server.ReviewerByEmailSet; import com.google.gerrit.server.ReviewerSet; import com.google.gerrit.server.StarredChangesUtil; import com.google.gerrit.server.cache.proto.Cache; -import com.google.gerrit.server.config.AllUsersName; import com.google.gerrit.server.index.change.StalenessChecker.RefStatePattern; import com.google.gerrit.server.notedb.ReviewerStateInternal; import com.google.gerrit.server.notedb.SubmitRequirementProtoConverter; @@ -128,7 +127,7 @@ public class ChangeField { .required() // The numeric change id is integer in string form .size(10) - .build(cd -> String.valueOf(cd.getVirtualId().get())); + .build(cd -> String.valueOf(cd.virtualId().get())); public static final IndexedField<ChangeData, String>.SearchSpec NUMERIC_ID_STR_SPEC = NUMERIC_ID_STR_FIELD.exact("legacy_id_str"); @@ -1697,12 +1696,6 @@ public class ChangeField { RefStatePattern.create( RefNames.REFS_USERS + "*/" + RefNames.EDIT_PREFIX + id + "/*") .toByteArray(project)); - result.add( - RefStatePattern.create(RefNames.refsStarredChangesPrefix(id) + "*") - .toByteArray(allUsers(cd))); - result.add( - RefStatePattern.create(RefNames.refsDraftCommentsPrefix(id) + "*") - .toByteArray(allUsers(cd))); return result; }, (cd, field) -> cd.setRefStatePatterns(field)); @@ -1747,10 +1740,6 @@ public class ChangeField { return in -> in.change() != null ? func.apply(in.change()) : null; } - private static AllUsersName allUsers(ChangeData cd) { - return cd.getAllUsersNameForIndexing(); - } - private static String truncateStringValueToMaxTermLength(String str) { return truncateStringValue(str, MAX_TERM_LENGTH); } diff --git a/java/com/google/gerrit/server/index/options/BuildBloomFilter.java b/java/com/google/gerrit/server/index/options/BuildBloomFilter.java new file mode 100644 index 0000000000..021f0fe42f --- /dev/null +++ b/java/com/google/gerrit/server/index/options/BuildBloomFilter.java @@ -0,0 +1,21 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.index.options; + +/** This enum can be used to decide if bloom filters for H2 disk caches should be built. */ +public enum BuildBloomFilter { + TRUE, + FALSE +} diff --git a/java/com/google/gerrit/server/mail/send/SmtpEmailSender.java b/java/com/google/gerrit/server/mail/send/SmtpEmailSender.java index c06cc1e8fb..14b303507e 100644 --- a/java/com/google/gerrit/server/mail/send/SmtpEmailSender.java +++ b/java/com/google/gerrit/server/mail/send/SmtpEmailSender.java @@ -159,7 +159,7 @@ public class SmtpEmailSender implements EmailSender { if (denyrcpt.contains(address) || denyrcpt.contains(domain) || denyrcpt.contains("@" + domain)) { - logger.atWarning().log("Not emailing %s (prohibited by sendemail.denyrcpt)", address); + logger.atInfo().log("Not emailing %s (prohibited by sendemail.denyrcpt)", address); return true; } diff --git a/java/com/google/gerrit/server/notedb/ChangeDraftUpdate.java b/java/com/google/gerrit/server/notedb/ChangeDraftUpdate.java index 0dcf786f51..c219387e00 100644 --- a/java/com/google/gerrit/server/notedb/ChangeDraftUpdate.java +++ b/java/com/google/gerrit/server/notedb/ChangeDraftUpdate.java @@ -29,6 +29,7 @@ import com.google.gerrit.entities.RefNames; import com.google.gerrit.exceptions.StorageException; import com.google.gerrit.server.GerritPersonIdent; import com.google.gerrit.server.config.AllUsersName; +import com.google.gerrit.server.query.change.ChangeNumberVirtualIdAlgorithm; import com.google.inject.assistedinject.Assisted; import com.google.inject.assistedinject.AssistedInject; import java.io.IOException; @@ -55,6 +56,8 @@ import org.eclipse.jgit.revwalk.RevWalk; * <p>This class is not thread safe. */ public class ChangeDraftUpdate extends AbstractChangeUpdate { + private final ChangeNumberVirtualIdAlgorithm virtualIdFunc; + public interface Factory { ChangeDraftUpdate create( ChangeNotes notes, @@ -99,6 +102,7 @@ public class ChangeDraftUpdate extends AbstractChangeUpdate { @GerritPersonIdent PersonIdent serverIdent, AllUsersName allUsers, ChangeNoteUtil noteUtil, + @Nullable ChangeNumberVirtualIdAlgorithm virtualIdFunc, @Assisted ChangeNotes notes, @Assisted("effective") Account.Id accountId, @Assisted("real") Account.Id realAccountId, @@ -106,6 +110,7 @@ public class ChangeDraftUpdate extends AbstractChangeUpdate { @Assisted Instant when) { super(noteUtil, serverIdent, notes, null, accountId, realAccountId, authorIdent, when); this.draftsProject = allUsers; + this.virtualIdFunc = virtualIdFunc; } @AssistedInject @@ -113,6 +118,7 @@ public class ChangeDraftUpdate extends AbstractChangeUpdate { @GerritPersonIdent PersonIdent serverIdent, AllUsersName allUsers, ChangeNoteUtil noteUtil, + @Nullable ChangeNumberVirtualIdAlgorithm virtualIdFunc, @Assisted Change change, @Assisted("effective") Account.Id accountId, @Assisted("real") Account.Id realAccountId, @@ -120,6 +126,7 @@ public class ChangeDraftUpdate extends AbstractChangeUpdate { @Assisted Instant when) { super(noteUtil, serverIdent, null, change, accountId, realAccountId, authorIdent, when); this.draftsProject = allUsers; + this.virtualIdFunc = virtualIdFunc; } public void putComment(HumanComment c) { @@ -179,6 +186,7 @@ public class ChangeDraftUpdate extends AbstractChangeUpdate { authorIdent, draftsProject, noteUtil, + virtualIdFunc, new Change(getChange()), accountId, realAccountId, @@ -285,7 +293,7 @@ public class ChangeDraftUpdate extends AbstractChangeUpdate { @Override protected String getRefName() { - return RefNames.refsDraftComments(getId(), accountId); + return RefNames.refsDraftComments(getVirtualId(), accountId); } @Override @@ -297,4 +305,11 @@ public class ChangeDraftUpdate extends AbstractChangeUpdate { public boolean isEmpty() { return delete.isEmpty() && put.isEmpty(); } + + private Change.Id getVirtualId() { + Change change = getChange(); + return virtualIdFunc == null + ? change.getId() + : virtualIdFunc.apply(change.getServerId(), change.getId()); + } } diff --git a/java/com/google/gerrit/server/notedb/ChangeNotes.java b/java/com/google/gerrit/server/notedb/ChangeNotes.java index 222be70ed0..da531e3ddb 100644 --- a/java/com/google/gerrit/server/notedb/ChangeNotes.java +++ b/java/com/google/gerrit/server/notedb/ChangeNotes.java @@ -166,6 +166,12 @@ public class ChangeNotes extends AbstractChangeNotes<ChangeNotes> { return new ChangeNotes(args, newChange(project, changeId), true, null).load(); } + public ChangeNotes create( + Project.NameKey project, Change.Id changeId, @Nullable ObjectId metaRevId) { + checkArgument(project != null, "project is required"); + return new ChangeNotes(args, newChange(project, changeId), true, null, metaRevId).load(); + } + public ChangeNotes create(Repository repository, Project.NameKey project, Change.Id changeId) { checkArgument(project != null, "project is required"); return new ChangeNotes(args, newChange(project, changeId), true, null).load(repository); @@ -533,12 +539,17 @@ public class ChangeNotes extends AbstractChangeNotes<ChangeNotes> { } public ImmutableListMultimap<ObjectId, HumanComment> getDraftComments(Account.Id author) { - return getDraftComments(author, null); + return getDraftComments(author, null, null); + } + + public ImmutableListMultimap<ObjectId, HumanComment> getDraftComments( + Account.Id author, @Nullable Change.Id virtualId) { + return getDraftComments(author, virtualId, null); } public ImmutableListMultimap<ObjectId, HumanComment> getDraftComments( - Account.Id author, @Nullable Ref ref) { - loadDraftComments(author, ref); + Account.Id author, @Nullable Change.Id virtualId, @Nullable Ref ref) { + loadDraftComments(author, virtualId, ref); // Filter out any zombie draft comments. These are drafts that are also in // the published map, and arise when the update to All-Users to delete them // during the publish operation failed. @@ -557,9 +568,10 @@ public class ChangeNotes extends AbstractChangeNotes<ChangeNotes> { * However, this method will load the comments if no draft comments have been loaded or if the * caller would like the drafts for another author. */ - private void loadDraftComments(Account.Id author, @Nullable Ref ref) { + private void loadDraftComments( + Account.Id author, @Nullable Change.Id virtualId, @Nullable Ref ref) { if (draftCommentNotes == null || !author.equals(draftCommentNotes.getAuthor()) || ref != null) { - draftCommentNotes = new DraftCommentNotes(args, getChangeId(), author, ref); + draftCommentNotes = new DraftCommentNotes(args, getChangeId(), virtualId, author, ref); draftCommentNotes.load(); } } @@ -581,14 +593,6 @@ public class ChangeNotes extends AbstractChangeNotes<ChangeNotes> { return robotCommentNotes; } - public boolean containsComment(HumanComment c) { - if (containsCommentPublished(c)) { - return true; - } - loadDraftComments(c.author.getId(), null); - return draftCommentNotes.containsComment(c); - } - public boolean containsCommentPublished(Comment c) { for (Comment l : getHumanComments().values()) { if (c.key.equals(l.key)) { diff --git a/java/com/google/gerrit/server/notedb/ChangeNotesParser.java b/java/com/google/gerrit/server/notedb/ChangeNotesParser.java index b6cdfac73a..e06dad3e18 100644 --- a/java/com/google/gerrit/server/notedb/ChangeNotesParser.java +++ b/java/com/google/gerrit/server/notedb/ChangeNotesParser.java @@ -880,7 +880,11 @@ class ChangeNotesParser { noteDbUtil .parseIdent(String.format("%s@%s", c.author.getId(), c.serverId)) - .ifPresent(id -> c.author = new Comment.Identity(id)); + .ifPresent( + id -> { + c.author = new Comment.Identity(id); + c.serverId = noteDbUtil.serverId; + }); humanComments.put(e.getKey(), c); } diff --git a/java/com/google/gerrit/server/notedb/ChangeNotesState.java b/java/com/google/gerrit/server/notedb/ChangeNotesState.java index 1715b438b4..1b1307831f 100644 --- a/java/com/google/gerrit/server/notedb/ChangeNotesState.java +++ b/java/com/google/gerrit/server/notedb/ChangeNotesState.java @@ -351,6 +351,7 @@ public abstract class ChangeNotesState { change.setOwner(c.owner()); change.setDest(BranchNameKey.create(change.getProject(), c.branch())); change.setCreatedOn(c.createdOn()); + change.setServerId(serverId()); copyNonConstructorColumnsTo(change); } diff --git a/java/com/google/gerrit/server/notedb/DraftCommentNotes.java b/java/com/google/gerrit/server/notedb/DraftCommentNotes.java index bdfe3782af..ae02708367 100644 --- a/java/com/google/gerrit/server/notedb/DraftCommentNotes.java +++ b/java/com/google/gerrit/server/notedb/DraftCommentNotes.java @@ -41,8 +41,15 @@ import org.eclipse.jgit.revwalk.RevCommit; public class DraftCommentNotes extends AbstractChangeNotes<DraftCommentNotes> { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + private final Change.Id virtualId; + public interface Factory { DraftCommentNotes create(Change.Id changeId, Account.Id accountId); + + DraftCommentNotes create( + @Assisted("changeId") Change.Id changeId, + @Assisted("virtualId") Change.Id virtualId, + Account.Id accountId); } private final Account.Id author; @@ -53,11 +60,26 @@ public class DraftCommentNotes extends AbstractChangeNotes<DraftCommentNotes> { @AssistedInject DraftCommentNotes(Args args, @Assisted Change.Id changeId, @Assisted Account.Id author) { - this(args, changeId, author, null); + this(args, changeId, null, author, null); + } + + @AssistedInject + DraftCommentNotes( + Args args, + @Assisted("changeId") Change.Id changeId, + @Assisted("virtualId") Change.Id virtualId, + @Assisted Account.Id author) { + this(args, changeId, virtualId, author, null); } - DraftCommentNotes(Args args, Change.Id changeId, Account.Id author, @Nullable Ref ref) { + DraftCommentNotes( + Args args, + Change.Id changeId, + @Nullable Change.Id virtualId, + Account.Id author, + @Nullable Ref ref) { super(args, changeId, null); + this.virtualId = virtualId; this.author = requireNonNull(author); this.ref = ref; if (ref != null) { @@ -93,7 +115,7 @@ public class DraftCommentNotes extends AbstractChangeNotes<DraftCommentNotes> { @Override protected String getRefName() { - return refsDraftComments(getChangeId(), author); + return refsDraftComments(virtualId != null ? virtualId : getChangeId(), author); } @Override diff --git a/java/com/google/gerrit/server/notedb/NoteDbUtil.java b/java/com/google/gerrit/server/notedb/NoteDbUtil.java index 5fc9244ea6..b1a4447d30 100644 --- a/java/com/google/gerrit/server/notedb/NoteDbUtil.java +++ b/java/com/google/gerrit/server/notedb/NoteDbUtil.java @@ -36,7 +36,7 @@ import org.eclipse.jgit.util.GitDateFormatter.Format; @Singleton public class NoteDbUtil { - private final String serverId; + final String serverId; private final ExternalIdCache externalIdCache; @Inject diff --git a/java/com/google/gerrit/server/patch/PatchFile.java b/java/com/google/gerrit/server/patch/PatchFile.java index 7a8180bd13..c3a6807fac 100644 --- a/java/com/google/gerrit/server/patch/PatchFile.java +++ b/java/com/google/gerrit/server/patch/PatchFile.java @@ -61,7 +61,7 @@ public class PatchFile { .filter(f -> f.getKey().equals(fileName)) .map(Map.Entry::getValue) .findFirst() - .orElse(FileDiffOutput.empty(fileName, ObjectId.zeroId(), ObjectId.zeroId())); + .orElseGet(() -> FileDiffOutput.empty(fileName, ObjectId.zeroId(), ObjectId.zeroId())); if (Patch.PATCHSET_LEVEL.equals(fileName)) { aTree = null; diff --git a/java/com/google/gerrit/server/permissions/ChangeControl.java b/java/com/google/gerrit/server/permissions/ChangeControl.java index 993c68d52a..db15da801d 100644 --- a/java/com/google/gerrit/server/permissions/ChangeControl.java +++ b/java/com/google/gerrit/server/permissions/ChangeControl.java @@ -62,7 +62,7 @@ class ChangeControl { /** Can this user see this change? */ boolean isVisible() { - if (getChange().isPrivate() && !isPrivateVisible(changeData)) { + if (changeData.isPrivateOrThrow() && !isPrivateVisible(changeData)) { return false; } // Does the user have READ permission on the destination? @@ -152,7 +152,7 @@ class ChangeControl { Permission.EDIT_TOPIC_NAME) // user can edit topic on a specific ref || getProjectControl().isAdmin(); } - return refControl.canForceEditTopicName(); + return refControl.canForceEditTopicName(isOwner()); } /** Can this user toggle WorkInProgress state? */ diff --git a/java/com/google/gerrit/server/permissions/DefaultRefFilter.java b/java/com/google/gerrit/server/permissions/DefaultRefFilter.java index eebaa8fad7..f1790454e5 100644 --- a/java/com/google/gerrit/server/permissions/DefaultRefFilter.java +++ b/java/com/google/gerrit/server/permissions/DefaultRefFilter.java @@ -27,7 +27,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.flogger.FluentLogger; -import com.google.gerrit.common.Nullable; import com.google.gerrit.entities.BranchNameKey; import com.google.gerrit.entities.Change; import com.google.gerrit.entities.RefNames; @@ -36,16 +35,16 @@ import com.google.gerrit.metrics.Description; import com.google.gerrit.metrics.MetricMaker; import com.google.gerrit.server.CurrentUser; import com.google.gerrit.server.config.GerritServerConfig; -import com.google.gerrit.server.git.SearchingChangeCacheImpl; +import com.google.gerrit.server.git.ChangesByProjectCache; import com.google.gerrit.server.git.TagCache; import com.google.gerrit.server.git.TagMatcher; import com.google.gerrit.server.logging.TraceContext; import com.google.gerrit.server.logging.TraceContext.TraceTimer; -import com.google.gerrit.server.notedb.ChangeNotes; import com.google.gerrit.server.permissions.PermissionBackend.RefFilterOptions; import com.google.gerrit.server.project.ProjectState; import com.google.gerrit.server.query.change.ChangeData; import com.google.inject.Inject; +import com.google.inject.Singleton; import com.google.inject.assistedinject.Assisted; import java.io.IOException; import java.util.ArrayList; @@ -65,6 +64,27 @@ class DefaultRefFilter { DefaultRefFilter create(ProjectControl projectControl); } + @Singleton + private static class Metrics { + final Counter0 fullFilterCount; + final Counter0 skipFilterCount; + + @Inject + Metrics(MetricMaker metricMaker) { + fullFilterCount = + metricMaker.newCounter( + "permissions/ref_filter/full_filter_count", + new Description("Rate of full ref filter operations").setRate()); + skipFilterCount = + metricMaker.newCounter( + "permissions/ref_filter/skip_filter_count", + new Description( + "Rate of ref filter operations where we skip full evaluation" + + " because the user can read all refs") + .setRate()); + } + } + private final TagCache tagCache; private final PermissionBackend permissionBackend; private final RefVisibilityControl refVisibilityControl; @@ -72,11 +92,9 @@ class DefaultRefFilter { private final CurrentUser user; private final ProjectState projectState; private final PermissionBackend.ForProject permissionBackendForProject; - private final @Nullable SearchingChangeCacheImpl searchingChangeDataProvider; + private final ChangesByProjectCache changesByProjectCache; private final ChangeData.Factory changeDataFactory; - private final ChangeNotes.Factory changeNotesFactory; - private final Counter0 fullFilterCount; - private final Counter0 skipFilterCount; + private final Metrics metrics; private final boolean skipFullRefEvaluationIfAllRefsAreVisible; @Inject @@ -85,17 +103,15 @@ class DefaultRefFilter { PermissionBackend permissionBackend, RefVisibilityControl refVisibilityControl, @GerritServerConfig Config config, - MetricMaker metricMaker, - @Nullable SearchingChangeCacheImpl searchingChangeDataProvider, + Metrics metrics, + ChangesByProjectCache changesByProjectCache, ChangeData.Factory changeDataFactory, - ChangeNotes.Factory changeNotesFactory, @Assisted ProjectControl projectControl) { this.tagCache = tagCache; this.permissionBackend = permissionBackend; this.refVisibilityControl = refVisibilityControl; - this.searchingChangeDataProvider = searchingChangeDataProvider; + this.changesByProjectCache = changesByProjectCache; this.changeDataFactory = changeDataFactory; - this.changeNotesFactory = changeNotesFactory; this.skipFullRefEvaluationIfAllRefsAreVisible = config.getBoolean("auth", "skipFullRefEvaluationIfAllRefsAreVisible", true); this.projectControl = projectControl; @@ -104,17 +120,7 @@ class DefaultRefFilter { this.projectState = projectControl.getProjectState(); this.permissionBackendForProject = permissionBackend.user(user).project(projectState.getNameKey()); - this.fullFilterCount = - metricMaker.newCounter( - "permissions/ref_filter/full_filter_count", - new Description("Rate of full ref filter operations").setRate()); - this.skipFilterCount = - metricMaker.newCounter( - "permissions/ref_filter/skip_filter_count", - new Description( - "Rate of ref filter operations where we skip full evaluation" - + " because the user can read all refs") - .setRate()); + this.metrics = metrics; } /** Filters given refs and tags by visibility. */ @@ -139,8 +145,7 @@ class DefaultRefFilter { Suppliers.memoize( () -> GitVisibleChangeFilter.getVisibleChanges( - searchingChangeDataProvider, - changeNotesFactory, + changesByProjectCache, changeDataFactory, projectState.getNameKey(), permissionBackendForProject, @@ -202,13 +207,13 @@ class DefaultRefFilter { logger.atFinest().log("User has READ on refs/* = %s", hasReadOnRefsStar); if (skipFullRefEvaluationIfAllRefsAreVisible && !projectState.isAllUsers()) { if (hasReadOnRefsStar) { - skipFilterCount.increment(); + metrics.skipFilterCount.increment(); logger.atFinest().log( "Fast path, all refs are visible because user has READ on refs/*: %s", refs); return new AutoValue_DefaultRefFilter_Result( ImmutableList.copyOf(refs), ImmutableList.of()); } else if (projectControl.allRefsAreVisible(ImmutableSet.of(RefNames.REFS_CONFIG))) { - skipFilterCount.increment(); + metrics.skipFilterCount.increment(); refs = fastHideRefsMetaConfig(refs); logger.atFinest().log( "Fast path, all refs except %s are visible: %s", RefNames.REFS_CONFIG, refs); @@ -217,7 +222,7 @@ class DefaultRefFilter { } } logger.atFinest().log("Doing full ref filtering"); - fullFilterCount.increment(); + metrics.fullFilterCount.increment(); boolean hasAccessDatabase = permissionBackend diff --git a/java/com/google/gerrit/server/permissions/GitVisibleChangeFilter.java b/java/com/google/gerrit/server/permissions/GitVisibleChangeFilter.java index 0e5ff486c6..640ea9a6b8 100644 --- a/java/com/google/gerrit/server/permissions/GitVisibleChangeFilter.java +++ b/java/com/google/gerrit/server/permissions/GitVisibleChangeFilter.java @@ -17,12 +17,9 @@ package com.google.gerrit.server.permissions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.flogger.FluentLogger; -import com.google.gerrit.common.Nullable; import com.google.gerrit.entities.Change; import com.google.gerrit.entities.Project; -import com.google.gerrit.exceptions.StorageException; -import com.google.gerrit.server.git.SearchingChangeCacheImpl; -import com.google.gerrit.server.notedb.ChangeNotes; +import com.google.gerrit.server.git.ChangesByProjectCache; import com.google.gerrit.server.query.change.ChangeData; import java.io.IOException; import java.util.HashMap; @@ -40,11 +37,7 @@ import org.eclipse.jgit.lib.Repository; * * <ul> * <li>For a low number of expected checks, we check visibility one-by-one. - * <li>For a high number of expected checks and settings where the change index is available, we - * load the N most recent changes from the index and filter them by visibility. This is fast, - * but comes with the caveat that older changes are pretended to be invisible. - * <li>For a high number of expected checks and settings where the change index is unavailable, we - * scan the repo and determine visibility one-by-one. This is *very* expensive. + * <li>For a high number of expected checks we use the ChangesByProjectCache. * </ul> * * <p>Changes that fail to load are pretended to be invisible. This is important on the Git paths as @@ -61,24 +54,23 @@ public class GitVisibleChangeFilter { /** Returns a map of all visible changes. Might pretend old changes are invisible. */ static ImmutableMap<Change.Id, ChangeData> getVisibleChanges( - @Nullable SearchingChangeCacheImpl searchingChangeCache, - ChangeNotes.Factory changeNotesFactory, + ChangesByProjectCache changesByProjectCache, ChangeData.Factory changeDataFactory, Project.NameKey projectName, PermissionBackend.ForProject forProject, Repository repository, ImmutableSet<Change.Id> changes) { - Stream<ChangeData> changeDatas; + Stream<ChangeData> changeDatas = Stream.empty(); if (changes.size() < CHANGE_LIMIT_FOR_DIRECT_FILTERING) { logger.atFine().log("Loading changes one by one for project %s", projectName); changeDatas = loadChangeDatasOneByOne(changes, changeDataFactory, projectName); - } else if (searchingChangeCache != null) { - logger.atFine().log("Loading changes from SearchingChangeCache for project %s", projectName); - changeDatas = searchingChangeCache.getChangeData(projectName); } else { - logger.atFine().log("Loading changes from all refs for project %s", projectName); - changeDatas = - scanRepoForChangeDatas(changeNotesFactory, changeDataFactory, repository, projectName); + logger.atFine().log("Loading changes from ChangesByProjectCache for project %s", projectName); + try { + changeDatas = changesByProjectCache.streamChangeDatas(projectName, repository); + } catch (IOException e) { + logger.atWarning().withCause(e).log("Unable to streamChangeDatas for %s", projectName); + } } HashMap<Change.Id, ChangeData> result = new HashMap<>(); changeDatas @@ -88,7 +80,11 @@ public class GitVisibleChangeFilter { try { return forProject.change(cd).test(ChangePermission.READ); } catch (PermissionBackendException e) { - throw new StorageException(e); + // This is almost the same as the message .testOrFalse() would log, but with the + // added context of the change and coming from this class + logger.atWarning().withCause(e).log( + "Cannot test read permission for %s; assuming not visible", cd); + return false; } }) .forEach( @@ -124,31 +120,4 @@ public class GitVisibleChangeFilter { }) .filter(Objects::nonNull); } - - /** Get a stream of all changes by scanning the repo. This is extremely slow. */ - private static Stream<ChangeData> scanRepoForChangeDatas( - ChangeNotes.Factory changeNotesFactory, - ChangeData.Factory changeDataFactory, - Repository repository, - Project.NameKey projectName) { - Stream<ChangeData> cds; - try { - cds = - changeNotesFactory - .scan(repository, projectName) - .map( - notesResult -> { - if (!notesResult.error().isPresent()) { - return changeDataFactory.create(notesResult.notes()); - } - logger.atWarning().withCause(notesResult.error().get()).log( - "Unable to load ChangeNotes for %s", notesResult.id()); - return null; - }) - .filter(Objects::nonNull); - } catch (IOException e) { - throw new StorageException(e); - } - return cds; - } } diff --git a/java/com/google/gerrit/server/permissions/PermissionBackend.java b/java/com/google/gerrit/server/permissions/PermissionBackend.java index eb5e053dc6..ac9ac98c42 100644 --- a/java/com/google/gerrit/server/permissions/PermissionBackend.java +++ b/java/com/google/gerrit/server/permissions/PermissionBackend.java @@ -274,7 +274,7 @@ public abstract class PermissionBackend { /** Returns an instance scoped for the change, and its destination ref and project. */ public ForChange change(ChangeData cd) { try { - return ref(cd.change().getDest().branch()).change(cd); + return ref(cd.branchOrThrow().branch()).change(cd); } catch (StorageException e) { return FailedPermissionBackend.change("unavailable", e); } diff --git a/java/com/google/gerrit/server/permissions/ProjectControl.java b/java/com/google/gerrit/server/permissions/ProjectControl.java index c23501265d..fab894e108 100644 --- a/java/com/google/gerrit/server/permissions/ProjectControl.java +++ b/java/com/google/gerrit/server/permissions/ProjectControl.java @@ -115,7 +115,7 @@ class ProjectControl { } ChangeControl controlFor(ChangeData cd) { - return new ChangeControl(controlForRef(cd.change().getDest()), cd); + return new ChangeControl(controlForRef(cd.branchOrThrow()), cd); } RefControl controlForRef(BranchNameKey ref) { @@ -366,7 +366,7 @@ class ProjectControl { @Override public ForChange change(ChangeData cd) { try { - checkProject(cd.change()); + checkProject(cd); return super.change(cd); } catch (StorageException e) { return FailedPermissionBackend.change("unavailable", e); @@ -379,13 +379,21 @@ class ProjectControl { return super.change(notes); } + private void checkProject(ChangeData cd) { + checkProject(cd.project()); + } + private void checkProject(Change change) { + checkProject(change.getProject()); + } + + private void checkProject(Project.NameKey changeProject) { Project.NameKey project = getProject().getNameKey(); checkArgument( - project.equals(change.getProject()), + project.equals(changeProject), "expected change in project %s, not %s", project, - change.getProject()); + changeProject); } @Override diff --git a/java/com/google/gerrit/server/permissions/RefControl.java b/java/com/google/gerrit/server/permissions/RefControl.java index ba292e67a2..7f9692bcf0 100644 --- a/java/com/google/gerrit/server/permissions/RefControl.java +++ b/java/com/google/gerrit/server/permissions/RefControl.java @@ -162,8 +162,8 @@ class RefControl { } /** Returns true if this user can force edit topic names. */ - boolean canForceEditTopicName() { - return canPerform(Permission.EDIT_TOPIC_NAME, false, true); + boolean canForceEditTopicName(boolean isChangeOwner) { + return canPerform(Permission.EDIT_TOPIC_NAME, isChangeOwner, true); } /** Returns true if this user can delete changes. */ diff --git a/java/com/google/gerrit/server/project/PeriodicProjectListCacheWarmer.java b/java/com/google/gerrit/server/project/PeriodicProjectListCacheWarmer.java index df2e1cf4d8..47d43b93a8 100644 --- a/java/com/google/gerrit/server/project/PeriodicProjectListCacheWarmer.java +++ b/java/com/google/gerrit/server/project/PeriodicProjectListCacheWarmer.java @@ -92,6 +92,11 @@ public class PeriodicProjectListCacheWarmer implements Runnable { } @Override + public String toString() { + return "Project List Cache Warmer"; + } + + @Override public void run() { logger.atFine().log("Loading project_list cache"); cache.refreshProjectList(); diff --git a/java/com/google/gerrit/server/project/Reachable.java b/java/com/google/gerrit/server/project/Reachable.java index 342c2bcc06..c935adf079 100644 --- a/java/com/google/gerrit/server/project/Reachable.java +++ b/java/com/google/gerrit/server/project/Reachable.java @@ -74,7 +74,7 @@ public class Reachable { Collection<Ref> filtered = optionalUserProvider .map(permissionBackend::user) - .orElse(permissionBackend.currentUser()) + .orElseGet(() -> permissionBackend.currentUser()) .project(project) .filter(refs, repo, RefFilterOptions.defaults()); Collection<RevCommit> visible = new ArrayList<>(); diff --git a/java/com/google/gerrit/server/project/SubmitRuleEvaluator.java b/java/com/google/gerrit/server/project/SubmitRuleEvaluator.java index 1d999dd9ed..c5928f6d7a 100644 --- a/java/com/google/gerrit/server/project/SubmitRuleEvaluator.java +++ b/java/com/google/gerrit/server/project/SubmitRuleEvaluator.java @@ -18,7 +18,6 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import com.google.common.collect.Streams; import com.google.common.flogger.FluentLogger; -import com.google.gerrit.entities.Change; import com.google.gerrit.entities.Project; import com.google.gerrit.entities.SubmitRecord; import com.google.gerrit.entities.SubmitTypeRecord; @@ -37,6 +36,7 @@ import com.google.gerrit.server.rules.DefaultSubmitRule; import com.google.gerrit.server.rules.PrologRule; import com.google.gerrit.server.rules.SubmitRule; import com.google.inject.Inject; +import com.google.inject.Singleton; import com.google.inject.assistedinject.Assisted; import java.util.List; import java.util.Optional; @@ -48,41 +48,51 @@ import java.util.Optional; public class SubmitRuleEvaluator { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + public interface Factory { + /** Returns a new {@link SubmitRuleEvaluator} with the specified options */ + SubmitRuleEvaluator create(SubmitRuleOptions options); + } + + @Singleton + private static class Metrics { + final Timer0 submitRuleEvaluationLatency; + final Timer0 submitTypeEvaluationLatency; + + @Inject + Metrics(MetricMaker metricMaker) { + submitRuleEvaluationLatency = + metricMaker.newTimer( + "change/submit_rule_evaluation", + new Description("Latency for evaluating submit rules on a change.") + .setCumulative() + .setUnit(Units.MILLISECONDS)); + submitTypeEvaluationLatency = + metricMaker.newTimer( + "change/submit_type_evaluation", + new Description("Latency for evaluating the submit type on a change.") + .setCumulative() + .setUnit(Units.MILLISECONDS)); + } + } + private final ProjectCache projectCache; private final PrologRule prologRule; private final PluginSetContext<SubmitRule> submitRules; - private final Timer0 submitRuleEvaluationLatency; - private final Timer0 submitTypeEvaluationLatency; + private final Metrics metrics; private final SubmitRuleOptions opts; private final CallerFinder callerFinder; - public interface Factory { - /** Returns a new {@link SubmitRuleEvaluator} with the specified options */ - SubmitRuleEvaluator create(SubmitRuleOptions options); - } - @Inject private SubmitRuleEvaluator( ProjectCache projectCache, PrologRule prologRule, PluginSetContext<SubmitRule> submitRules, - MetricMaker metricMaker, + Metrics metrics, @Assisted SubmitRuleOptions options) { this.projectCache = projectCache; this.prologRule = prologRule; this.submitRules = submitRules; - this.submitRuleEvaluationLatency = - metricMaker.newTimer( - "change/submit_rule_evaluation", - new Description("Latency for evaluating submit rules on a change.") - .setCumulative() - .setUnit(Units.MILLISECONDS)); - this.submitTypeEvaluationLatency = - metricMaker.newTimer( - "change/submit_type_evaluation", - new Description("Latency for evaluating the submit type on a change.") - .setCumulative() - .setUnit(Units.MILLISECONDS)); + this.metrics = metrics; this.opts = options; @@ -106,26 +116,13 @@ public class SubmitRuleEvaluator { logger.atFine().log( "Evaluate submit rules for change %d (caller: %s)", cd.change().getId().get(), callerFinder.findCallerLazy()); - try (Timer0.Context ignored = submitRuleEvaluationLatency.start()) { - Change change; - ProjectState projectState; - try { - change = cd.change(); - if (change == null) { - throw new StorageException("Change not found"); - } - - Project.NameKey name = cd.project(); - Optional<ProjectState> projectStateOptional = projectCache.get(name); - if (!projectStateOptional.isPresent()) { - throw new NoSuchProjectException(name); - } - projectState = projectStateOptional.get(); - } catch (NoSuchProjectException e) { - throw new IllegalStateException("Unable to find project while evaluating submit rule", e); + try (Timer0.Context ignored = metrics.submitRuleEvaluationLatency.start()) { + if (cd.change() == null) { + throw new StorageException("Change not found"); } - if (change.isClosed() && (!opts.recomputeOnClosedChanges() || OnlineReindexMode.isActive())) { + if (cd.change().isClosed() + && (!opts.recomputeOnClosedChanges() || OnlineReindexMode.isActive())) { return cd.notes().getSubmitRecords().stream() .map( r -> { @@ -139,6 +136,15 @@ public class SubmitRuleEvaluator { .collect(toImmutableList()); } + ProjectState projectState = + projectCache + .get(cd.project()) + .orElseThrow( + () -> + new IllegalStateException( + "Unable to find project while evaluating submit rule", + new NoSuchProjectException(cd.project()))); + // We evaluate all the plugin-defined evaluators, // and then we collect the results in one list. return Streams.stream(submitRules) @@ -173,7 +179,7 @@ public class SubmitRuleEvaluator { * @return record from the evaluated rules. */ public SubmitTypeRecord getSubmitType(ChangeData cd) { - try (Timer0.Context ignored = submitTypeEvaluationLatency.start()) { + try (Timer0.Context ignored = metrics.submitTypeEvaluationLatency.start()) { try { Project.NameKey name = cd.project(); Optional<ProjectState> project = projectCache.get(name); diff --git a/java/com/google/gerrit/server/query/account/AccountQueryProcessor.java b/java/com/google/gerrit/server/query/account/AccountQueryProcessor.java index eef913e13a..d812eefa49 100644 --- a/java/com/google/gerrit/server/query/account/AccountQueryProcessor.java +++ b/java/com/google/gerrit/server/query/account/AccountQueryProcessor.java @@ -34,6 +34,7 @@ import com.google.gerrit.server.index.account.AccountSchemaDefinitions; import com.google.gerrit.server.notedb.Sequences; import com.google.inject.Inject; import com.google.inject.Provider; +import com.google.inject.Singleton; /** * Query processor for the account index. @@ -46,6 +47,14 @@ public class AccountQueryProcessor extends QueryProcessor<AccountState> { private final Sequences sequences; private final IndexConfig indexConfig; + @Singleton + protected static class AccountQueryMetrics extends QueryProcessor.Metrics { + @Inject + protected AccountQueryMetrics(MetricMaker metricMaker) { + super(metricMaker); + } + } + static { // It is assumed that basic rewrites do not touch visibleto predicates. checkState( @@ -57,14 +66,14 @@ public class AccountQueryProcessor extends QueryProcessor<AccountState> { protected AccountQueryProcessor( Provider<CurrentUser> userProvider, AccountLimits.Factory limitsFactory, - MetricMaker metricMaker, + AccountQueryMetrics accountQueryMetrics, IndexConfig indexConfig, AccountIndexCollection indexes, AccountIndexRewriter rewriter, AccountControl.Factory accountControlFactory, Sequences sequences) { super( - metricMaker, + accountQueryMetrics, AccountSchemaDefinitions.INSTANCE, indexConfig, indexes, diff --git a/java/com/google/gerrit/server/query/change/ChangeData.java b/java/com/google/gerrit/server/query/change/ChangeData.java index cbd8d8308d..4ea84fb55e 100644 --- a/java/com/google/gerrit/server/query/change/ChangeData.java +++ b/java/com/google/gerrit/server/query/change/ChangeData.java @@ -40,6 +40,7 @@ import com.google.common.primitives.Ints; import com.google.gerrit.common.Nullable; import com.google.gerrit.entities.Account; import com.google.gerrit.entities.AttentionSetUpdate; +import com.google.gerrit.entities.BranchNameKey; import com.google.gerrit.entities.Change; import com.google.gerrit.entities.ChangeMessage; import com.google.gerrit.entities.Comment; @@ -74,7 +75,7 @@ import com.google.gerrit.server.change.CommentThreads; import com.google.gerrit.server.change.MergeabilityCache; import com.google.gerrit.server.change.PureRevert; import com.google.gerrit.server.config.AllUsersName; -import com.google.gerrit.server.config.GerritServerId; +import com.google.gerrit.server.config.SkipCurrentRulesEvaluationOnClosedChanges; import com.google.gerrit.server.config.TrackingFooters; import com.google.gerrit.server.git.GitRepositoryManager; import com.google.gerrit.server.git.MergeUtilFactory; @@ -105,6 +106,7 @@ import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.function.Function; @@ -236,23 +238,46 @@ public class ChangeData { } public ChangeData create(Project.NameKey project, Change.Id id) { - return assistedFactory.create(project, id, null, null); + return assistedFactory.create(project, id, null, null, null); + } + + public ChangeData create(Project.NameKey project, Change.Id id, ObjectId metaRevision) { + ChangeData cd = assistedFactory.create(project, id, null, null, null); + cd.setMetaRevision(metaRevision); + return cd; + } + + public ChangeData createNonPrivate(BranchNameKey branch, Change.Id id, ObjectId metaRevision) { + ChangeData cd = create(branch.project(), id, metaRevision); + cd.branch = branch.branch(); + cd.isPrivate = false; + return cd; } public ChangeData create(Change change) { - return assistedFactory.create(change.getProject(), change.getId(), change, null); + return create(change, null); + } + + public ChangeData create(Change change, Change.Id virtualId) { + return assistedFactory.create( + change.getProject(), + change.getId(), + !Objects.equals(virtualId, change.getId()) ? virtualId : null, + change, + null); } public ChangeData create(ChangeNotes notes) { return assistedFactory.create( - notes.getChange().getProject(), notes.getChangeId(), notes.getChange(), notes); + notes.getChange().getProject(), notes.getChangeId(), null, notes.getChange(), notes); } } public interface AssistedFactory { ChangeData create( Project.NameKey project, - Change.Id id, + @Assisted("changeId") Change.Id id, + @Assisted("virtualId") @Nullable Change.Id virtualId, @Nullable Change change, @Nullable ChangeNotes notes); } @@ -271,7 +296,7 @@ public class ChangeData { */ public static ChangeData createForTest( Project.NameKey project, Change.Id id, int currentPatchSetId, ObjectId commitId) { - return createForTest(project, id, currentPatchSetId, commitId, null, null, null); + return createForTest(project, id, currentPatchSetId, commitId, null, null); } /** @@ -284,7 +309,6 @@ public class ChangeData { * @param id change ID * @param currentPatchSetId current patchset number * @param commitId commit SHA1 of the current patchset - * @param serverId Gerrit server id * @param virtualIdAlgo algorithm for virtualising the Change number * @param changeNotes notes associated with the Change * @return instance for testing. @@ -294,7 +318,6 @@ public class ChangeData { Change.Id id, int currentPatchSetId, ObjectId commitId, - @Nullable String serverId, @Nullable ChangeNumberVirtualIdAlgorithm virtualIdAlgo, @Nullable ChangeNotes changeNotes) { ChangeData cd = @@ -316,11 +339,12 @@ public class ChangeData { null, null, null, - serverId, virtualIdAlgo, + false, project, id, null, + null, changeNotes); cd.currentPatchSet = PatchSet.builder() @@ -351,6 +375,7 @@ public class ChangeData { private final SubmitRequirementsEvaluator submitRequirementsEvaluator; private final SubmitRequirementsUtil submitRequirementsUtil; private final SubmitRuleEvaluator.Factory submitRuleEvaluatorFactory; + private final boolean skipCurrentRulesEvaluationOnClosedChanges; // Required assisted injected fields. private final Project.NameKey project; @@ -371,6 +396,8 @@ public class ChangeData { private PatchSet currentPatchSet; private Collection<PatchSet> patchSets; private ListMultimap<PatchSet.Id, PatchSetApproval> allApprovals; + + private ListMultimap<PatchSet.Id, PatchSetApproval> allApprovalsWithCopied; private List<PatchSetApproval> currentApprovals; private List<String> currentFiles; private Optional<DiffSummary> diffSummary; @@ -380,7 +407,10 @@ public class ChangeData { private List<ChangeMessage> messages; private Optional<ChangedLines> changedLines; private SubmitTypeRecord submitTypeRecord; + private String branch; + private Boolean isPrivate; private Boolean mergeable; + private ObjectId metaRevision; private Set<String> hashtags; /** * Map from {@link com.google.gerrit.entities.Account.Id} to the tip of the edit ref for this @@ -413,9 +443,9 @@ public class ChangeData { private Optional<Instant> mergedOn; private ImmutableSetMultimap<NameKey, RefState> refStates; private ImmutableList<byte[]> refStatePatterns; - private String gerritServerId; private String changeServerId; private ChangeNumberVirtualIdAlgorithm virtualIdFunc; + private Change.Id virtualId; @Inject private ChangeData( @@ -436,10 +466,11 @@ public class ChangeData { SubmitRequirementsEvaluator submitRequirementsEvaluator, SubmitRequirementsUtil submitRequirementsUtil, SubmitRuleEvaluator.Factory submitRuleEvaluatorFactory, - @GerritServerId String gerritServerId, ChangeNumberVirtualIdAlgorithm virtualIdFunc, + @SkipCurrentRulesEvaluationOnClosedChanges Boolean skipCurrentRulesEvaluationOnClosedChange, @Assisted Project.NameKey project, - @Assisted Change.Id id, + @Assisted("changeId") Change.Id id, + @Assisted("virtualId") @Nullable Change.Id virtualId, @Assisted @Nullable Change change, @Assisted @Nullable ChangeNotes notes) { this.approvalsUtil = approvalsUtil; @@ -459,6 +490,7 @@ public class ChangeData { this.submitRequirementsEvaluator = submitRequirementsEvaluator; this.submitRequirementsUtil = submitRequirementsUtil; this.submitRuleEvaluatorFactory = submitRuleEvaluatorFactory; + this.skipCurrentRulesEvaluationOnClosedChanges = skipCurrentRulesEvaluationOnClosedChange; this.project = project; this.legacyId = id; @@ -467,8 +499,8 @@ public class ChangeData { this.notes = notes; this.changeServerId = notes == null ? null : notes.getServerId(); - this.gerritServerId = gerritServerId; this.virtualIdFunc = virtualIdFunc; + this.virtualId = virtualId; } /** @@ -589,18 +621,88 @@ public class ChangeData { return legacyId; } - public Change.Id getVirtualId() { - if (virtualIdFunc == null || changeServerId == null || changeServerId.equals(gerritServerId)) { - return legacyId; + public static void ensureChangeServerId(Iterable<ChangeData> changes) { + ChangeData first = Iterables.getFirst(changes, null); + if (first == null) { + return; + } + + for (ChangeData cd : changes) { + cd.changeServerId(); + } + } + + @Nullable + public String changeServerId() { + if (changeServerId == null) { + if (!lazyload()) { + return null; + } + changeServerId = notes().getServerId(); } + return changeServerId; + } - return Change.id(virtualIdFunc.apply(changeServerId, legacyId.get())); + public Change.Id virtualId() { + if (virtualId == null) { + return virtualIdFunc == null ? legacyId : virtualIdFunc.apply(changeServerId, legacyId); + } + return virtualId; } public Project.NameKey project() { return project; } + public BranchNameKey branchOrThrow() { + if (change == null) { + if (branch != null) { + return BranchNameKey.create(project, branch); + } + throwIfNotLazyLoad("branch"); + change(); + } + return change.getDest(); + } + + public boolean isPrivateOrThrow() { + if (change == null) { + if (isPrivate != null) { + return isPrivate; + } + throwIfNotLazyLoad("isPrivate"); + change(); + } + return change.isPrivate(); + } + + public ChangeData setMetaRevision(ObjectId metaRevision) { + this.metaRevision = metaRevision; + return this; + } + + public ObjectId metaRevisionOrThrow() { + if (notes == null) { + if (metaRevision != null) { + return metaRevision; + } + if (refStates != null) { + Set<RefState> refs = refStates.get(project); + if (refs != null) { + String metaRef = RefNames.changeMetaRef(getId()); + for (RefState r : refs) { + if (r.ref().equals(metaRef)) { + return r.id(); + } + } + } + } + throwIfNotLazyLoad("metaRevision"); + notes(); + } + return notes.getRevision(); + } + boolean fastIsVisibleTo(CurrentUser user) { return visibleTo == user; } @@ -611,7 +713,7 @@ public class ChangeData { public Change change() { if (change == null && lazyload()) { - reloadChange(); + loadChange(); } return change; } @@ -621,13 +723,19 @@ public class ChangeData { } public Change reloadChange() { + metaRevision = null; + return loadChange(); + } + + private Change loadChange() { try { - notes = notesFactory.createChecked(project, legacyId); + notes = notesFactory.createChecked(project, legacyId, metaRevision); } catch (NoSuchChangeException e) { throw new StorageException("Unable to load change " + legacyId, e); } change = notes.getChange(); changeServerId = notes.getServerId(); + metaRevision = null; setPatchSets(null); return change; } @@ -645,7 +753,8 @@ public class ChangeData { if (!lazyload()) { throw new StorageException("ChangeNotes not available, lazyLoad = false"); } - notes = notesFactory.create(project(), legacyId); + notes = notesFactory.create(project(), legacyId, metaRevision); + change = notes.getChange(); } return notes; } @@ -852,6 +961,16 @@ public class ChangeData { return allApprovals; } + public ListMultimap<PatchSet.Id, PatchSetApproval> conditionallyLoadApprovalsWithCopied() { + if (allApprovalsWithCopied == null) { + if (!lazyload()) { + return ImmutableListMultimap.of(); + } + allApprovalsWithCopied = approvalsUtil.byChangeIncludingCopiedApprovals(notes()); + } + return allApprovalsWithCopied; + } + /* @return legacy submit ('SUBM') approval label */ // TODO(mariasavtchouk): Deprecate legacy submit label, // see com.google.gerrit.entities.LabelId.LEGACY_SUBMIT_NAME @@ -861,11 +980,7 @@ public class ChangeData { public ReviewerSet reviewers() { if (reviewers == null) { - if (!lazyload()) { - // We are not allowed to load values from NoteDb. Reviewers were not populated with values - // from the index. However, we need these values for permission checks. - throw new IllegalStateException("reviewers not populated"); - } + throwIfNotLazyLoad("reviewers"); reviewers = approvalsUtil.getReviewers(notes()); } return reviewers; @@ -1078,6 +1193,9 @@ public class ChangeData { project(), getId().get()); return Collections.emptyList(); } + if (skipCurrentRulesEvaluationOnClosedChanges && change().isClosed()) { + return notes().getSubmitRecords(); + } records = submitRuleEvaluatorFactory.create(options).evaluate(this); submitRecords.put(options, records); if (!change().isClosed() && submitRecords.size() == 1) { @@ -1120,8 +1238,6 @@ public class ChangeData { mergeable = true; } else if (c.isAbandoned()) { return null; - } else if (c.isWorkInProgress()) { - return null; } else { if (!lazyload()) { return null; @@ -1275,7 +1391,7 @@ public class ChangeData { if (!lazyload()) { return ImmutableMap.of(); } - starRefs = requireNonNull(starredChangesUtil).byChange(legacyId); + starRefs = requireNonNull(starredChangesUtil).byChange(virtualId()); } return starRefs; } @@ -1293,7 +1409,7 @@ public class ChangeData { if (!lazyload()) { return ImmutableSet.of(); } - starsOf = StarsOf.create(accountId, starredChangesUtil.getLabels(accountId, legacyId)); + starsOf = StarsOf.create(accountId, starredChangesUtil.getLabels(accountId, virtualId())); } } return starsOf.stars(); @@ -1389,6 +1505,14 @@ public class ChangeData { this.refStatePatterns = ImmutableList.copyOf(refStatePatterns); } + private void throwIfNotLazyLoad(String field) { + if (!lazyload()) { + // We are not allowed to load values from NoteDb. 'field' was not populated, however, + // we need this value for permission checks. + throw new IllegalStateException("'" + field + "' field not populated"); + } + } + @AutoValue abstract static class ReviewedByEvent { private static ReviewedByEvent create(ChangeMessage msg) { @@ -1431,7 +1555,7 @@ public class ChangeData { // this is suboptimal, but is ok for the purposes of // draftsByUser(), and easier than trying to rebuild the change at // this point. - && !notes().getDraftComments(account, ref).isEmpty()) { + && !notes().getDraftComments(account, virtualId(), ref).isEmpty()) { draftsByUser.put(account, ref.getObjectId()); } } diff --git a/java/com/google/gerrit/server/query/change/ChangeNumberBitmapMaskAlgorithm.java b/java/com/google/gerrit/server/query/change/ChangeNumberBitmapMaskAlgorithm.java index 726a3767c4..95c287a21e 100644 --- a/java/com/google/gerrit/server/query/change/ChangeNumberBitmapMaskAlgorithm.java +++ b/java/com/google/gerrit/server/query/change/ChangeNumberBitmapMaskAlgorithm.java @@ -16,7 +16,9 @@ package com.google.gerrit.server.query.change; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.gerrit.entities.Change; import com.google.gerrit.server.config.GerritImportedServerIds; +import com.google.gerrit.server.config.GerritServerId; import com.google.inject.Inject; import com.google.inject.ProvisionException; import com.google.inject.Singleton; @@ -38,9 +40,11 @@ public class ChangeNumberBitmapMaskAlgorithm implements ChangeNumberVirtualIdAlg Integer.BYTES * 8 - CHANGE_NUM_BIT_LEN; // Allows up to 64 ServerIds private final ImmutableMap<String, Integer> serverIdCodes; + private final String localServerId; @Inject public ChangeNumberBitmapMaskAlgorithm( + @GerritServerId String localServerId, @GerritImportedServerIds ImmutableList<String> importedServerIds) { if (importedServerIds.size() >= 1 << SERVER_ID_BIT_LEN) { throw new ProvisionException( @@ -54,10 +58,17 @@ public class ChangeNumberBitmapMaskAlgorithm implements ChangeNumberVirtualIdAlg } serverIdCodes = serverIdCodesBuilder.build(); + this.localServerId = localServerId; } @Override - public int apply(String changeServerId, int changeNum) { + public Change.Id apply(String changeServerId, Change.Id changeNumId) { + if (changeServerId == null || localServerId.equals(changeServerId)) { + return changeNumId; + } + + int changeNum = changeNumId.get(); + if ((changeNum & LEGACY_ID_BIT_MASK) != changeNum) { throw new IllegalArgumentException( String.format( @@ -71,6 +82,6 @@ public class ChangeNumberBitmapMaskAlgorithm implements ChangeNumberVirtualIdAlg } int virtualId = (changeNum & LEGACY_ID_BIT_MASK) | (encodedServerId << CHANGE_NUM_BIT_LEN); - return virtualId; + return Change.id(virtualId); } } diff --git a/java/com/google/gerrit/server/query/change/ChangeNumberVirtualIdAlgorithm.java b/java/com/google/gerrit/server/query/change/ChangeNumberVirtualIdAlgorithm.java index ab217050b9..6daf16f266 100644 --- a/java/com/google/gerrit/server/query/change/ChangeNumberVirtualIdAlgorithm.java +++ b/java/com/google/gerrit/server/query/change/ChangeNumberVirtualIdAlgorithm.java @@ -14,6 +14,7 @@ package com.google.gerrit.server.query.change; +import com.google.gerrit.entities.Change; import com.google.inject.ImplementedBy; /** @@ -31,5 +32,5 @@ public interface ChangeNumberVirtualIdAlgorithm { * @param legacyChangeNum legacy change number * @return virtual id which combines serverId and legacyChangeNum together */ - int apply(String serverId, int legacyChangeNum); + Change.Id apply(String serverId, Change.Id legacyChangeNum); } diff --git a/java/com/google/gerrit/server/query/change/ChangeQueryBuilder.java b/java/com/google/gerrit/server/query/change/ChangeQueryBuilder.java index 57b59ef180..816936b220 100644 --- a/java/com/google/gerrit/server/query/change/ChangeQueryBuilder.java +++ b/java/com/google/gerrit/server/query/change/ChangeQueryBuilder.java @@ -502,18 +502,22 @@ public class ChangeQueryBuilder extends QueryBuilder<ChangeData, ChangeQueryBuil @Inject protected ChangeQueryBuilder(Arguments args) { this(mydef, args); - setupAliases(); } @VisibleForTesting protected ChangeQueryBuilder(Definition<ChangeData, ChangeQueryBuilder> def, Arguments args) { super(def, args.opFactories); this.args = args; + setupAliases(); } private void setupAliases() { - setOperatorAliases(args.operatorAliasConfig.getChangeQueryOperatorAliases()); - hasOperandAliases = args.hasOperandAliasConfig.getChangeQueryHasOperandAliases(); + if (args.operatorAliasConfig != null) { + setOperatorAliases(args.operatorAliasConfig.getChangeQueryOperatorAliases()); + } + if (args.hasOperandAliasConfig != null) { + hasOperandAliases = args.hasOperandAliasConfig.getChangeQueryHasOperandAliases(); + } } public ChangeQueryBuilder asUser(CurrentUser user) { diff --git a/java/com/google/gerrit/server/query/change/ChangeQueryProcessor.java b/java/com/google/gerrit/server/query/change/ChangeQueryProcessor.java index b7dc127051..3097224246 100644 --- a/java/com/google/gerrit/server/query/change/ChangeQueryProcessor.java +++ b/java/com/google/gerrit/server/query/change/ChangeQueryProcessor.java @@ -43,6 +43,7 @@ import com.google.gerrit.server.index.change.IndexedChangeQuery; import com.google.gerrit.server.notedb.Sequences; import com.google.inject.Inject; import com.google.inject.Provider; +import com.google.inject.Singleton; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -66,6 +67,14 @@ public class ChangeQueryProcessor extends QueryProcessor<ChangeData> private final Sequences sequences; private final IndexConfig indexConfig; + @Singleton + protected static class ChangeQueryMetrics extends QueryProcessor.Metrics { + @Inject + protected ChangeQueryMetrics(MetricMaker metricMaker) { + super(metricMaker); + } + } + static { // It is assumed that basic rewrites do not touch visibleto predicates. checkState( @@ -77,7 +86,7 @@ public class ChangeQueryProcessor extends QueryProcessor<ChangeData> ChangeQueryProcessor( Provider<CurrentUser> userProvider, AccountLimits.Factory limitsFactory, - MetricMaker metricMaker, + ChangeQueryMetrics changeQueryMetrics, IndexConfig indexConfig, ChangeIndexCollection indexes, ChangeIndexRewriter rewriter, @@ -85,7 +94,7 @@ public class ChangeQueryProcessor extends QueryProcessor<ChangeData> ChangeIsVisibleToPredicate.Factory changeIsVisibleToPredicateFactory, DynamicSet<ChangePluginDefinedInfoFactory> changePluginDefinedInfoFactories) { super( - metricMaker, + changeQueryMetrics, ChangeSchemaDefinitions.INSTANCE, indexConfig, indexes, diff --git a/java/com/google/gerrit/server/query/change/EqualsLabelPredicates.java b/java/com/google/gerrit/server/query/change/EqualsLabelPredicates.java index ffd4497394..0d6dc3c4e1 100644 --- a/java/com/google/gerrit/server/query/change/EqualsLabelPredicates.java +++ b/java/com/google/gerrit/server/query/change/EqualsLabelPredicates.java @@ -74,7 +74,7 @@ public class EqualsLabelPredicates { LabelPredicate.Args args, String label, int expVal, - Account.Id account, + @Nullable Account.Id account, @Nullable Integer count) { super(ChangeField.LABEL_SPEC, ChangeField.formatLabel(label, expVal, account, count)); this.matcher = new Matcher(args, label, expVal, account, count); @@ -108,7 +108,7 @@ public class EqualsLabelPredicates { @Nullable protected final Integer count; /** Account ID that has voted on the label. */ - protected final Account.Id account; + @Nullable protected final Account.Id account; protected final AccountGroup.UUID group; @@ -120,7 +120,7 @@ public class EqualsLabelPredicates { LabelPredicate.Args args, String label, int expVal, - Account.Id account, + @Nullable Account.Id account, @Nullable Integer count) { this.permissionBackend = args.permissionBackend; this.accountResolver = args.accountResolver; @@ -246,9 +246,10 @@ public class EqualsLabelPredicates { } private boolean isMagicUser() { - return account.equals(ChangeQueryBuilder.OWNER_ACCOUNT_ID) - || account.equals(ChangeQueryBuilder.NON_UPLOADER_ACCOUNT_ID) - || account.equals(ChangeQueryBuilder.NON_CONTRIBUTOR_ACCOUNT_ID); + return account != null + && (account.equals(ChangeQueryBuilder.OWNER_ACCOUNT_ID) + || account.equals(ChangeQueryBuilder.NON_UPLOADER_ACCOUNT_ID) + || account.equals(ChangeQueryBuilder.NON_CONTRIBUTOR_ACCOUNT_ID)); } } diff --git a/java/com/google/gerrit/server/query/change/MagicLabelPredicates.java b/java/com/google/gerrit/server/query/change/MagicLabelPredicates.java index 9ee4852288..420ab61de7 100644 --- a/java/com/google/gerrit/server/query/change/MagicLabelPredicates.java +++ b/java/com/google/gerrit/server/query/change/MagicLabelPredicates.java @@ -80,7 +80,7 @@ public class MagicLabelPredicates { public IndexMatcher( LabelPredicate.Args args, MagicLabelVote magicLabelVote, - Account.Id account, + @Nullable Account.Id account, @Nullable Integer count) { super(args, magicLabelVote, account, count); } @@ -102,7 +102,7 @@ public class MagicLabelPredicates { public IndexMagicLabelPredicate( LabelPredicate.Args args, MagicLabelVote magicLabelVote, - Account.Id account, + @Nullable Account.Id account, @Nullable Integer count) { super( ChangeField.LABEL_SPEC, @@ -128,7 +128,7 @@ public class MagicLabelPredicates { private abstract static class Matcher { protected final LabelPredicate.Args args; protected final MagicLabelVote magicLabelVote; - protected final Account.Id account; + @Nullable protected final Account.Id account; @Nullable protected final Integer count; public Matcher( @@ -139,7 +139,7 @@ public class MagicLabelPredicates { public Matcher( LabelPredicate.Args args, MagicLabelVote magicLabelVote, - Account.Id account, + @Nullable Account.Id account, @Nullable Integer count) { this.account = account; this.args = args; @@ -180,9 +180,12 @@ public class MagicLabelPredicates { } public boolean ignoresUploaderApprovals() { - logger.atFine().log("account = %d", account.get()); - return account.equals(ChangeQueryBuilder.NON_UPLOADER_ACCOUNT_ID) - || account.equals(ChangeQueryBuilder.NON_CONTRIBUTOR_ACCOUNT_ID); + logger.atFine().log("account = %s", account); + if (account != null) { + return account.equals(ChangeQueryBuilder.NON_UPLOADER_ACCOUNT_ID) + || account.equals(ChangeQueryBuilder.NON_CONTRIBUTOR_ACCOUNT_ID); + } + return false; } private boolean matchAny(ChangeData changeData, LabelType labelType) { diff --git a/java/com/google/gerrit/server/query/change/OutputStreamQuery.java b/java/com/google/gerrit/server/query/change/OutputStreamQuery.java index 961404adb7..d21f5b62a3 100644 --- a/java/com/google/gerrit/server/query/change/OutputStreamQuery.java +++ b/java/com/google/gerrit/server/query/change/OutputStreamQuery.java @@ -303,7 +303,7 @@ public class OutputStreamQuery { rw, c, d.patchSets(), - includeApprovals ? d.approvals().asMap() : null, + includeApprovals ? d.conditionallyLoadApprovalsWithCopied().asMap() : null, includeFiles, d.change(), labelTypes, diff --git a/java/com/google/gerrit/server/query/change/PredicateArgs.java b/java/com/google/gerrit/server/query/change/PredicateArgs.java index ebe4390f08..02d2ca6951 100644 --- a/java/com/google/gerrit/server/query/change/PredicateArgs.java +++ b/java/com/google/gerrit/server/query/change/PredicateArgs.java @@ -71,7 +71,7 @@ public class PredicateArgs { * * @param args arguments to be parsed */ - PredicateArgs(String args) throws QueryParseException { + public PredicateArgs(String args) throws QueryParseException { positional = new ArrayList<>(); keyValue = new HashMap<>(); diff --git a/java/com/google/gerrit/server/query/group/GroupQueryProcessor.java b/java/com/google/gerrit/server/query/group/GroupQueryProcessor.java index 344a978e93..74c8d397d4 100644 --- a/java/com/google/gerrit/server/query/group/GroupQueryProcessor.java +++ b/java/com/google/gerrit/server/query/group/GroupQueryProcessor.java @@ -34,6 +34,7 @@ import com.google.gerrit.server.index.group.GroupSchemaDefinitions; import com.google.gerrit.server.notedb.Sequences; import com.google.inject.Inject; import com.google.inject.Provider; +import com.google.inject.Singleton; /** * Query processor for the group index. @@ -47,6 +48,14 @@ public class GroupQueryProcessor extends QueryProcessor<InternalGroup> { private final Sequences sequences; private final IndexConfig indexConfig; + @Singleton + protected static class GroupQueryMetrics extends QueryProcessor.Metrics { + @Inject + protected GroupQueryMetrics(MetricMaker metricMaker) { + super(metricMaker); + } + } + static { // It is assumed that basic rewrites do not touch visibleto predicates. checkState( @@ -58,14 +67,14 @@ public class GroupQueryProcessor extends QueryProcessor<InternalGroup> { protected GroupQueryProcessor( Provider<CurrentUser> userProvider, AccountLimits.Factory limitsFactory, - MetricMaker metricMaker, + GroupQueryMetrics groupQueryMetrics, IndexConfig indexConfig, GroupIndexCollection indexes, GroupIndexRewriter rewriter, GroupControl.GenericFactory groupControlFactory, Sequences sequences) { super( - metricMaker, + groupQueryMetrics, GroupSchemaDefinitions.INSTANCE, indexConfig, indexes, diff --git a/java/com/google/gerrit/server/query/project/ProjectQueryProcessor.java b/java/com/google/gerrit/server/query/project/ProjectQueryProcessor.java index 3877c2546c..ddc7ccc9af 100644 --- a/java/com/google/gerrit/server/query/project/ProjectQueryProcessor.java +++ b/java/com/google/gerrit/server/query/project/ProjectQueryProcessor.java @@ -34,6 +34,7 @@ import com.google.gerrit.server.permissions.PermissionBackend; import com.google.gerrit.server.project.ProjectCache; import com.google.inject.Inject; import com.google.inject.Provider; +import com.google.inject.Singleton; /** * Query processor for the project index. @@ -49,6 +50,14 @@ public class ProjectQueryProcessor extends QueryProcessor<ProjectData> { private final ProjectCache projectCache; private final IndexConfig indexConfig; + @Singleton + protected static class ProjectQueryMetrics extends QueryProcessor.Metrics { + @Inject + protected ProjectQueryMetrics(MetricMaker metricMaker) { + super(metricMaker); + } + } + static { // It is assumed that basic rewrites do not touch visibleto predicates. checkState( @@ -60,14 +69,14 @@ public class ProjectQueryProcessor extends QueryProcessor<ProjectData> { protected ProjectQueryProcessor( Provider<CurrentUser> userProvider, AccountLimits.Factory limitsFactory, - MetricMaker metricMaker, + ProjectQueryMetrics projectQueryMetrics, IndexConfig indexConfig, ProjectIndexCollection indexes, ProjectIndexRewriter rewriter, PermissionBackend permissionBackend, ProjectCache projectCache) { super( - metricMaker, + projectQueryMetrics, ProjectSchemaDefinitions.INSTANCE, indexConfig, indexes, diff --git a/java/com/google/gerrit/server/restapi/account/StarredChanges.java b/java/com/google/gerrit/server/restapi/account/StarredChanges.java index 173f24b6f7..8137ec99fa 100644 --- a/java/com/google/gerrit/server/restapi/account/StarredChanges.java +++ b/java/com/google/gerrit/server/restapi/account/StarredChanges.java @@ -73,7 +73,7 @@ public class StarredChanges IdentifiedUser user = parent.getUser(); ChangeResource change = changes.parse(TopLevelResource.INSTANCE, id); if (starredChangesUtil - .getLabels(user.getAccountId(), change.getId()) + .getLabels(user.getAccountId(), change.getVirtualId()) .contains(StarredChangesUtil.DEFAULT_LABEL)) { return new AccountResource.StarredChange(user, change); } @@ -131,7 +131,7 @@ public class StarredChanges try { starredChangesUtil.star( - self.get().getAccountId(), change.getId(), StarredChangesUtil.Operation.ADD); + self.get().getAccountId(), change.getVirtualId(), StarredChangesUtil.Operation.ADD); } catch (MutuallyExclusiveLabelsException e) { throw new ResourceConflictException(e.getMessage()); } catch (IllegalLabelException e) { @@ -179,7 +179,7 @@ public class StarredChanges throw new AuthException("not allowed remove starred change"); } starredChangesUtil.star( - self.get().getAccountId(), rsrc.getChange().getId(), StarredChangesUtil.Operation.REMOVE); + self.get().getAccountId(), rsrc.getVirtualId(), StarredChangesUtil.Operation.REMOVE); return Response.none(); } } diff --git a/java/com/google/gerrit/server/restapi/change/Files.java b/java/com/google/gerrit/server/restapi/change/Files.java index 7699873658..96e5645d4e 100644 --- a/java/com/google/gerrit/server/restapi/change/Files.java +++ b/java/com/google/gerrit/server/restapi/change/Files.java @@ -173,7 +173,7 @@ public class Files implements ChildCollection<RevisionResource, FileResource> { } else if (parentNum != 0) { int parents = gApi.changes() - .id(resource.getChange().getChangeId()) + .id(resource.getChange().getProject().get(), resource.getChange().getChangeId()) .revision(resource.getPatchSet().id().get()) .commit(false) .parents diff --git a/java/com/google/gerrit/server/restapi/change/Move.java b/java/com/google/gerrit/server/restapi/change/Move.java index c3688d62b0..2b0de12ee9 100644 --- a/java/com/google/gerrit/server/restapi/change/Move.java +++ b/java/com/google/gerrit/server/restapi/change/Move.java @@ -286,6 +286,9 @@ public class Move implements RestModifyView<ChangeResource, MoveInput>, UiAction .setTitle("Move change to a different branch") .setVisible(false); + if (!moveEnabled) { + return description; + } Change change = rsrc.getChange(); if (!change.isNew()) { return description; diff --git a/java/com/google/gerrit/server/restapi/config/GetServerInfo.java b/java/com/google/gerrit/server/restapi/config/GetServerInfo.java index 0116804472..63f2239407 100644 --- a/java/com/google/gerrit/server/restapi/config/GetServerInfo.java +++ b/java/com/google/gerrit/server/restapi/config/GetServerInfo.java @@ -52,6 +52,7 @@ import com.google.gerrit.server.config.AnonymousCowardName; import com.google.gerrit.server.config.AuthConfig; import com.google.gerrit.server.config.ConfigResource; import com.google.gerrit.server.config.ConfigUtil; +import com.google.gerrit.server.config.GerritInstanceId; import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.config.SitePaths; import com.google.gerrit.server.documentation.QueryDocumentationExecutor; @@ -92,6 +93,7 @@ public class GetServerInfo implements RestReadView<ConfigResource> { private final ProjectCache projectCache; private final AgreementJson agreementJson; private final SitePaths sitePaths; + private final @Nullable @GerritInstanceId String instanceId; @Inject public GetServerInfo( @@ -113,7 +115,8 @@ public class GetServerInfo implements RestReadView<ConfigResource> { QueryDocumentationExecutor docSearcher, ProjectCache projectCache, AgreementJson agreementJson, - SitePaths sitePaths) { + SitePaths sitePaths, + @Nullable @GerritInstanceId String instanceId) { this.config = config; this.accountVisibilityProvider = accountVisibilityProvider; this.accountDefaultDisplayName = accountDefaultDisplayName; @@ -133,6 +136,7 @@ public class GetServerInfo implements RestReadView<ConfigResource> { this.projectCache = projectCache; this.agreementJson = agreementJson; this.sitePaths = sitePaths; + this.instanceId = instanceId; } @Override @@ -289,7 +293,7 @@ public class GetServerInfo implements RestReadView<ConfigResource> { info.editGpgKeys = toBoolean(enableSignedPush && config.getBoolean("gerrit", null, "editGpgKeys", true)); info.primaryWeblinkName = config.getString("gerrit", null, "primaryWeblinkName"); - info.instanceId = config.getString("gerrit", null, "instanceId"); + info.instanceId = instanceId; return info; } diff --git a/java/com/google/gerrit/server/restapi/project/CreateAccessChange.java b/java/com/google/gerrit/server/restapi/project/CreateAccessChange.java index 65182db0e2..458ae4d9ba 100644 --- a/java/com/google/gerrit/server/restapi/project/CreateAccessChange.java +++ b/java/com/google/gerrit/server/restapi/project/CreateAccessChange.java @@ -17,6 +17,7 @@ package com.google.gerrit.server.restapi.project; import static com.google.gerrit.server.project.ProjectCache.illegalState; import static com.google.gerrit.server.update.context.RefUpdateContext.RefUpdateType.CHANGE_MODIFICATION; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.gerrit.entities.AccessSection; @@ -141,7 +142,15 @@ public class CreateAccessChange implements RestModifyView<ProjectResource, Proje throw new IllegalStateException(e); } - md.setMessage("Review access change"); + if (!Strings.isNullOrEmpty(input.message)) { + if (!input.message.endsWith("\n")) { + input.message += "\n"; + } + md.setMessage(input.message); + } else { + md.setMessage("Review access change\n"); + } + md.setInsertChangeId(true); Change.Id changeId = Change.id(seq.nextChangeId()); try (RefUpdateContext ctx = RefUpdateContext.open(CHANGE_MODIFICATION)) { diff --git a/java/com/google/gerrit/server/restapi/project/DeleteRef.java b/java/com/google/gerrit/server/restapi/project/DeleteRef.java index b7fe46e6de..388946edc0 100644 --- a/java/com/google/gerrit/server/restapi/project/DeleteRef.java +++ b/java/com/google/gerrit/server/restapi/project/DeleteRef.java @@ -117,9 +117,12 @@ public class DeleteRef { .check(RefPermission.DELETE); try (Repository repository = repoManager.openRepository(projectState.getNameKey())) { - RefUpdate.Result result; + Ref refObj = repository.exactRef(ref); + if (refObj == null) { + throw new ResourceConflictException(String.format("ref %s doesn't exist", ref)); + } RefUpdate u = repository.updateRef(ref); - u.setExpectedOldObjectId(repository.exactRef(ref).getObjectId()); + u.setExpectedOldObjectId(refObj.getObjectId()); u.setNewObjectId(ObjectId.zeroId()); u.setForceUpdate(true); refDeletionValidator.validateRefOperation( @@ -127,7 +130,7 @@ public class DeleteRef { identifiedUser.get(), u, /* pushOptions */ ImmutableListMultimap.of()); - result = u.delete(); + RefUpdate.Result result = u.delete(); switch (result) { case NEW: @@ -252,7 +255,7 @@ public class DeleteRef { RefUpdate u = r.updateRef(refName); u.setForceUpdate(true); - u.setExpectedOldObjectId(r.exactRef(refName).getObjectId()); + u.setExpectedOldObjectId(ref.getObjectId()); u.setNewObjectId(ObjectId.zeroId()); refDeletionValidator.validateRefOperation( projectState.getName(), diff --git a/java/com/google/gerrit/server/schema/CloudSpannerAccountPatchReviewStore.java b/java/com/google/gerrit/server/schema/CloudSpannerAccountPatchReviewStore.java new file mode 100644 index 0000000000..d993c4a133 --- /dev/null +++ b/java/com/google/gerrit/server/schema/CloudSpannerAccountPatchReviewStore.java @@ -0,0 +1,66 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.server.schema; + +import com.google.gerrit.exceptions.DuplicateKeyException; +import com.google.gerrit.exceptions.StorageException; +import com.google.gerrit.server.config.GerritServerConfig; +import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.config.ThreadSettingsConfig; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import java.sql.SQLException; +import java.sql.Statement; +import org.eclipse.jgit.lib.Config; + +@Singleton +public class CloudSpannerAccountPatchReviewStore extends JdbcAccountPatchReviewStore { + + private static final int ERR_DUP_KEY = 1022; + private static final int ERR_DUP_ENTRY = 1062; + private static final int ERR_DUP_UNIQUE = 1169; + + @Inject + CloudSpannerAccountPatchReviewStore( + @GerritServerConfig Config cfg, + SitePaths sitePaths, + ThreadSettingsConfig threadSettingsConfig) { + super(cfg, sitePaths, threadSettingsConfig); + } + + @Override + public StorageException convertError(String op, SQLException err) { + switch (err.getErrorCode()) { + case ERR_DUP_KEY: + case ERR_DUP_ENTRY: + case ERR_DUP_UNIQUE: + return new DuplicateKeyException("ACCOUNT_PATCH_REVIEWS", err); + + default: + return new StorageException(op + " failure on ACCOUNT_PATCH_REVIEWS", err); + } + } + + @Override + protected void doCreateTable(Statement stmt) throws SQLException { + stmt.executeUpdate( + "CREATE TABLE IF NOT EXISTS account_patch_reviews (" + + "account_id INT64 NOT NULL DEFAULT (0)," + + "change_id INT64 NOT NULL DEFAULT (0)," + + "patch_set_id INT64 NOT NULL DEFAULT (0)," + + "file_name STRING(MAX) NOT NULL DEFAULT ('')" + + ") PRIMARY KEY(change_id, patch_set_id, account_id, file_name)"); + } +} diff --git a/java/com/google/gerrit/server/schema/JdbcAccountPatchReviewStore.java b/java/com/google/gerrit/server/schema/JdbcAccountPatchReviewStore.java index 189d448d2b..8cc140edd9 100644 --- a/java/com/google/gerrit/server/schema/JdbcAccountPatchReviewStore.java +++ b/java/com/google/gerrit/server/schema/JdbcAccountPatchReviewStore.java @@ -53,8 +53,9 @@ public abstract class JdbcAccountPatchReviewStore implements AccountPatchReviewStore, LifecycleListener { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - // DB_CLOSE_DELAY=-1: By default the content of an in-memory H2 database is lost at the moment the - // last connection is closed. This option keeps the content as long as the VM lives. + // DB_CLOSE_DELAY=-1: By default the content of an in-memory H2 database is lost + // at the moment the last connection is closed. This option keeps the content as + // long as the VM lives. @VisibleForTesting public static final String TEST_IN_MEMORY_URL = "jdbc:h2:mem:account_patch_reviews;DB_CLOSE_DELAY=-1"; @@ -64,6 +65,7 @@ public abstract class JdbcAccountPatchReviewStore private static final String MARIADB = "mariadb"; private static final String MYSQL = "mysql"; private static final String POSTGRESQL = "postgresql"; + private static final String CLOUDSPANNER = "cloudspanner"; private static final String URL = "url"; public static class JdbcAccountPatchReviewStoreModule extends LifecycleModule { @@ -85,6 +87,8 @@ public abstract class JdbcAccountPatchReviewStore impl = MysqlAccountPatchReviewStore.class; } else if (url.contains(MARIADB)) { impl = MariaDBAccountPatchReviewStore.class; + } else if (url.contains(CLOUDSPANNER)) { + impl = CloudSpannerAccountPatchReviewStore.class; } else { throw new IllegalArgumentException( "unsupported driver type for account patch reviews db: " + url); @@ -111,6 +115,9 @@ public abstract class JdbcAccountPatchReviewStore if (url.contains(MARIADB)) { return new MariaDBAccountPatchReviewStore(cfg, sitePaths, threadSettingsConfig); } + if (url.contains(CLOUDSPANNER)) { + return new CloudSpannerAccountPatchReviewStore(cfg, sitePaths, threadSettingsConfig); + } throw new IllegalArgumentException( "unsupported driver type for account patch reviews db: " + url); } @@ -164,6 +171,9 @@ public abstract class JdbcAccountPatchReviewStore if (url.contains(MARIADB)) { return "org.mariadb.jdbc.Driver"; } + if (url.contains(CLOUDSPANNER)) { + return "com.google.cloud.spanner.jdbc.JdbcDriver"; + } return "org.h2.Driver"; } diff --git a/java/com/google/gerrit/server/submit/MergeIfNecessary.java b/java/com/google/gerrit/server/submit/MergeIfNecessary.java index 75136f50f5..b8417b8bc1 100644 --- a/java/com/google/gerrit/server/submit/MergeIfNecessary.java +++ b/java/com/google/gerrit/server/submit/MergeIfNecessary.java @@ -49,7 +49,7 @@ public class MergeIfNecessary extends SubmitStrategy { static boolean dryRun( SubmitDryRun.Arguments args, CodeReviewCommit mergeTip, CodeReviewCommit toMerge) { - return args.mergeUtil.canFastForward(args.mergeSorter, mergeTip, args.rw, toMerge) - || args.mergeUtil.canMerge(args.mergeSorter, args.repo, mergeTip, toMerge); + return args.mergeUtil.canFastForwardOrMerge( + args.mergeSorter, mergeTip, args.rw, args.repo, toMerge); } } diff --git a/java/com/google/gerrit/server/submit/RebaseSorter.java b/java/com/google/gerrit/server/submit/RebaseSorter.java index e960284d14..3645d3fb76 100644 --- a/java/com/google/gerrit/server/submit/RebaseSorter.java +++ b/java/com/google/gerrit/server/submit/RebaseSorter.java @@ -40,7 +40,7 @@ public class RebaseSorter { private final CurrentUser caller; private final CodeReviewRevWalk rw; private final RevFlag canMergeFlag; - private final Set<RevCommit> uninterestingBranchTips; + private final RevCommit initialTip; private final Set<RevCommit> alreadyAccepted; private final Provider<InternalChangeQuery> queryProvider; private final Set<CodeReviewCommit> incoming; @@ -48,7 +48,7 @@ public class RebaseSorter { public RebaseSorter( CurrentUser caller, CodeReviewRevWalk rw, - Set<RevCommit> uninterestingBranchTips, + RevCommit initialTip, Set<RevCommit> alreadyAccepted, RevFlag canMergeFlag, Provider<InternalChangeQuery> queryProvider, @@ -56,7 +56,7 @@ public class RebaseSorter { this.caller = caller; this.rw = rw; this.canMergeFlag = canMergeFlag; - this.uninterestingBranchTips = uninterestingBranchTips; + this.initialTip = initialTip; this.alreadyAccepted = alreadyAccepted; this.queryProvider = queryProvider; this.incoming = incoming; @@ -70,8 +70,8 @@ public class RebaseSorter { rw.resetRetain(canMergeFlag); rw.markStart(n); - for (RevCommit uninterestingBranchTip : uninterestingBranchTips) { - rw.markUninteresting(uninterestingBranchTip); + if (initialTip != null) { + rw.markUninteresting(initialTip); } CodeReviewCommit c; diff --git a/java/com/google/gerrit/server/submit/SubmitStrategy.java b/java/com/google/gerrit/server/submit/SubmitStrategy.java index c7b322e118..bdda3fc5dd 100644 --- a/java/com/google/gerrit/server/submit/SubmitStrategy.java +++ b/java/com/google/gerrit/server/submit/SubmitStrategy.java @@ -22,7 +22,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import com.google.common.flogger.FluentLogger; -import com.google.gerrit.entities.BooleanProjectConfig; import com.google.gerrit.entities.BranchNameKey; import com.google.gerrit.entities.Change; import com.google.gerrit.entities.SubmissionId; @@ -218,18 +217,11 @@ public abstract class SubmitStrategy { projectCache.get(destBranch.project()).orElseThrow(illegalState(destBranch.project())); this.mergeSorter = new MergeSorter(caller, rw, alreadyAccepted, canMergeFlag, queryProvider, incoming); - Set<RevCommit> uninterestingBranchTips; - if (project.is(BooleanProjectConfig.CREATE_NEW_CHANGE_FOR_ALL_NOT_IN_TARGET)) { - RevCommit initialTip = mergeTip.getInitialTip(); - uninterestingBranchTips = initialTip == null ? Set.of() : Set.of(initialTip); - } else { - uninterestingBranchTips = alreadyAccepted; - } this.rebaseSorter = new RebaseSorter( caller, rw, - uninterestingBranchTips, + mergeTip.getInitialTip(), alreadyAccepted, canMergeFlag, queryProvider, diff --git a/java/com/google/gerrit/sshd/ChangeArgumentParser.java b/java/com/google/gerrit/sshd/ChangeArgumentParser.java index 92019ad8a3..3e9dd082f4 100644 --- a/java/com/google/gerrit/sshd/ChangeArgumentParser.java +++ b/java/com/google/gerrit/sshd/ChangeArgumentParser.java @@ -18,7 +18,7 @@ import com.google.gerrit.common.Nullable; import com.google.gerrit.entities.Change; import com.google.gerrit.entities.Project; import com.google.gerrit.extensions.restapi.AuthException; -import com.google.gerrit.extensions.restapi.RestApiException; +import com.google.gerrit.server.CurrentUser; import com.google.gerrit.server.change.ChangeFinder; import com.google.gerrit.server.change.ChangeResource; import com.google.gerrit.server.notedb.ChangeNotes; @@ -30,27 +30,27 @@ import com.google.gerrit.server.project.ProjectState; import com.google.gerrit.server.restapi.change.ChangesCollection; import com.google.gerrit.sshd.BaseCommand.UnloggedFailure; import com.google.inject.Inject; +import com.google.inject.Provider; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; public class ChangeArgumentParser { private final ChangesCollection changesCollection; private final ChangeFinder changeFinder; - private final ChangeNotes.Factory changeNotesFactory; private final PermissionBackend permissionBackend; + private final Provider<CurrentUser> currentUserProvider; @Inject ChangeArgumentParser( ChangesCollection changesCollection, ChangeFinder changeFinder, - ChangeNotes.Factory changeNotesFactory, - PermissionBackend permissionBackend) { + PermissionBackend permissionBackend, + Provider<CurrentUser> currentUserProvider) { this.changesCollection = changesCollection; this.changeFinder = changeFinder; - this.changeNotesFactory = changeNotesFactory; this.permissionBackend = permissionBackend; + this.currentUserProvider = currentUserProvider; } public void addChange(String id, Map<Change.Id, ChangeResource> changes) @@ -68,9 +68,22 @@ public class ChangeArgumentParser { String id, Map<Change.Id, ChangeResource> changes, @Nullable ProjectState projectState, - boolean useIndex) + @SuppressWarnings( + "unused") /* Issue 325821304: the useIndex parameter was introduced back in Gerrit + * v2.13 + * when ReviewDb was around and the changeFinder was purely relying on + * Lucene. + * Fast-forward to v3.7 and the situation is exactly the opposite: + * changeFinder uses Lucene or NoteDb depending on the format of the + * change id. + * TODO: The useIndex is effectively useless right now, but the method + * signature needs to be preserved in a stable (almost EOL) release + * like v3.7. + * The method signature can be amended the parameter removed once this + * change is merged to master. */ + boolean useIndex) throws UnloggedFailure, PermissionBackendException { - List<ChangeNotes> matched = useIndex ? changeFinder.find(id) : changeFromNotesFactory(id); + List<ChangeNotes> matched = changeFinder.find(id); List<ChangeNotes> toAdd = new ArrayList<>(changes.size()); boolean canMaintainServer; try { @@ -105,26 +118,10 @@ public class ChangeArgumentParser { } else if (toAdd.size() > 1) { throw new UnloggedFailure(1, "\"" + id + "\" matches multiple changes"); } - Change.Id cId = toAdd.get(0).getChangeId(); + ChangeNotes changeNotes = toAdd.get(0); ChangeResource changeResource; - try { - changeResource = changesCollection.parse(cId); - } catch (RestApiException e) { - throw new UnloggedFailure(1, "\"" + id + "\" no such change", e); - } - changes.put(cId, changeResource); - } - - private List<ChangeNotes> changeFromNotesFactory(String id) throws UnloggedFailure { - return changeNotesFactory.createUsingIndexLookup(parseId(id)); - } - - private List<Change.Id> parseId(String id) throws UnloggedFailure { - try { - return Arrays.asList(Change.id(Integer.parseInt(id))); - } catch (NumberFormatException e) { - throw new UnloggedFailure(2, "Invalid change ID " + id, e); - } + changeResource = changesCollection.parse(changeNotes, currentUserProvider.get()); + changes.put(changeNotes.getChangeId(), changeResource); } private boolean inProject(ProjectState projectState, Project.NameKey project) { diff --git a/java/com/google/gerrit/sshd/InactiveAccountDisconnector.java b/java/com/google/gerrit/sshd/InactiveAccountDisconnector.java index 1086626ce6..2f96915508 100644 --- a/java/com/google/gerrit/sshd/InactiveAccountDisconnector.java +++ b/java/com/google/gerrit/sshd/InactiveAccountDisconnector.java @@ -39,7 +39,9 @@ public class InactiveAccountDisconnector implements AccountActivationListener { sshDaemon, (sshId, sshSession, abstractSession, ioSession) -> { CurrentUser sessionUser = sshSession.getUser(); - if (sessionUser.isIdentifiedUser() && sessionUser.getAccountId().get() == id) { + if (sessionUser != null + && sessionUser.isIdentifiedUser() + && sessionUser.getAccountId().get() == id) { logger.atInfo().log( "Disconnecting SSH session %s because user %s(%d) got deactivated", abstractSession, sessionUser.getLoggableName(), id); diff --git a/java/com/google/gerrit/sshd/InvalidKeyAlgorithmException.java b/java/com/google/gerrit/sshd/InvalidKeyAlgorithmException.java new file mode 100644 index 0000000000..5f09658ba2 --- /dev/null +++ b/java/com/google/gerrit/sshd/InvalidKeyAlgorithmException.java @@ -0,0 +1,44 @@ +// Copyright (C) 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.sshd; + +import java.security.PublicKey; +import java.security.spec.InvalidKeySpecException; + +public class InvalidKeyAlgorithmException extends InvalidKeySpecException { + private final String invalidKeyAlgo; + private final String expectedKeyAlgo; + private final PublicKey publicKey; + + public InvalidKeyAlgorithmException( + String invalidKeyAlgo, String expectedKeyAlgo, PublicKey publicKey) { + super("Key algorithm mismatch: expected " + expectedKeyAlgo + " but got " + invalidKeyAlgo); + this.invalidKeyAlgo = invalidKeyAlgo; + this.expectedKeyAlgo = expectedKeyAlgo; + this.publicKey = publicKey; + } + + public String getInvalidKeyAlgo() { + return invalidKeyAlgo; + } + + public String getExpectedKeyAlgo() { + return expectedKeyAlgo; + } + + public PublicKey getPublicKey() { + return publicKey; + } +} diff --git a/java/com/google/gerrit/sshd/SshDaemon.java b/java/com/google/gerrit/sshd/SshDaemon.java index cc35a32beb..af7d22bee0 100644 --- a/java/com/google/gerrit/sshd/SshDaemon.java +++ b/java/com/google/gerrit/sshd/SshDaemon.java @@ -77,6 +77,7 @@ import org.apache.sshd.common.file.nonefs.NoneFileSystemFactory; import org.apache.sshd.common.forward.DefaultForwarderFactory; import org.apache.sshd.common.future.CloseFuture; import org.apache.sshd.common.future.SshFutureListener; +import org.apache.sshd.common.global.KeepAliveHandler; import org.apache.sshd.common.io.AbstractIoServiceFactory; import org.apache.sshd.common.io.IoAcceptor; import org.apache.sshd.common.io.IoServiceFactory; @@ -109,7 +110,6 @@ import org.apache.sshd.server.auth.pubkey.UserAuthPublicKeyFactory; import org.apache.sshd.server.command.CommandFactory; import org.apache.sshd.server.forward.ForwardingFilter; import org.apache.sshd.server.global.CancelTcpipForwardHandler; -import org.apache.sshd.server.global.KeepAliveHandler; import org.apache.sshd.server.global.NoMoreSessionsHandler; import org.apache.sshd.server.global.TcpipForwardHandler; import org.apache.sshd.server.session.ServerSessionImpl; diff --git a/java/com/google/gerrit/sshd/SshKeyCacheImpl.java b/java/com/google/gerrit/sshd/SshKeyCacheImpl.java index 628a0503c0..58e331bb65 100644 --- a/java/com/google/gerrit/sshd/SshKeyCacheImpl.java +++ b/java/com/google/gerrit/sshd/SshKeyCacheImpl.java @@ -19,6 +19,7 @@ import static com.google.gerrit.server.account.externalids.ExternalId.SCHEME_USE import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.flogger.FluentLogger; +import com.google.gerrit.exceptions.InvalidSshKeyException; import com.google.gerrit.server.account.AccountSshKey; import com.google.gerrit.server.account.VersionedAuthorizedKeys; import com.google.gerrit.server.account.externalids.ExternalId; @@ -144,7 +145,16 @@ public class SshKeyCacheImpl implements SshKeyCache { // to do with the key object, and instead we must abort this load. // throw e; - } catch (Exception e) { + } catch (InvalidKeyAlgorithmException e) { + logger.atWarning().withCause(e).log( + "SSH key %d of account %s has an invalid algorithm %s: fixing the algorithm to %s", + k.seq(), k.accountId(), e.getInvalidKeyAlgo(), e.getExpectedKeyAlgo()); + if (fixKeyAlgorithm(k, e.getExpectedKeyAlgo())) { + kl.add(new SshKeyCacheEntry(k.accountId(), e.getPublicKey())); + } else { + markInvalid(k); + } + } catch (Throwable e) { markInvalid(k); } } @@ -158,5 +168,20 @@ public class SshKeyCacheImpl implements SshKeyCache { "Failed to mark SSH key %d of account %s invalid", k.seq(), k.accountId()); } } + + private boolean fixKeyAlgorithm(AccountSshKey k, String keyAlgo) { + try { + logger.atInfo().log( + "Fixing SSH key %d of account %s algorithm to %s", k.seq(), k.accountId(), keyAlgo); + authorizedKeys.deleteKey(k.accountId(), k.seq()); + String sshKey = k.sshPublicKey(); + authorizedKeys.addKey(k.accountId(), keyAlgo + sshKey.substring(sshKey.indexOf(' '))); + return true; + } catch (IOException | ConfigInvalidException | InvalidSshKeyException e) { + logger.atSevere().withCause(e).log( + "Failed to fix SSH key %d of account %s with algo %s", k.seq(), k.accountId(), keyAlgo); + return false; + } + } } } diff --git a/java/com/google/gerrit/sshd/SshPluginStarterCallback.java b/java/com/google/gerrit/sshd/SshPluginStarterCallback.java index 8711fe641d..f807b19c16 100644 --- a/java/com/google/gerrit/sshd/SshPluginStarterCallback.java +++ b/java/com/google/gerrit/sshd/SshPluginStarterCallback.java @@ -25,6 +25,10 @@ import com.google.inject.Inject; import com.google.inject.Key; import com.google.inject.Provider; import com.google.inject.Singleton; +import com.google.inject.TypeLiteral; +import com.google.inject.internal.MoreTypes; +import java.util.ArrayList; +import java.util.List; import org.apache.sshd.server.command.Command; @Singleton @@ -65,9 +69,9 @@ class SshPluginStarterCallback implements StartPluginListener, ReloadPluginListe try { return plugin.getSshInjector().getProvider(key); } catch (RuntimeException err) { - if (!providesDynamicOptions(plugin)) { + if (!providesDynamicOptions(plugin) && !providesCommandInterceptor(plugin)) { logger.atWarning().withCause(err).log( - "Plugin %s did not define its top-level command nor any DynamicOptions", + "Plugin %s did not define its top-level command, any DynamicOptions, nor any Ssh*CommandInterceptors", plugin.getName()); } } @@ -78,4 +82,16 @@ class SshPluginStarterCallback implements StartPluginListener, ReloadPluginListe private boolean providesDynamicOptions(Plugin plugin) { return dynamicBeans.plugins().contains(plugin.getName()); } + + private boolean providesCommandInterceptor(Plugin plugin) { + List<TypeLiteral<?>> typeLiterals = new ArrayList<>(2); + typeLiterals.add( + MoreTypes.canonicalizeForKey( + (TypeLiteral<?>) TypeLiteral.get(SshExecuteCommandInterceptor.class))); + typeLiterals.add( + MoreTypes.canonicalizeForKey( + (TypeLiteral<?>) TypeLiteral.get(SshCreateCommandInterceptor.class))); + return plugin.getSshInjector().getAllBindings().keySet().stream() + .anyMatch(key -> typeLiterals.contains(key.getTypeLiteral())); + } } diff --git a/java/com/google/gerrit/sshd/SshUtil.java b/java/com/google/gerrit/sshd/SshUtil.java index abbd81d83a..29d0e90f9f 100644 --- a/java/com/google/gerrit/sshd/SshUtil.java +++ b/java/com/google/gerrit/sshd/SshUtil.java @@ -57,7 +57,12 @@ public class SshUtil { throw new InvalidKeySpecException("No key string"); } final byte[] bin = BaseEncoding.base64().decode(s); - return new ByteArrayBuffer(bin).getRawPublicKey(); + String publicKeyAlgo = new ByteArrayBuffer(bin).getString(); + PublicKey publicKey = new ByteArrayBuffer(bin).getRawPublicKey(); + if (!key.algorithm().equals(publicKeyAlgo)) { + throw new InvalidKeyAlgorithmException(key.algorithm(), publicKeyAlgo, publicKey); + } + return publicKey; } catch (RuntimeException | SshException e) { throw new InvalidKeySpecException("Cannot parse key", e); } diff --git a/java/com/google/gerrit/sshd/commands/SetReviewersCommand.java b/java/com/google/gerrit/sshd/commands/SetReviewersCommand.java index 4f23d1df83..f42eb5cb53 100644 --- a/java/com/google/gerrit/sshd/commands/SetReviewersCommand.java +++ b/java/com/google/gerrit/sshd/commands/SetReviewersCommand.java @@ -27,6 +27,8 @@ import com.google.gerrit.server.permissions.PermissionBackendException; import com.google.gerrit.server.project.ProjectState; import com.google.gerrit.server.restapi.change.DeleteReviewer; import com.google.gerrit.server.restapi.change.PostReviewers; +import com.google.gerrit.server.update.RetryHelper; +import com.google.gerrit.server.update.RetryableAction; import com.google.gerrit.sshd.ChangeArgumentParser; import com.google.gerrit.sshd.CommandMetaData; import com.google.gerrit.sshd.SshCommand; @@ -89,6 +91,8 @@ public class SetReviewersCommand extends SshCommand { @Inject private ChangeArgumentParser changeArgumentParser; + @Inject private RetryHelper retryHelper; + private Set<Account.Id> toRemove = new HashSet<>(); private Map<Change.Id, ChangeResource> changes = new LinkedHashMap<>(); @@ -121,7 +125,15 @@ public class SetReviewersCommand extends SshCommand { ReviewerResource rsrc = reviewerFactory.create(changeRsrc, reviewer); String error = null; try { - deleteReviewer.apply(rsrc, new DeleteReviewerInput()); + retryHelper + .action( + RetryableAction.ActionType.CHANGE_UPDATE, + "removeReviewers", + () -> { + deleteReviewer.apply(rsrc, new DeleteReviewerInput()); + return null; + }) + .call(); } catch (ResourceNotFoundException e) { error = String.format("could not remove %s: not found", reviewer); } catch (Exception e) { @@ -139,15 +151,26 @@ public class SetReviewersCommand extends SshCommand { ReviewerInput input = new ReviewerInput(); input.reviewer = reviewer; input.confirmed = true; - String error; + var error = + new Object() { + String value; + }; try { - error = postReviewers.apply(changeRsrc, input).value().error; + retryHelper + .action( + RetryableAction.ActionType.CHANGE_UPDATE, + "applyReview", + () -> { + error.value = postReviewers.apply(changeRsrc, input).value().error; + return null; + }) + .call(); } catch (Exception e) { - error = String.format("could not add %s: %s", reviewer, e.getMessage()); + error.value = String.format("could not add %s: %s", reviewer, e.getMessage()); } - if (error != null) { + if (error.value != null) { ok = false; - writeError("error", error); + writeError("error", error.value); } } diff --git a/java/com/google/gerrit/sshd/commands/SetTopicCommand.java b/java/com/google/gerrit/sshd/commands/SetTopicCommand.java index 244fdbe92c..b5907404ba 100644 --- a/java/com/google/gerrit/sshd/commands/SetTopicCommand.java +++ b/java/com/google/gerrit/sshd/commands/SetTopicCommand.java @@ -16,13 +16,13 @@ package com.google.gerrit.sshd.commands; import com.google.gerrit.entities.Change; import com.google.gerrit.exceptions.StorageException; +import com.google.gerrit.extensions.api.changes.TopicInput; import com.google.gerrit.extensions.restapi.BadRequestException; +import com.google.gerrit.extensions.restapi.ResourceNotFoundException; import com.google.gerrit.server.ChangeUtil; import com.google.gerrit.server.change.ChangeResource; -import com.google.gerrit.server.change.SetTopicOp; import com.google.gerrit.server.permissions.PermissionBackendException; -import com.google.gerrit.server.update.BatchUpdate; -import com.google.gerrit.server.util.time.TimeUtil; +import com.google.gerrit.server.restapi.change.PutTopic; import com.google.gerrit.sshd.ChangeArgumentParser; import com.google.gerrit.sshd.CommandMetaData; import com.google.gerrit.sshd.SshCommand; @@ -34,9 +34,8 @@ import org.kohsuke.args4j.Option; @CommandMetaData(name = "set-topic", description = "Set the topic for one or more changes") public class SetTopicCommand extends SshCommand { - private final BatchUpdate.Factory updateFactory; private final ChangeArgumentParser changeArgumentParser; - private final SetTopicOp.Factory topicOpFactory; + private final PutTopic putTopic; private Map<Change.Id, ChangeResource> changes = new LinkedHashMap<>(); @@ -62,17 +61,14 @@ public class SetTopicCommand extends SshCommand { private String topic; @Inject - SetTopicCommand( - BatchUpdate.Factory updateFactory, - ChangeArgumentParser changeArgumentParser, - SetTopicOp.Factory topicOpFactory) { - this.updateFactory = updateFactory; + SetTopicCommand(ChangeArgumentParser changeArgumentParser, PutTopic putTopic) { this.changeArgumentParser = changeArgumentParser; - this.topicOpFactory = topicOpFactory; + this.putTopic = putTopic; } @Override public void run() throws Exception { + boolean ok = true; if (topic != null) { topic = topic.trim(); } @@ -83,11 +79,28 @@ public class SetTopicCommand extends SshCommand { } for (ChangeResource r : changes.values()) { - SetTopicOp op = topicOpFactory.create(topic); - try (BatchUpdate u = updateFactory.create(r.getChange().getProject(), user, TimeUtil.now())) { - u.addOp(r.getId(), op); - u.execute(); + TopicInput input = new TopicInput(); + input.topic = topic; + try { + putTopic.apply(r, input); + } catch (ResourceNotFoundException e) { + ok = false; + writeError( + "error", + String.format( + "could not add topic to change %d: not found", r.getChange().getChangeId())); + } catch (Exception e) { + ok = false; + writeError( + "error", + String.format( + "could not add topic to change %d: %s", + r.getChange().getChangeId(), e.getMessage())); } } + + if (!ok) { + throw die("one or more updates failed"); + } } } diff --git a/java/com/google/gerrit/testing/InMemoryModule.java b/java/com/google/gerrit/testing/InMemoryModule.java index 52ac58a183..00020302a6 100644 --- a/java/com/google/gerrit/testing/InMemoryModule.java +++ b/java/com/google/gerrit/testing/InMemoryModule.java @@ -81,10 +81,10 @@ import com.google.gerrit.server.config.SitePath; import com.google.gerrit.server.config.TrackingFooters; import com.google.gerrit.server.config.TrackingFootersProvider; import com.google.gerrit.server.experiments.ConfigExperimentFeatures.ConfigExperimentFeaturesModule; +import com.google.gerrit.server.git.ChangesByProjectCache; import com.google.gerrit.server.git.GarbageCollection; import com.google.gerrit.server.git.GitRepositoryManager; import com.google.gerrit.server.git.PerThreadRequestScope; -import com.google.gerrit.server.git.SearchingChangeCacheImpl.SearchingChangeCacheImplModule; import com.google.gerrit.server.git.WorkQueue; import com.google.gerrit.server.git.WorkQueue.WorkQueueModule; import com.google.gerrit.server.group.testing.TestGroupBackend; @@ -200,7 +200,7 @@ public class InMemoryModule extends FactoryModule { factory(PluginUser.Factory.class); install(new PluginApiModule()); install(new DefaultPermissionBackendModule()); - install(new SearchingChangeCacheImplModule()); + install(new ChangesByProjectCache.Module(ChangesByProjectCache.UseIndex.TRUE, cfg)); factory(GarbageCollection.Factory.class); install(new AuditModule()); install(new SubscriptionGraphModule()); diff --git a/javatests/com/google/gerrit/acceptance/api/change/ChangeIT.java b/javatests/com/google/gerrit/acceptance/api/change/ChangeIT.java index 29ea7f45ce..8e2bd8bdd4 100644 --- a/javatests/com/google/gerrit/acceptance/api/change/ChangeIT.java +++ b/javatests/com/google/gerrit/acceptance/api/change/ChangeIT.java @@ -32,6 +32,7 @@ import static com.google.gerrit.acceptance.testsuite.project.TestProjectUpdate.b import static com.google.gerrit.acceptance.testsuite.project.TestProjectUpdate.labelPermissionKey; import static com.google.gerrit.acceptance.testsuite.project.TestProjectUpdate.permissionKey; import static com.google.gerrit.entities.RefNames.changeMetaRef; +import static com.google.gerrit.extensions.client.ChangeStatus.ABANDONED; import static com.google.gerrit.extensions.client.ChangeStatus.MERGED; import static com.google.gerrit.extensions.client.ListChangesOption.ALL_REVISIONS; import static com.google.gerrit.extensions.client.ListChangesOption.CHANGE_ACTIONS; @@ -3545,6 +3546,74 @@ public class ChangeIT extends AbstractDaemonTest { } @Test + @GerritConfig(name = "change.skipCurrentRulesEvaluationOnClosedChanges", value = "true") + public void checkLabelsNotUpdatedForMergedChange() throws Exception { + PushOneCommit.Result r = createChange(); + gApi.changes().id(r.getChangeId()).revision(r.getCommit().name()).review(ReviewInput.approve()); + gApi.changes().id(r.getChangeId()).revision(r.getCommit().name()).submit(); + + ChangeInfo change = gApi.changes().id(r.getChangeId()).get(); + assertThat(change.status).isEqualTo(MERGED); + assertThat(change.submissionId).isNotNull(); + assertThat(change.labels.keySet()).containsExactly(LabelId.CODE_REVIEW); + assertThat(change.permittedLabels.keySet()).containsExactly(LabelId.CODE_REVIEW); + assertPermitted(change, LabelId.CODE_REVIEW, 2); + + LabelType verified = TestLabels.verified(); + AccountGroup.UUID registeredUsers = systemGroupBackend.getGroup(REGISTERED_USERS).getUUID(); + String heads = RefNames.REFS_HEADS + "*"; + + // add new label and assert that it's returned for existing changes + try (ProjectConfigUpdate u = updateProject(project)) { + u.getConfig().upsertLabelType(verified); + u.save(); + } + projectOperations + .project(project) + .forUpdate() + .add(allowLabel(verified.getName()).ref(heads).group(registeredUsers).range(-1, 1)) + .update(); + + change = gApi.changes().id(r.getChangeId()).get(); + assertThat(change.labels.keySet()).containsExactly(LabelId.CODE_REVIEW); + assertThat(change.permittedLabels.keySet()) + .containsExactly(LabelId.CODE_REVIEW, LabelId.VERIFIED); + assertPermitted(change, LabelId.CODE_REVIEW, 2); + } + + @Test + @GerritConfig(name = "change.skipCurrentRulesEvaluationOnClosedChanges", value = "true") + public void checkLabelsNotUpdatedForAbandonedChange() throws Exception { + PushOneCommit.Result r = createChange(); + gApi.changes().id(r.getChangeId()).abandon(); + + ChangeInfo change = gApi.changes().id(r.getChangeId()).get(); + assertThat(change.status).isEqualTo(ABANDONED); + assertThat(change.labels.keySet()).isEmpty(); + assertThat(change.submitRecords).isEmpty(); + + LabelType verified = TestLabels.verified(); + AccountGroup.UUID registeredUsers = systemGroupBackend.getGroup(REGISTERED_USERS).getUUID(); + String heads = RefNames.REFS_HEADS + "*"; + + // add new label and assert that it's returned for existing changes + try (ProjectConfigUpdate u = updateProject(project)) { + u.getConfig().upsertLabelType(verified); + u.save(); + } + projectOperations + .project(project) + .forUpdate() + .add(allowLabel(verified.getName()).ref(heads).group(registeredUsers).range(-1, 1)) + .update(); + + change = gApi.changes().id(r.getChangeId()).get(); + assertThat(change.labels.keySet()).isEmpty(); + assertThat(change.permittedLabels.keySet()).isEmpty(); + assertThat(change.submitRecords).isEmpty(); + } + + @Test public void notifyConfigForDirectoryTriggersEmail() throws Exception { // Configure notifications on project level. RevCommit oldHead = projectOperations.project(project).getHead("master"); @@ -4371,6 +4440,8 @@ public class ChangeIT extends AbstractDaemonTest { public void changeQueryReturnsMergeableWhenGerritIndexMergeable() throws Exception { String changeId = createChange().getChangeId(); assertThat(gApi.changes().query(changeId).get().get(0).mergeable).isTrue(); + gApi.changes().id(changeId).setWorkInProgress(); + assertThat(gApi.changes().query(changeId).get().get(0).mergeable).isTrue(); } @Test diff --git a/javatests/com/google/gerrit/acceptance/api/group/GroupsIT.java b/javatests/com/google/gerrit/acceptance/api/group/GroupsIT.java index 9456a31439..6dbbe9ac76 100644 --- a/javatests/com/google/gerrit/acceptance/api/group/GroupsIT.java +++ b/javatests/com/google/gerrit/acceptance/api/group/GroupsIT.java @@ -1285,16 +1285,24 @@ public class GroupsIT extends AbstractDaemonTest { } @Test - public void pushToGroupBranchForReviewForAllUsersRepoIsRejectedOnSubmit() throws Throwable { + public void pushToGroupBranchForReviewForAllUsersRepoIsRejectedOnSubmitForGroupFiles() + throws Throwable { + String error = "update to group files (group.config, members, subgroups) not allowed"; pushToGroupBranchForReviewAndSubmit( - allUsers, RefNames.refsGroups(adminGroupUuid()), "group update not allowed"); + allUsers, RefNames.refsGroups(adminGroupUuid()), "group.config", error); + pushToGroupBranchForReviewAndSubmit( + allUsers, RefNames.refsGroups(adminGroupUuid()), "members", error); + pushToGroupBranchForReviewAndSubmit( + allUsers, RefNames.refsGroups(adminGroupUuid()), "subgroups", error); + pushToGroupBranchForReviewAndSubmit( + allUsers, RefNames.refsGroups(adminGroupUuid()), "destinations/myreviews", null); } @Test public void pushToGroupBranchForReviewForNonAllUsersRepoAndSubmit() throws Throwable { String groupRef = RefNames.refsGroups(adminGroupUuid()); createBranch(project, groupRef); - pushToGroupBranchForReviewAndSubmit(project, groupRef, null); + pushToGroupBranchForReviewAndSubmit(project, groupRef, "group.config", null); } @Test @@ -1576,7 +1584,8 @@ public class GroupsIT extends AbstractDaemonTest { } private void pushToGroupBranchForReviewAndSubmit( - Project.NameKey project, String groupRef, String expectedError) throws Throwable { + Project.NameKey project, String groupRef, String fileName, String expectedError) + throws Throwable { projectOperations .project(project) .forUpdate() @@ -1594,7 +1603,7 @@ public class GroupsIT extends AbstractDaemonTest { PushOneCommit.Result r = pushFactory - .create(admin.newIdent(), repo, "Update group config", "group.config", "some content") + .create(admin.newIdent(), repo, "Update group config", fileName, "some content") .to(MagicBranch.NEW_CHANGE + groupRef); r.assertOkStatus(); assertThat(r.getChange().change().getDest().branch()).isEqualTo(groupRef); @@ -1603,7 +1612,7 @@ public class GroupsIT extends AbstractDaemonTest { ThrowingRunnable submit = () -> gApi.changes().id(r.getChangeId()).current().submit(); if (expectedError != null) { Throwable thrown = assertThrows(ResourceConflictException.class, submit); - assertThat(thrown).hasMessageThat().contains("group update not allowed"); + assertThat(thrown).hasMessageThat().contains(expectedError); } else { submit.run(); } diff --git a/javatests/com/google/gerrit/acceptance/api/project/AccessReviewIT.java b/javatests/com/google/gerrit/acceptance/api/project/AccessReviewIT.java new file mode 100644 index 0000000000..553650aa4e --- /dev/null +++ b/javatests/com/google/gerrit/acceptance/api/project/AccessReviewIT.java @@ -0,0 +1,102 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.acceptance.api.project; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.gerrit.truth.ConfigSubject.assertThat; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.gerrit.acceptance.AbstractDaemonTest; +import com.google.gerrit.acceptance.RestResponse; +import com.google.gerrit.acceptance.testsuite.project.ProjectOperations; +import com.google.gerrit.entities.Project; +import com.google.gerrit.extensions.api.access.AccessSectionInfo; +import com.google.gerrit.extensions.api.access.PermissionInfo; +import com.google.gerrit.extensions.api.access.PermissionRuleInfo; +import com.google.gerrit.extensions.api.access.ProjectAccessInput; +import com.google.gerrit.extensions.common.ChangeInfo; +import com.google.gerrit.server.group.SystemGroupBackend; +import com.google.inject.Inject; +import java.util.HashMap; +import java.util.List; +import org.junit.Before; +import org.junit.Test; + +public class AccessReviewIT extends AbstractDaemonTest { + @Inject private ProjectOperations projectOperations; + + private Project.NameKey defaultMessageProject; + private Project.NameKey customMessageProject; + + @Before + public void setUp() throws Exception { + defaultMessageProject = projectOperations.newProject().create(); + customMessageProject = projectOperations.newProject().create(); + } + + @Test + public void createPermissionsChangeWithDefaultMessage() throws Exception { + ProjectAccessInput in = new ProjectAccessInput(); + in.add = new HashMap<>(); + + AccessSectionInfo a = new AccessSectionInfo(); + PermissionInfo p = new PermissionInfo(null, null); + p.rules = + ImmutableMap.of( + SystemGroupBackend.REGISTERED_USERS.get(), + new PermissionRuleInfo(PermissionRuleInfo.Action.ALLOW, false)); + a.permissions = ImmutableMap.of("read", p); + in.add = ImmutableMap.of("refs/heads/*", a); + + RestResponse rep = + adminRestSession.put("/projects/" + defaultMessageProject.get() + "/access:review", in); + rep.assertCreated(); + + List<ChangeInfo> result = + gApi.changes() + .query("project:" + defaultMessageProject.get() + " AND ref:refs/meta/config") + .get(); + assertThat(Iterables.getOnlyElement(result).subject).isEqualTo("Review access change"); + } + + @Test + public void createPermissionsChangeWithCustomMessage() throws Exception { + ProjectAccessInput in = new ProjectAccessInput(); + String customMessage = "UNIT-42: Allow registered users to read 'main' branch"; + in.add = new HashMap<>(); + + AccessSectionInfo a = new AccessSectionInfo(); + PermissionInfo p = new PermissionInfo(null, null); + p.rules = + ImmutableMap.of( + SystemGroupBackend.REGISTERED_USERS.get(), + new PermissionRuleInfo(PermissionRuleInfo.Action.ALLOW, false)); + a.permissions = ImmutableMap.of("read", p); + in.add = ImmutableMap.of("refs/heads/main", a); + in.message = customMessage; + + RestResponse rep = + adminRestSession.put("/projects/" + customMessageProject.get() + "/access:review", in); + rep.assertCreated(); + + List<ChangeInfo> result = + gApi.changes() + .query("project:" + customMessageProject.get() + " AND ref:refs/meta/config") + .get(); + + assertThat(Iterables.getOnlyElement(result).subject).isEqualTo(customMessage); + } +} diff --git a/javatests/com/google/gerrit/acceptance/config/GerritInstanceIdIT.java b/javatests/com/google/gerrit/acceptance/config/GerritInstanceIdIT.java index 0dd6a83e84..d5e935114f 100644 --- a/javatests/com/google/gerrit/acceptance/config/GerritInstanceIdIT.java +++ b/javatests/com/google/gerrit/acceptance/config/GerritInstanceIdIT.java @@ -15,6 +15,7 @@ package com.google.gerrit.acceptance.config; import static com.google.common.truth.Truth.assertThat; +import static com.google.gerrit.server.config.GerritInstanceIdProvider.INSTANCE_ID_SYSTEM_PROPERTY; import com.google.gerrit.acceptance.AbstractDaemonTest; import org.junit.Test; @@ -28,6 +29,13 @@ public class GerritInstanceIdIT extends AbstractDaemonTest { } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") + @GerritSystemProperty(name = INSTANCE_ID_SYSTEM_PROPERTY, value = "sysPropInstanceId") + public void instanceIdSystemPropertyOverridesConfig() { + assertThat(instanceId).isEqualTo("sysPropInstanceId"); + } + + @Test public void shouldReturnNullWhenNotDefined() { assertThat(instanceId).isNull(); } diff --git a/javatests/com/google/gerrit/acceptance/rest/change/AbstractSubmit.java b/javatests/com/google/gerrit/acceptance/rest/change/AbstractSubmit.java index f9fb92c687..585d704ca2 100644 --- a/javatests/com/google/gerrit/acceptance/rest/change/AbstractSubmit.java +++ b/javatests/com/google/gerrit/acceptance/rest/change/AbstractSubmit.java @@ -188,7 +188,7 @@ public abstract class AbstractSubmit extends AbstractDaemonTest { .create( admin.newIdent(), testRepo, - "parent 2", + "parent 1", ImmutableMap.of("foo", "foo-2", "bar", "bar-2")) .to("refs/heads/master"); @@ -207,7 +207,7 @@ public abstract class AbstractSubmit extends AbstractDaemonTest { .create( admin.newIdent(), testRepo, - "parent 1", + "parent 2", ImmutableMap.of("foo", "foo-1", "bar", "bar-1")) .to("refs/heads/stable"); @@ -566,6 +566,25 @@ public abstract class AbstractSubmit extends AbstractDaemonTest { } @Test + public void submitParentIsWorkInProgressChange() throws Throwable { + PushOneCommit.Result parent = pushTo("refs/for/master%wip"); + PushOneCommit.Result change = createChange(); + Change.Id num = parent.getChange().getId(); + if (getSubmitType() == SubmitType.CHERRY_PICK) { + submit(change.getChangeId()); + } else { + submitWithConflict( + change.getChangeId(), + "Failed to submit 2 changes due to the following problems:\n" + + "Change " + + num + + ": Change " + + num + + " is work in progress"); + } + } + + @Test public void submitWithHiddenBranchInSameTopic() throws Throwable { assume().that(isSubmitWholeTopicEnabled()).isTrue(); PushOneCommit.Result visible = createChange("refs/for/master%topic=" + name("topic")); diff --git a/javatests/com/google/gerrit/acceptance/rest/config/ServerInfoIT.java b/javatests/com/google/gerrit/acceptance/rest/config/ServerInfoIT.java index 364ce8447b..b8b63e6ae6 100644 --- a/javatests/com/google/gerrit/acceptance/rest/config/ServerInfoIT.java +++ b/javatests/com/google/gerrit/acceptance/rest/config/ServerInfoIT.java @@ -15,12 +15,14 @@ package com.google.gerrit.acceptance.rest.config; import static com.google.common.truth.Truth.assertThat; +import static com.google.gerrit.server.config.GerritInstanceIdProvider.INSTANCE_ID_SYSTEM_PROPERTY; import static java.nio.charset.StandardCharsets.UTF_8; import com.google.gerrit.acceptance.AbstractDaemonTest; import com.google.gerrit.acceptance.NoHttpd; import com.google.gerrit.acceptance.UseSsh; import com.google.gerrit.acceptance.config.GerritConfig; +import com.google.gerrit.acceptance.config.GerritSystemProperty; import com.google.gerrit.common.RawInputUtil; import com.google.gerrit.extensions.api.plugins.InstallPluginInput; import com.google.gerrit.extensions.client.AccountFieldName; @@ -225,4 +227,11 @@ public class ServerInfoIT extends AbstractDaemonTest { ServerInfo i = gApi.config().server().getInfo(); assertThat(i.download.schemes).isEmpty(); } + + @Test + @GerritSystemProperty(name = INSTANCE_ID_SYSTEM_PROPERTY, value = "sysPropInstanceId") + public void instanceIdFromSystemProperty() throws Exception { + ServerInfo i = gApi.config().server().getInfo(); + assertThat(i.gerrit.instanceId).isEqualTo("sysPropInstanceId"); + } } diff --git a/javatests/com/google/gerrit/acceptance/rest/project/CreateProjectIT.java b/javatests/com/google/gerrit/acceptance/rest/project/CreateProjectIT.java index 5f602500dc..59e23a9f59 100644 --- a/javatests/com/google/gerrit/acceptance/rest/project/CreateProjectIT.java +++ b/javatests/com/google/gerrit/acceptance/rest/project/CreateProjectIT.java @@ -39,6 +39,7 @@ import com.google.gerrit.acceptance.testsuite.request.RequestScopeOperations; import com.google.gerrit.common.data.GlobalCapability; import com.google.gerrit.entities.AccountGroup; import com.google.gerrit.entities.BooleanProjectConfig; +import com.google.gerrit.entities.InternalGroup; import com.google.gerrit.entities.Project; import com.google.gerrit.entities.RefNames; import com.google.gerrit.extensions.api.projects.BranchInfo; @@ -271,13 +272,11 @@ public class CreateProjectIT extends AbstractDaemonTest { in.owners = Lists.newArrayListWithCapacity(3); in.owners.add("Anonymous Users"); // by name in.owners.add(SystemGroupBackend.REGISTERED_USERS.get()); // by UUID - in.owners.add( - Integer.toString( - groupCache - .get(AccountGroup.nameKey("Administrators")) - .orElse(null) - .getId() - .get())); // by ID + Optional<InternalGroup> group = groupCache.get(AccountGroup.nameKey("Administrators")); + if (group.isPresent()) { + in.owners.add(Integer.toString(group.get().getId().get())); // by ID + } + gApi.projects().create(in); Optional<ProjectState> projectState = projectCache.get(Project.nameKey(newProjectName)); Set<AccountGroup.UUID> expectedOwnerIds = Sets.newHashSetWithExpectedSize(3); diff --git a/javatests/com/google/gerrit/acceptance/server/util/WorkQueueIT.java b/javatests/com/google/gerrit/acceptance/server/util/WorkQueueIT.java new file mode 100644 index 0000000000..21a4d96658 --- /dev/null +++ b/javatests/com/google/gerrit/acceptance/server/util/WorkQueueIT.java @@ -0,0 +1,85 @@ +// Copyright (C) 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.acceptance.server.util; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.gerrit.acceptance.AbstractDaemonTest; +import com.google.gerrit.extensions.annotations.Exports; +import com.google.gerrit.server.git.WorkQueue; +import com.google.inject.AbstractModule; +import com.google.inject.Inject; +import com.google.inject.Module; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import org.junit.Test; + +public class WorkQueueIT extends AbstractDaemonTest { + public static class TestListener implements WorkQueue.TaskListener { + + @Override + public void onStart(WorkQueue.Task<?> task) {} + + @Override + public void onStop(WorkQueue.Task<?> task) { + try { + Thread.sleep(FIXED_RATE_SCHEDULE_INTERVAL_MILLI_SEC); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + + private static final Integer FIXED_RATE_SCHEDULE_INITIAL_DELAY = 0; + private static final Integer FIXED_RATE_SCHEDULE_INTERVAL_MILLI_SEC = 1000; + private static final Integer POOL_CORE_SIZE = 8; + private static final String QUEUE_NAME = "test-Queue"; + private static final Integer EXCEPT_RUN_TIMES = 2; + private final CountDownLatch downLatch = new CountDownLatch(EXCEPT_RUN_TIMES); + @Inject private WorkQueue workQueue; + private TestListener testListener; + + @Override + public Module createModule() { + return new AbstractModule() { + @Override + public void configure() { + testListener = new TestListener(); + bind(WorkQueue.TaskListener.class) + .annotatedWith(Exports.named("listener")) + .toInstance(testListener); + } + }; + } + + @Test + public void testScheduleAtFixedRate() throws InterruptedException { + ScheduledExecutorService testExecutor = workQueue.createQueue(POOL_CORE_SIZE, QUEUE_NAME); + ScheduledFuture<?> unusedFuture = + testExecutor.scheduleAtFixedRate( + downLatch::countDown, + FIXED_RATE_SCHEDULE_INITIAL_DELAY, + FIXED_RATE_SCHEDULE_INTERVAL_MILLI_SEC, + TimeUnit.MILLISECONDS); + + boolean ifRunMoreThanOnce = + downLatch.await( + EXCEPT_RUN_TIMES * FIXED_RATE_SCHEDULE_INTERVAL_MILLI_SEC, TimeUnit.MILLISECONDS); + assertThat(ifRunMoreThanOnce).isTrue(); + testExecutor.shutdownNow(); + } +} diff --git a/javatests/com/google/gerrit/acceptance/ssh/CustomIndexIT.java b/javatests/com/google/gerrit/acceptance/ssh/CustomIndexIT.java index 434071ff9c..fee413ab8d 100644 --- a/javatests/com/google/gerrit/acceptance/ssh/CustomIndexIT.java +++ b/javatests/com/google/gerrit/acceptance/ssh/CustomIndexIT.java @@ -14,9 +14,29 @@ package com.google.gerrit.acceptance.ssh; +import static com.google.common.truth.Truth.assertThat; + import com.google.gerrit.index.IndexType; +import com.google.gerrit.index.Schema; +import com.google.gerrit.index.project.ProjectIndex; +import com.google.gerrit.index.testing.AbstractFakeIndex; +import com.google.gerrit.index.testing.FakeIndexVersionManager; +import com.google.gerrit.server.config.GerritServerConfig; +import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.index.AbstractIndexModule; +import com.google.gerrit.server.index.VersionManager; +import com.google.gerrit.server.index.account.AccountIndex; +import com.google.gerrit.server.index.change.ChangeIndex; +import com.google.gerrit.server.index.change.ChangeIndexCollection; +import com.google.gerrit.server.index.group.GroupIndex; +import com.google.gerrit.server.query.change.ChangeData; import com.google.gerrit.testing.ConfigSuite; +import com.google.inject.Inject; +import com.google.inject.Module; +import com.google.inject.assistedinject.Assisted; +import java.util.Map; import org.eclipse.jgit.lib.Config; +import org.junit.Test; /** * Tests for a defaulted custom index configuration. This unknown type is the opposite of {@link @@ -24,10 +44,70 @@ import org.eclipse.jgit.lib.Config; */ public class CustomIndexIT extends AbstractIndexTests { + @Override + public Module createModule() { + return CustomIndexModule.latestVersion(false); + } + @ConfigSuite.Default public static Config customIndexType() { Config config = new Config(); config.setString("index", null, "type", "custom"); return config; } + + @Inject private ChangeIndexCollection changeIndex; + + @Test + public void customIndexModuleIsBound() throws Exception { + assertThat(changeIndex.getSearchIndex()).isInstanceOf(CustomModuleFakeIndexChange.class); + } +} + +class CustomIndexModule extends AbstractIndexModule { + + public static CustomIndexModule latestVersion(boolean secondary) { + return new CustomIndexModule(null, -1 /* direct executor */, secondary); + } + + private CustomIndexModule(Map<String, Integer> singleVersions, int threads, boolean secondary) { + super(singleVersions, threads, secondary); + } + + @Override + protected Class<? extends AccountIndex> getAccountIndex() { + return AbstractFakeIndex.FakeAccountIndex.class; + } + + @Override + protected Class<? extends ChangeIndex> getChangeIndex() { + return CustomModuleFakeIndexChange.class; + } + + @Override + protected Class<? extends GroupIndex> getGroupIndex() { + return AbstractFakeIndex.FakeGroupIndex.class; + } + + @Override + protected Class<? extends ProjectIndex> getProjectIndex() { + return AbstractFakeIndex.FakeProjectIndex.class; + } + + @Override + protected Class<? extends VersionManager> getVersionManager() { + return FakeIndexVersionManager.class; + } +} + +class CustomModuleFakeIndexChange extends AbstractFakeIndex.FakeChangeIndex { + + @com.google.inject.Inject + CustomModuleFakeIndexChange( + SitePaths sitePaths, + ChangeData.Factory changeDataFactory, + @Assisted Schema<ChangeData> schema, + @GerritServerConfig Config cfg) { + super(sitePaths, changeDataFactory, schema, cfg); + } } diff --git a/javatests/com/google/gerrit/acceptance/ssh/QueryIT.java b/javatests/com/google/gerrit/acceptance/ssh/QueryIT.java index 912c4649ea..8f4458d839 100644 --- a/javatests/com/google/gerrit/acceptance/ssh/QueryIT.java +++ b/javatests/com/google/gerrit/acceptance/ssh/QueryIT.java @@ -24,6 +24,8 @@ import com.google.gerrit.acceptance.NoHttpd; import com.google.gerrit.acceptance.PushOneCommit; import com.google.gerrit.acceptance.SshSession; import com.google.gerrit.acceptance.UseSsh; +import com.google.gerrit.entities.LabelId; +import com.google.gerrit.entities.Project; import com.google.gerrit.extensions.annotations.Exports; import com.google.gerrit.extensions.api.changes.ReviewInput; import com.google.gerrit.extensions.api.changes.ReviewerInput; @@ -325,6 +327,33 @@ public class QueryIT extends AbstractDaemonTest { } } + @Test + public void allApprovalsAllPatchSetsOptionsWithCopyConditionJSON() throws Exception { + // Copy min Code-Review votes + try (ProjectConfigUpdate u = updateProject(Project.NameKey.parse("All-Projects"))) { + u.getConfig().updateLabelType(LabelId.CODE_REVIEW, b -> b.setCopyCondition("is:MIN")); + u.save(); + } + + // Create a change and add Code-Review -2 on first patch-set + String changeId = createChange().getChangeId(); + gApi.changes().id(changeId).current().review(ReviewInput.reject()); + + // Create second patch-set + amendChange(changeId); + + // Assert that second patch-set has Code-Review -2 vote + List<ChangeAttribute> changes = + executeSuccessfulQuery("--all-approvals --patch-sets " + changeId); + assertThat(changes).hasSize(1); + assertThat(changes.get(0).patchSets).hasSize(2); + assertThat(changes.get(0).patchSets.get(1).approvals).isNotNull(); + assertThat(changes.get(0).patchSets.get(1).approvals).hasSize(1); + assertThat(changes.get(0).patchSets.get(1).approvals.get(0).type) + .isEqualTo(LabelId.CODE_REVIEW); + assertThat(changes.get(0).patchSets.get(1).approvals.get(0).value).isEqualTo("-2"); + } + protected static class SamplePluginModule extends AbstractModule { @Override public void configure() { diff --git a/javatests/com/google/gerrit/entities/converter/ChangeProtoConverterTest.java b/javatests/com/google/gerrit/entities/converter/ChangeProtoConverterTest.java index bbf10bd6f5..812a0dfa37 100644 --- a/javatests/com/google/gerrit/entities/converter/ChangeProtoConverterTest.java +++ b/javatests/com/google/gerrit/entities/converter/ChangeProtoConverterTest.java @@ -32,6 +32,7 @@ import org.junit.Test; public class ChangeProtoConverterTest { private final ChangeProtoConverter changeProtoConverter = ChangeProtoConverter.INSTANCE; + private static final String TEST_SERVER_ID = "test-server-id"; @Test public void allValuesConvertedToProto() { @@ -42,6 +43,7 @@ public class ChangeProtoConverterTest { Account.id(35), BranchNameKey.create(Project.nameKey("project 67"), "branch 74"), Instant.ofEpochMilli(987654L)); + change.setServerId(TEST_SERVER_ID); change.setLastUpdatedOn(Instant.ofEpochMilli(1234567L)); change.setStatus(Change.Status.MERGED); change.setCurrentPatchSet( @@ -89,6 +91,7 @@ public class ChangeProtoConverterTest { Account.id(35), BranchNameKey.create(Project.nameKey("project 67"), "branch-74"), Instant.ofEpochMilli(987654L)); + change.setServerId(TEST_SERVER_ID); Entities.Change proto = changeProtoConverter.toProto(change); @@ -124,6 +127,7 @@ public class ChangeProtoConverterTest { Account.id(35), BranchNameKey.create(Project.nameKey("project 67"), "branch-74"), Instant.ofEpochMilli(987654L)); + change.setServerId(TEST_SERVER_ID); // O as ID actually means that no current patch set is present. change.setCurrentPatchSet(PatchSet.id(Change.id(14), 0), null, null); @@ -161,6 +165,7 @@ public class ChangeProtoConverterTest { Account.id(35), BranchNameKey.create(Project.nameKey("project 67"), "branch-74"), Instant.ofEpochMilli(987654L)); + change.setServerId(TEST_SERVER_ID); change.setCurrentPatchSet(PatchSet.id(Change.id(14), 23), "subject ABC", null); Entities.Change proto = changeProtoConverter.toProto(change); @@ -189,7 +194,7 @@ public class ChangeProtoConverterTest { } @Test - public void allValuesConvertedToProtoAndBackAgain() { + public void allValuesConvertedToProtoAndBackAgainExceptServerId() { Change change = new Change( Change.key("change 1"), @@ -197,6 +202,7 @@ public class ChangeProtoConverterTest { Account.id(35), BranchNameKey.create(Project.nameKey("project 67"), "branch-74"), Instant.ofEpochMilli(987654L)); + change.setServerId(TEST_SERVER_ID); change.setLastUpdatedOn(Instant.ofEpochMilli(1234567L)); change.setStatus(Change.Status.MERGED); change.setCurrentPatchSet( @@ -209,6 +215,11 @@ public class ChangeProtoConverterTest { change.setRevertOf(Change.id(180)); Change convertedChange = changeProtoConverter.fromProto(changeProtoConverter.toProto(change)); + + // Change serverId is not one of the protobuf definitions, hence is not supposed to be converted + // from proto + assertThat(convertedChange.getServerId()).isNull(); + change.setServerId(null); assertEqualChange(convertedChange, change); } @@ -275,6 +286,7 @@ public class ChangeProtoConverterTest { .hasFields( ImmutableMap.<String, Type>builder() .put("changeId", Change.Id.class) + .put("serverId", String.class) .put("changeKey", Change.Key.class) .put("createdOn", Instant.class) .put("lastUpdatedOn", Instant.class) @@ -298,6 +310,7 @@ public class ChangeProtoConverterTest { // an AutoValue. private static void assertEqualChange(Change change, Change expectedChange) { assertThat(change.getChangeId()).isEqualTo(expectedChange.getChangeId()); + assertThat(change.getServerId()).isEqualTo(expectedChange.getServerId()); assertThat(change.getKey()).isEqualTo(expectedChange.getKey()); assertThat(change.getCreatedOn()).isEqualTo(expectedChange.getCreatedOn()); assertThat(change.getLastUpdatedOn()).isEqualTo(expectedChange.getLastUpdatedOn()); diff --git a/javatests/com/google/gerrit/httpd/auth/container/HttpAuthFilterTest.java b/javatests/com/google/gerrit/httpd/auth/container/HttpAuthFilterTest.java new file mode 100644 index 0000000000..a5f8349570 --- /dev/null +++ b/javatests/com/google/gerrit/httpd/auth/container/HttpAuthFilterTest.java @@ -0,0 +1,78 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.httpd.auth.container; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Mockito.doReturn; + +import com.google.gerrit.extensions.registration.DynamicItem; +import com.google.gerrit.httpd.WebSession; +import com.google.gerrit.server.account.externalids.ExternalIdKeyFactory; +import com.google.gerrit.server.config.AuthConfig; +import com.google.gerrit.util.http.testutil.FakeHttpServletRequest; +import java.io.IOException; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.StrictStubs.class) +public class HttpAuthFilterTest { + + private static String DISPLAYNAME_HEADER = "displaynameHeader"; + private static String DISPLAYNAME = "displayname"; + + @Mock private DynamicItem<WebSession> webSession; + @Mock private ExternalIdKeyFactory externalIdKeyFactory; + @Mock private AuthConfig authConfig; + + @Test + public void getRemoteDisplaynameShouldReturnDisplaynameHeaderWhenHeaderIsConfiguredAndSet() + throws IOException { + doReturn(DISPLAYNAME_HEADER).when(authConfig).getHttpDisplaynameHeader(); + HttpAuthFilter httpAuthFilter = + new HttpAuthFilter(webSession, authConfig, externalIdKeyFactory); + + FakeHttpServletRequest req = new FakeHttpServletRequest(); + req.addHeader(DISPLAYNAME_HEADER, DISPLAYNAME); + + assertThat(httpAuthFilter.getRemoteDisplayname(req)).isEqualTo(DISPLAYNAME); + } + + @Test + public void getRemoteDisplaynameShouldReturnNullWhenDisplaynameHeaderIsConfiguredAndNotSet() + throws IOException { + doReturn(DISPLAYNAME_HEADER).when(authConfig).getHttpDisplaynameHeader(); + HttpAuthFilter httpAuthFilter = + new HttpAuthFilter(webSession, authConfig, externalIdKeyFactory); + + FakeHttpServletRequest req = new FakeHttpServletRequest(); + + assertThat(httpAuthFilter.getRemoteDisplayname(req)).isNull(); + } + + @Test + public void getRemoteDisplaynameShouldReturnNullWhenDisplaynameHeaderIsConfiguredAndEmpty() + throws IOException { + doReturn(DISPLAYNAME_HEADER).when(authConfig).getHttpDisplaynameHeader(); + HttpAuthFilter httpAuthFilter = + new HttpAuthFilter(webSession, authConfig, externalIdKeyFactory); + + FakeHttpServletRequest req = new FakeHttpServletRequest(); + req.addHeader(DISPLAYNAME_HEADER, ""); + + assertThat(httpAuthFilter.getRemoteDisplayname(req)).isNull(); + } +} diff --git a/javatests/com/google/gerrit/metrics/dropwizard/BUILD b/javatests/com/google/gerrit/metrics/dropwizard/BUILD index e236f30ce5..42cf11139e 100644 --- a/javatests/com/google/gerrit/metrics/dropwizard/BUILD +++ b/javatests/com/google/gerrit/metrics/dropwizard/BUILD @@ -8,6 +8,7 @@ junit_tests( deps = [ "//java/com/google/gerrit/metrics", "//java/com/google/gerrit/metrics/dropwizard", + "//java/com/google/gerrit/testing:gerrit-test-util", "//lib/mockito", "//lib/truth", "@dropwizard-core//jar", diff --git a/javatests/com/google/gerrit/metrics/dropwizard/BucketedCallbackTest.java b/javatests/com/google/gerrit/metrics/dropwizard/BucketedCallbackTest.java new file mode 100644 index 0000000000..e87a208e93 --- /dev/null +++ b/javatests/com/google/gerrit/metrics/dropwizard/BucketedCallbackTest.java @@ -0,0 +1,101 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.metrics.dropwizard; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.gerrit.testing.GerritJUnit.assertThrows; + +import com.codahale.metrics.MetricRegistry; +import com.google.gerrit.metrics.Description; +import org.junit.Before; +import org.junit.Test; + +public class BucketedCallbackTest { + + private MetricRegistry registry; + + private DropWizardMetricMaker metrics; + + private static final String CODE_NAME = "name"; + private static final String KEY_NAME = "foo"; + private static final String OTHER_KEY_NAME = "bar"; + private static final String COLLIDING_KEY_NAME1 = "foo1"; + private static final String COLLIDING_KEY_NAME2 = "foo2"; + private static final String COLLIDING_SUBMETRIC_NAME = "foocollision"; + + private String metricName(String fieldValues) { + return CODE_NAME + "/" + fieldValues; + } + + @Before + public void setup() { + registry = new MetricRegistry(); + metrics = new DropWizardMetricMaker(registry, null); + } + + @Test + public void shouldRegisterMetricWithNewKey() { + BucketedCallback<Long> bc = new CallbackMetricTestImpl(); + + bc.getOrCreate(KEY_NAME); + assertThat(registry.getNames()).containsExactly(metricName(KEY_NAME)); + + bc.getOrCreate(OTHER_KEY_NAME); + assertThat(registry.getNames()) + .containsExactly(metricName(KEY_NAME), metricName(OTHER_KEY_NAME)); + } + + @Test + public void shouldNotReRegisterPreviouslyRegisteredMetric() { + BucketedCallback<Long> bc = new CallbackMetricTestImpl(); + bc.getOrCreate(KEY_NAME); + bc.getOrCreate(KEY_NAME); + assertThat(registry.getNames()).containsExactly(metricName(KEY_NAME)); + } + + @Test + public void shouldStoreKeyValueInCellsAndRegisterSubmetricName() { + BucketedCallback<Long> bc = new CallbackMetricTestImpl(); + bc.getOrCreate(COLLIDING_KEY_NAME1); + assertThat(bc.getCells().keySet()).containsExactly(COLLIDING_KEY_NAME1); + assertThat(registry.getNames()).containsExactly(metricName(COLLIDING_SUBMETRIC_NAME)); + } + + @Test + public void shouldErrorIfKeyIsDifferentButNameCollides() { + BucketedCallback<Long> bc = new CallbackMetricTestImpl(); + bc.getOrCreate(COLLIDING_KEY_NAME1); + + assertThrows(IllegalArgumentException.class, () -> bc.getOrCreate(COLLIDING_KEY_NAME2)); + assertThat(bc.getCells().keySet()).containsExactly(COLLIDING_KEY_NAME1); + assertThat(registry.getNames()).containsExactly(metricName(COLLIDING_SUBMETRIC_NAME)); + } + + private class CallbackMetricTestImpl extends BucketedCallback<Long> { + + CallbackMetricTestImpl() { + super(metrics, registry, CODE_NAME, Long.class, new Description("description")); + } + + @Override + String name(Object key) { + if (key.equals(COLLIDING_KEY_NAME1) || key.equals(COLLIDING_KEY_NAME2)) { + return COLLIDING_SUBMETRIC_NAME; + } else { + return key.toString(); + } + } + } +} diff --git a/javatests/com/google/gerrit/server/cache/h2/H2CacheTest.java b/javatests/com/google/gerrit/server/cache/h2/H2CacheTest.java index 16fd4cab69..8c4eb089e8 100644 --- a/javatests/com/google/gerrit/server/cache/h2/H2CacheTest.java +++ b/javatests/com/google/gerrit/server/cache/h2/H2CacheTest.java @@ -71,7 +71,8 @@ public class H2CacheTest { version, 1 << 20, expireAfterWrite, - refreshAfterWrite); + refreshAfterWrite, + true); } @Test diff --git a/javatests/com/google/gerrit/server/events/EventDeserializerTest.java b/javatests/com/google/gerrit/server/events/EventDeserializerTest.java index 390aa844c8..05d6df75c8 100644 --- a/javatests/com/google/gerrit/server/events/EventDeserializerTest.java +++ b/javatests/com/google/gerrit/server/events/EventDeserializerTest.java @@ -227,6 +227,21 @@ public class EventDeserializerTest { } @Test + public void projectHeadUpdatedEvent() { + ProjectHeadUpdatedEvent event = new ProjectHeadUpdatedEvent(); + event.projectName = "test_project"; + event.oldHead = "refs/heads/master"; + event.newHead = "refs/heads/main"; + + ProjectHeadUpdatedEvent actual = roundTrip(event); + + assertThat(actual).isNotNull(); + assertThat(actual.projectName).isEqualTo(event.projectName); + assertThat(actual.oldHead).isEqualTo(event.oldHead); + assertThat(actual.newHead).isEqualTo(event.newHead); + } + + @Test public void shouldSerializeAllProjectsToString() { String allProjectsString = "foobar"; AllProjectsName allProjectsNameKey = new AllProjectsName(allProjectsString); diff --git a/javatests/com/google/gerrit/server/group/db/AbstractGroupTest.java b/javatests/com/google/gerrit/server/group/db/AbstractGroupTest.java index 65eb5b0afb..e0f4b63150 100644 --- a/javatests/com/google/gerrit/server/group/db/AbstractGroupTest.java +++ b/javatests/com/google/gerrit/server/group/db/AbstractGroupTest.java @@ -142,7 +142,7 @@ public class AbstractGroupTest { return GroupConfig.loadForGroup(allUsersName, allUsersRepo, uuid) .getLoadedGroup() .map(InternalGroup::getName) - .orElse("Group " + uuid); + .orElseGet(() -> "Group " + uuid); } catch (IOException | ConfigInvalidException e) { return "Group " + uuid; } diff --git a/javatests/com/google/gerrit/server/index/change/ChangeFieldTest.java b/javatests/com/google/gerrit/server/index/change/ChangeFieldTest.java index 59b354c0b0..4ce4262475 100644 --- a/javatests/com/google/gerrit/server/index/change/ChangeFieldTest.java +++ b/javatests/com/google/gerrit/server/index/change/ChangeFieldTest.java @@ -158,7 +158,7 @@ public class ChangeFieldTest { public void tolerateNullValuesForInsertion() { Project.NameKey project = Project.nameKey("project"); ChangeData cd = - ChangeData.createForTest(project, Change.id(1), 1, ObjectId.zeroId(), null, null, null); + ChangeData.createForTest(project, Change.id(1), 1, ObjectId.zeroId(), null, null); assertThat(ChangeField.ADDED_LINES_SPEC.setIfPossible(cd, new FakeStoredValue(null))).isTrue(); } @@ -166,7 +166,7 @@ public class ChangeFieldTest { public void tolerateNullValuesForDeletion() { Project.NameKey project = Project.nameKey("project"); ChangeData cd = - ChangeData.createForTest(project, Change.id(1), 1, ObjectId.zeroId(), null, null, null); + ChangeData.createForTest(project, Change.id(1), 1, ObjectId.zeroId(), null, null); assertThat(ChangeField.DELETED_LINES_SPEC.setIfPossible(cd, new FakeStoredValue(null))) .isTrue(); } diff --git a/javatests/com/google/gerrit/server/notedb/ImportedChangeNotesTest.java b/javatests/com/google/gerrit/server/notedb/ImportedChangeNotesTest.java index 57be12c982..366cbf736e 100644 --- a/javatests/com/google/gerrit/server/notedb/ImportedChangeNotesTest.java +++ b/javatests/com/google/gerrit/server/notedb/ImportedChangeNotesTest.java @@ -133,6 +133,7 @@ public class ImportedChangeNotesTest extends AbstractChangeNotesTest { assertThat(comments).hasSize(1); HumanComment gotComment = comments.entries().asList().get(0).getValue(); assertThat(gotComment.author.getId()).isEqualTo(otherUser.getAccountId()); + assertThat(gotComment.serverId).isEqualTo(LOCAL_SERVER_ID); } @Test diff --git a/javatests/com/google/gerrit/server/permissions/RefControlTest.java b/javatests/com/google/gerrit/server/permissions/RefControlTest.java index c5bef59e8a..43b0ebabad 100644 --- a/javatests/com/google/gerrit/server/permissions/RefControlTest.java +++ b/javatests/com/google/gerrit/server/permissions/RefControlTest.java @@ -981,6 +981,20 @@ public class RefControlTest { } @Test + public void changeOwnerEditTopicName() throws Exception { + projectOperations + .project(localKey) + .forUpdate() + .add(allow(EDIT_TOPIC_NAME).ref("refs/heads/*").group(CHANGE_OWNER).force(true)) + .update(); + + ProjectControl u = user(localKey, DEVS); + assertWithMessage("u can edit topic name") + .that(u.controlForRef("refs/heads/master").canForceEditTopicName(true)) + .isTrue(); + } + + @Test public void unblockForceEditTopicName() throws Exception { projectOperations .project(localKey) @@ -991,7 +1005,7 @@ public class RefControlTest { ProjectControl u = user(localKey, DEVS); assertWithMessage("u can edit topic name") - .that(u.controlForRef("refs/heads/master").canForceEditTopicName()) + .that(u.controlForRef("refs/heads/master").canForceEditTopicName(false)) .isTrue(); } @@ -1010,7 +1024,7 @@ public class RefControlTest { ProjectControl u = user(localKey, REGISTERED_USERS); assertWithMessage("u can't edit topic name") - .that(u.controlForRef("refs/heads/master").canForceEditTopicName()) + .that(u.controlForRef("refs/heads/master").canForceEditTopicName(false)) .isFalse(); } diff --git a/javatests/com/google/gerrit/server/query/change/AbstractQueryChangesTest.java b/javatests/com/google/gerrit/server/query/change/AbstractQueryChangesTest.java index 2d7ed10b6c..d654e81eb5 100644 --- a/javatests/com/google/gerrit/server/query/change/AbstractQueryChangesTest.java +++ b/javatests/com/google/gerrit/server/query/change/AbstractQueryChangesTest.java @@ -83,6 +83,7 @@ import com.google.gerrit.extensions.api.groups.GroupInput; import com.google.gerrit.extensions.api.projects.ConfigInput; import com.google.gerrit.extensions.api.projects.ProjectInput; import com.google.gerrit.extensions.client.InheritableBoolean; +import com.google.gerrit.extensions.client.ListChangesOption; import com.google.gerrit.extensions.client.ProjectWatchInfo; import com.google.gerrit.extensions.client.ReviewerState; import com.google.gerrit.extensions.common.AccountInfo; @@ -2889,7 +2890,8 @@ public abstract class AbstractQueryChangesTest extends GerritServerTests { } @Test - public void byStar() throws Exception { + public void byStar_withStarOptionSet() throws Exception { + // When star option is set, the 'starred' field is set in the change infos in response. repo = createAndOpenProject("repo"); Change change1 = insert("repo", newChangeWithStatus(repo, Change.Status.MERGED)); @@ -2902,6 +2904,50 @@ public abstract class AbstractQueryChangesTest extends GerritServerTests { // check default star assertQuery("has:star", change1); assertQuery("is:starred", change1); + + // The 'Star' bit in the change data is also set correctly + List<ChangeInfo> changeInfos = + gApi.changes().query("has:star").withOptions(ListChangesOption.STAR).get(); + assertThat(changeInfos.get(0).starred).isTrue(); + } + + @Test + public void byStar_withStarOptionNotSet() throws Exception { + // When star option is not set, the 'starred' field is not set in the change infos in response. + repo = createAndOpenProject("repo"); + Change change1 = insert("repo", newChangeWithStatus(repo, Change.Status.MERGED)); + + Account.Id user2 = + accountManager.authenticate(authRequestFactory.createForUser("anotheruser")).getAccountId(); + requestContext.setContext(newRequestContext(user2)); + + gApi.accounts().self().starChange(change1.getId().toString()); + + // check default star + assertQuery("has:star", change1); + assertQuery("is:starred", change1); + + // The 'Star' bit in the change data is not set if the backfilling option is not set + List<ChangeInfo> changeInfos = gApi.changes().query("has:star").get(); + assertThat(changeInfos.get(0).starred).isNull(); + } + + @Test + public void byStar_withStarOptionSet_notPopulatedForAnonymousUsers() throws Exception { + // Create a random change and star it as some user + repo = createAndOpenProject("repo"); + Change change1 = insert("repo", newChangeWithStatus(repo, Change.Status.NEW)); + Account.Id user2 = + accountManager.authenticate(authRequestFactory.createForUser("anotheruser")).getAccountId(); + requestContext.setContext(newRequestContext(user2)); + gApi.accounts().self().starChange(change1.getId().toString()); + + // Request a change query for all open changes. The star field is not set on the single change. + requestContext.setContext(anonymousUserProvider::get); + List<ChangeInfo> changeInfos = + gApi.changes().query("is:open").withOptions(ListChangesOption.STAR).get(); + assertThat(changeInfos.get(0)._number).isEqualTo(change1.getId().get()); + assertThat(changeInfos.get(0).starred).isNull(); } @Test @@ -3402,6 +3448,14 @@ public abstract class AbstractQueryChangesTest extends GerritServerTests { assertQuery("reviewer:self", change); assertThat(indexer.reindexIfStale(project, change.getId()).get()).isTrue(); assertQuery("reviewer:self"); + + // Index is not stale when a draft comment exists + DraftInput in = new DraftInput(); + in.line = 1; + in.message = "nit: trailing whitespace"; + in.path = Patch.COMMIT_MSG; + gApi.changes().id(project.get(), change.getId().get()).current().createDraft(in); + assertThat(indexer.reindexIfStale(project, change.getId()).get()).isFalse(); } @Test diff --git a/javatests/com/google/gerrit/server/query/change/ChangeDataTest.java b/javatests/com/google/gerrit/server/query/change/ChangeDataTest.java index 0ce00ebfc7..f954a573eb 100644 --- a/javatests/com/google/gerrit/server/query/change/ChangeDataTest.java +++ b/javatests/com/google/gerrit/server/query/change/ChangeDataTest.java @@ -34,8 +34,6 @@ import org.mockito.junit.MockitoJUnitRunner; @RunWith(MockitoJUnitRunner.class) public class ChangeDataTest { - private static final String GERRIT_SERVER_ID = UUID.randomUUID().toString(); - @Mock private ChangeNotes changeNotesMock; @Test @@ -55,7 +53,7 @@ public class ChangeDataTest { @Test public void getChangeVirtualIdUsingAlgorithm() throws Exception { Project.NameKey project = Project.nameKey("project"); - final int encodedChangeNum = 12345678; + final Change.Id encodedChangeNum = Change.id(12345678); when(changeNotesMock.getServerId()).thenReturn(UUID.randomUUID().toString()); @@ -65,11 +63,10 @@ public class ChangeDataTest { Change.id(1), 1, ObjectId.zeroId(), - GERRIT_SERVER_ID, (s, c) -> encodedChangeNum, changeNotesMock); - assertThat(cd.getVirtualId().get()).isEqualTo(encodedChangeNum); + assertThat(cd.virtualId().get()).isEqualTo(encodedChangeNum.get()); } private static PatchSet newPatchSet(Change.Id changeId, int num) { diff --git a/javatests/com/google/gerrit/server/query/change/FakeQueryChangesTest.java b/javatests/com/google/gerrit/server/query/change/FakeQueryChangesTest.java index 72fc6d270d..7641544d56 100644 --- a/javatests/com/google/gerrit/server/query/change/FakeQueryChangesTest.java +++ b/javatests/com/google/gerrit/server/query/change/FakeQueryChangesTest.java @@ -25,6 +25,8 @@ import static org.junit.Assume.assumeFalse; import static org.junit.Assume.assumeTrue; import com.google.gerrit.acceptance.UseClockStep; +import com.google.gerrit.entities.Account; +import com.google.gerrit.entities.Change; import com.google.gerrit.entities.Permission; import com.google.gerrit.entities.Project; import com.google.gerrit.extensions.common.ChangeInfo; @@ -84,11 +86,48 @@ public abstract class FakeQueryChangesTest extends AbstractQueryChangesTest { AbstractFakeIndex<?, ?, ?> idx = (AbstractFakeIndex<?, ?, ?>) changeIndexCollection.getSearchIndex(); newQuery("status:new").withLimit(5).get(); + // Since the limit of the query (i.e. 5) is more than the total number of changes (i.e. 4), + // only 1 index search is expected. assertThatSearchQueryWasNotPaginated(idx.getQueryCount()); } @Test @UseClockStep + public void queryRightNumberOfTimes() throws Exception { + TestRepository<Repository> repo = createAndOpenProject("repo"); + Account.Id user2 = + accountManager.authenticate(authRequestFactory.createForUser("anotheruser")).getAccountId(); + + // create 1 visible change + Change visibleChange1 = insert("repo", newChangeWithStatus(repo, Change.Status.NEW)); + + // create 4 private changes + Change invisibleChange2 = insert("repo", newChangeWithStatus(repo, Change.Status.NEW), user2); + Change invisibleChange3 = insert("repo", newChangeWithStatus(repo, Change.Status.NEW), user2); + Change invisibleChange4 = insert("repo", newChangeWithStatus(repo, Change.Status.NEW), user2); + Change invisibleChange5 = insert("repo", newChangeWithStatus(repo, Change.Status.NEW), user2); + gApi.changes().id(invisibleChange2.getChangeId()).setPrivate(true, null); + gApi.changes().id(invisibleChange3.getChangeId()).setPrivate(true, null); + gApi.changes().id(invisibleChange4.getChangeId()).setPrivate(true, null); + gApi.changes().id(invisibleChange5.getChangeId()).setPrivate(true, null); + + AbstractFakeIndex<?, ?, ?> idx = + (AbstractFakeIndex<?, ?, ?>) changeIndexCollection.getSearchIndex(); + idx.resetQueryCount(); + List<ChangeInfo> queryResult = newQuery("status:new").withLimit(2).get(); + assertThat(queryResult).hasSize(1); + assertThat(queryResult.get(0).changeId).isEqualTo(visibleChange1.getKey().get()); + + // Since the limit of the query (i.e. 2), 2 index searches are expected in fact: + // 1: The first query will return invisibleChange5, invisibleChange4 and invisibleChange3, + // 2: Another query is needed to back-fill the limit requested by the user. + // even if one result in the second query is skipped because it is not visible, + // there are no more results to query. + assertThatSearchQueryWasPaginated(idx.getQueryCount(), 2); + } + + @Test + @UseClockStep public void noLimitQueryPaginates() throws Exception { assumeFalse(PaginationType.NONE == getCurrentPaginationType()); @@ -119,45 +158,15 @@ public abstract class FakeQueryChangesTest extends AbstractQueryChangesTest { @UseClockStep public void noLimitQueryDoesNotPaginatesWithNonePaginationType() throws Exception { assumeTrue(PaginationType.NONE == getCurrentPaginationType()); - AbstractFakeIndex idx = setupRepoWithFourChanges(); + AbstractFakeIndex<?, ?, ?> idx = setupRepoWithFourChanges(); newQuery("status:new").withNoLimit().get(); assertThatSearchQueryWasNotPaginated(idx.getQueryCount()); } @Test @UseClockStep - public void invisibleChangesNotPaginatedWithNonePaginationType() throws Exception { - assumeTrue(PaginationType.NONE == getCurrentPaginationType()); - AbstractFakeIndex idx = setupRepoWithFourChanges(); - final int LIMIT = 3; - - projectOperations - .project(allProjectsName) - .forUpdate() - .removeAllAccessSections() - .add(allow(Permission.READ).ref("refs/*").group(SystemGroupBackend.REGISTERED_USERS)) - .update(); - - // Set queryLimit to 3 - projectOperations - .project(allProjects) - .forUpdate() - .add(allowCapability(QUERY_LIMIT).group(ANONYMOUS_USERS).range(0, LIMIT)) - .update(); - - requestContext.setContext(anonymousUserProvider::get); - List<ChangeInfo> result = newQuery("status:new").withLimit(LIMIT).get(); - assertThat(result.size()).isEqualTo(0); - assertThatSearchQueryWasNotPaginated(idx.getQueryCount()); - assertThat(idx.getResultsSizes().get(0)).isEqualTo(LIMIT + 1); - } - - @Test - @UseClockStep public void invisibleChangesPaginatedWithPagination() throws Exception { - assumeFalse(PaginationType.NONE == getCurrentPaginationType()); - - AbstractFakeIndex idx = setupRepoWithFourChanges(); + AbstractFakeIndex<?, ?, ?> idx = setupRepoWithFourChanges(); final int LIMIT = 3; projectOperations @@ -199,7 +208,8 @@ public abstract class FakeQueryChangesTest extends AbstractQueryChangesTest { .forUpdate() .add(allowCapability(QUERY_LIMIT).group(REGISTERED_USERS).range(0, LIMIT)) .update(); - AbstractFakeIndex idx = (AbstractFakeIndex) changeIndexCollection.getSearchIndex(); + AbstractFakeIndex<?, ?, ?> idx = + (AbstractFakeIndex<?, ?, ?>) changeIndexCollection.getSearchIndex(); // 2 index searches are expected. The first index search will run with size 3 (i.e. // the configured query-limit+1), and then we will paginate to get the remaining // changes with the second index search. @@ -213,7 +223,7 @@ public abstract class FakeQueryChangesTest extends AbstractQueryChangesTest { public void internalQueriesDoNotPaginateWithNonePaginationType() throws Exception { assumeTrue(PaginationType.NONE == getCurrentPaginationType()); - AbstractFakeIndex idx = setupRepoWithFourChanges(); + AbstractFakeIndex<?, ?, ?> idx = setupRepoWithFourChanges(); // 1 index search is expected since we are not paginating. executeQuery("status:new"); assertThatSearchQueryWasNotPaginated(idx.getQueryCount()); @@ -231,7 +241,7 @@ public abstract class FakeQueryChangesTest extends AbstractQueryChangesTest { assertThat(queryCount).isEqualTo(expectedPages); } - private AbstractFakeIndex setupRepoWithFourChanges() throws Exception { + private AbstractFakeIndex<?, ?, ?> setupRepoWithFourChanges() throws Exception { try (TestRepository<Repository> testRepo = createAndOpenProject("repo")) { insert("repo", newChange(testRepo)); insert("repo", newChange(testRepo)); @@ -246,6 +256,6 @@ public abstract class FakeQueryChangesTest extends AbstractQueryChangesTest { .add(allowCapability(QUERY_LIMIT).group(REGISTERED_USERS).range(0, 2)) .update(); - return (AbstractFakeIndex) changeIndexCollection.getSearchIndex(); + return (AbstractFakeIndex<?, ?, ?>) changeIndexCollection.getSearchIndex(); } } diff --git a/javatests/com/google/gerrit/server/query/change/LuceneQueryChangesTest.java b/javatests/com/google/gerrit/server/query/change/LuceneQueryChangesTest.java index 7f383f948b..d9a0767df7 100644 --- a/javatests/com/google/gerrit/server/query/change/LuceneQueryChangesTest.java +++ b/javatests/com/google/gerrit/server/query/change/LuceneQueryChangesTest.java @@ -20,6 +20,7 @@ import static com.google.gerrit.common.data.GlobalCapability.QUERY_LIMIT; import static com.google.gerrit.server.group.SystemGroupBackend.REGISTERED_USERS; import static com.google.gerrit.testing.GerritJUnit.assertThrows; +import com.google.gerrit.entities.Account; import com.google.gerrit.entities.Change; import com.google.gerrit.extensions.restapi.BadRequestException; import com.google.gerrit.server.config.AllProjectsName; @@ -89,4 +90,27 @@ public abstract class LuceneQueryChangesTest extends AbstractQueryChangesTest { Change[] expected = new Change[] {change6, change5, change4, change3, change2, change1}; assertQuery(newQuery("project:repo").withNoLimit(), expected); } + + @Test + public void skipChangesNotVisible() throws Exception { + // create 1 new change on a repo + repo = createAndOpenProject("repo"); + Change visibleChange = insert("repo", newChangeWithStatus(repo, Change.Status.NEW)); + Change[] expected = new Change[] {visibleChange}; + + // pagination does not need to restart the datasource, the request is fulfilled + assertQuery(newQuery("status:new").withLimit(1), expected); + + // create 2 new private changes + Account.Id user2 = + accountManager.authenticate(authRequestFactory.createForUser("anotheruser")).getAccountId(); + + Change invisibleChange1 = insert("repo", newChangeWithStatus(repo, Change.Status.NEW), user2); + Change invisibleChange2 = insert("repo", newChangeWithStatus(repo, Change.Status.NEW), user2); + gApi.changes().id(invisibleChange1.getChangeId()).setPrivate(true, null); + gApi.changes().id(invisibleChange2.getChangeId()).setPrivate(true, null); + + // pagination should back-fill when the results skipped because of the visibility + assertQuery(newQuery("status:new").withLimit(1), expected); + } } diff --git a/javatests/com/google/gerrit/sshd/BUILD b/javatests/com/google/gerrit/sshd/BUILD index 3e11ff22e2..44b9c62a79 100644 --- a/javatests/com/google/gerrit/sshd/BUILD +++ b/javatests/com/google/gerrit/sshd/BUILD @@ -4,8 +4,11 @@ junit_tests( name = "sshd_tests", srcs = glob(["**/*.java"]), deps = [ + "//java/com/google/gerrit/entities", "//java/com/google/gerrit/extensions:api", + "//java/com/google/gerrit/server", "//java/com/google/gerrit/sshd", + "//java/com/google/gerrit/testing:gerrit-test-util", "//lib/mina:sshd", "//lib/truth", ], diff --git a/javatests/com/google/gerrit/sshd/SshUtilTest.java b/javatests/com/google/gerrit/sshd/SshUtilTest.java new file mode 100644 index 0000000000..1585bc3a97 --- /dev/null +++ b/javatests/com/google/gerrit/sshd/SshUtilTest.java @@ -0,0 +1,49 @@ +// Copyright (C) 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.sshd; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.gerrit.testing.GerritJUnit.assertThrows; + +import com.google.gerrit.entities.Account; +import com.google.gerrit.server.account.AccountSshKey; +import java.security.spec.InvalidKeySpecException; +import org.junit.Test; + +public class SshUtilTest { + private static final Account.Id TEST_ACCOUNT_ID = Account.id(1); + private static final int TEST_SSHKEY_SEQUENCE = 1; + private static final String INVALID_ALGO = "invalid-algo"; + private static final String VALID_OPENSSH_RSA_KEY = + "AAAAB3NzaC1yc2EAAAABIwAAAIEA0R66EoZ7hFp81w9sAJqu34UFyE+w36H/mobUqnT5Lns7PcTOJh3sgMJAlswX2lFAWqvF2gd2PRMpMhbfEU4iq2SfY8x+RDCJ4ZQWESln/587T41BlQjOXzu3W1bqgmtHnRCte3DjyWDvM/fucnUMSwOgP+FVEZCLTrk3thLMWsU="; + private static final Object VALID_SSH_RSA_ALGO = "ssh-rsa"; + + @Test + public void shouldFailParsingOpenSshKeyWithInvalidAlgo() { + String sshKeyWithInvalidAlgo = String.format("%s %s", INVALID_ALGO, VALID_OPENSSH_RSA_KEY); + AccountSshKey sshKey = + AccountSshKey.create(TEST_ACCOUNT_ID, TEST_SSHKEY_SEQUENCE, sshKeyWithInvalidAlgo); + assertThrows(InvalidKeySpecException.class, () -> SshUtil.parse(sshKey)); + } + + @Test + public void shouldParseSshKeyWithAlgoMatchingKey() { + String sshKeyWithValidKeyAlgo = + String.format("%s %s", VALID_SSH_RSA_ALGO, VALID_OPENSSH_RSA_KEY); + AccountSshKey sshKey = + AccountSshKey.create(TEST_ACCOUNT_ID, TEST_SSHKEY_SEQUENCE, sshKeyWithValidKeyAlgo); + assertThat(sshKey).isNotNull(); + } +} diff --git a/modules/jgit b/modules/jgit -Subproject 74fa245b3c3ccf13afcbec7911c7c8459e48527 +Subproject c0b415fb028b4c1f29b6df749323bbb11599495 diff --git a/plugins/delete-project b/plugins/delete-project -Subproject b080ed4630104cee0078f6be3561600ed1c3647 +Subproject 9378a0e55daf9e24b8863a2605e6a1f1828f73a diff --git a/polygerrit-ui/app/api/checks.ts b/polygerrit-ui/app/api/checks.ts index 98aaa9cca7..f66c3739e6 100644 --- a/polygerrit-ui/app/api/checks.ts +++ b/polygerrit-ui/app/api/checks.ts @@ -375,9 +375,11 @@ export declare interface CheckResult { * responsible for not killing the browser. :-) * * For now this is just a plain unformatted string. The only formatting - * applied is the one that Gerrit also applies to human comments. TBD: Both - * human comments and check result messages should get richer formatting - * options. + * applied is the one that Gerrit also applies to human comments. + * + * To provide richer formatting to the check result messages you should use + * the `check-result-expanded` plugin endpoint to attach a Web Component. + * See `Documentation/pg-plugin-endpoints.txt`. */ message?: string; diff --git a/polygerrit-ui/app/api/rest-api.ts b/polygerrit-ui/app/api/rest-api.ts index 244002e0bd..993f24dc8c 100644 --- a/polygerrit-ui/app/api/rest-api.ts +++ b/polygerrit-ui/app/api/rest-api.ts @@ -987,7 +987,7 @@ export declare interface RevisionInfo { commit?: CommitInfo; files?: {[filename: string]: FileInfo}; reviewed?: boolean; - commit_with_footers?: boolean; + commit_with_footers?: string; push_certificate?: PushCertificateInfo; description?: string; basePatchNum?: BasePatchSetNum; diff --git a/polygerrit-ui/app/elements/admin/gr-admin-view/gr-admin-view.ts b/polygerrit-ui/app/elements/admin/gr-admin-view/gr-admin-view.ts index 04887adbe0..21e032b816 100644 --- a/polygerrit-ui/app/elements/admin/gr-admin-view/gr-admin-view.ts +++ b/polygerrit-ui/app/elements/admin/gr-admin-view/gr-admin-view.ts @@ -17,7 +17,6 @@ import '../gr-repo-commands/gr-repo-commands'; import '../gr-repo-dashboards/gr-repo-dashboards'; import '../gr-repo-detail-list/gr-repo-detail-list'; import '../gr-repo-list/gr-repo-list'; -import {getBaseUrl} from '../../../utils/url-util'; import {navigationToken} from '../../core/gr-navigation/gr-navigation'; import { AccountDetailInfo, @@ -600,12 +599,7 @@ export class GrAdminView extends LitElement { // private but used in test computeLinkURL(link?: NavLink | SubsectionInterface) { - if (!link || typeof link.url === 'undefined') return ''; - - if ((link as NavLink).target || !(link as NavLink).noBaseUrl) { - return link.url; - } - return `//${window.location.host}${getBaseUrl()}${link.url}`; + return link?.url || ''; } private computeSelectedClass( diff --git a/polygerrit-ui/app/elements/admin/gr-admin-view/gr-admin-view_test.ts b/polygerrit-ui/app/elements/admin/gr-admin-view/gr-admin-view_test.ts index 8cf4e057d5..d184f35550 100644 --- a/polygerrit-ui/app/elements/admin/gr-admin-view/gr-admin-view_test.ts +++ b/polygerrit-ui/app/elements/admin/gr-admin-view/gr-admin-view_test.ts @@ -6,7 +6,7 @@ import '../../../test/common-test-setup'; import './gr-admin-view'; import {AdminSubsectionLink, GrAdminView} from './gr-admin-view'; -import {stubBaseUrl, stubElement, stubRestApi} from '../../../test/test-utils'; +import {stubElement, stubRestApi} from '../../../test/test-utils'; import {GerritView} from '../../../services/router/router-model'; import {query, queryAll, queryAndAssert} from '../../../test/test-utils'; import {GrRepoList} from '../gr-repo-list/gr-repo-list'; @@ -47,29 +47,9 @@ suite('gr-admin-view tests', () => { }); test('link URLs', () => { - assert.equal( - element.computeLinkURL({name: '', url: '/test', noBaseUrl: true}), - '//' + window.location.host + '/test' - ); - - stubBaseUrl('/foo'); - assert.equal( - element.computeLinkURL({name: '', url: '/test', noBaseUrl: true}), - '//' + window.location.host + '/foo/test' - ); - assert.equal( - element.computeLinkURL({name: '', url: '/test', noBaseUrl: false}), - '/test' - ); - assert.equal( - element.computeLinkURL({ - name: '', - url: '/test', - target: '_blank', - noBaseUrl: false, - }), - '/test' - ); + assert.equal(element.computeLinkURL({name: '', url: '/test'}), '/test'); + assert.equal(element.computeLinkURL({name: '', url: ''}), ''); + assert.equal(element.computeLinkURL(undefined), ''); }); test('current page gets selected and is displayed', async () => { @@ -78,7 +58,6 @@ suite('gr-admin-view tests', () => { name: 'Repositories', url: '/admin/repos', view: 'gr-repo-list' as GerritView, - noBaseUrl: false, }, ]; @@ -154,7 +133,6 @@ suite('gr-admin-view tests', () => { capability: undefined, url: '/internal/link/url', name: 'internal link text', - noBaseUrl: true, view: undefined, viewableToAll: true, target: null, @@ -163,7 +141,6 @@ suite('gr-admin-view tests', () => { capability: undefined, url: 'http://external/link/url', name: 'external link text', - noBaseUrl: false, view: undefined, viewableToAll: true, target: '_blank', @@ -349,7 +326,6 @@ suite('gr-admin-view tests', () => { const expectedFilteredLinks = [ { name: 'Repositories', - noBaseUrl: true, url: '/admin/repos', view: 'gr-repo-list' as GerritView, viewableToAll: true, @@ -399,7 +375,6 @@ suite('gr-admin-view tests', () => { { name: 'Groups', section: 'Groups', - noBaseUrl: true, url: '/admin/groups', view: 'gr-admin-group-list' as GerritView, }, @@ -407,7 +382,6 @@ suite('gr-admin-view tests', () => { name: 'Plugins', capability: 'viewPlugins', section: 'Plugins', - noBaseUrl: true, url: '/admin/plugins', view: 'gr-plugin-list' as GerritView, }, @@ -544,29 +518,17 @@ suite('gr-admin-view tests', () => { <gr-page-nav class="navStyles"> <ul class="sectionContent"> <li class="sectionTitle selected"> - <a - class="title" - href="//localhost:9876/admin/repos" - rel="noopener" - > + <a class="title" href="/admin/repos" rel="noopener"> Repositories </a> </li> <li class="sectionTitle"> - <a - class="title" - href="//localhost:9876/admin/groups" - rel="noopener" - > + <a class="title" href="/admin/groups" rel="noopener"> Groups </a> </li> <li class="sectionTitle"> - <a - class="title" - href="//localhost:9876/admin/plugins" - rel="noopener" - > + <a class="title" href="/admin/plugins" rel="noopener"> Plugins </a> </li> diff --git a/polygerrit-ui/app/elements/admin/gr-plugin-list/gr-plugin-list_test.ts b/polygerrit-ui/app/elements/admin/gr-plugin-list/gr-plugin-list_test.ts index fa1a6a8372..7c41120618 100644 --- a/polygerrit-ui/app/elements/admin/gr-plugin-list/gr-plugin-list_test.ts +++ b/polygerrit-ui/app/elements/admin/gr-plugin-list/gr-plugin-list_test.ts @@ -339,7 +339,7 @@ suite('gr-plugin-list tests', () => { }); }); - suite('list with less then 26 plugins', () => { + suite('list with less than 26 plugins', () => { setup(async () => { plugins = createPluginObjectList(25); stubRestApi('getPlugins').returns(Promise.resolve(plugins)); diff --git a/polygerrit-ui/app/elements/admin/gr-repo-detail-list/gr-repo-detail-list_test.ts b/polygerrit-ui/app/elements/admin/gr-repo-detail-list/gr-repo-detail-list_test.ts index 391a22aeab..5e25b33ab6 100644 --- a/polygerrit-ui/app/elements/admin/gr-repo-detail-list/gr-repo-detail-list_test.ts +++ b/polygerrit-ui/app/elements/admin/gr-repo-detail-list/gr-repo-detail-list_test.ts @@ -2089,7 +2089,7 @@ suite('gr-repo-detail-list', () => { }); }); - suite('list with less then 25 branches', () => { + suite('list with less than 25 branches', () => { setup(async () => { branches = createBranchesList(25); stubRestApi('getRepoBranches').returns(Promise.resolve(branches)); @@ -2226,7 +2226,7 @@ suite('gr-repo-detail-list', () => { }); }); - suite('list with less then 25 tags', () => { + suite('list with less than 25 tags', () => { setup(async () => { tags = createTagsList(25); stubRestApi('getRepoTags').returns(Promise.resolve(tags)); diff --git a/polygerrit-ui/app/elements/admin/gr-repo-list/gr-repo-list_test.ts b/polygerrit-ui/app/elements/admin/gr-repo-list/gr-repo-list_test.ts index b80db4c349..36674f5081 100644 --- a/polygerrit-ui/app/elements/admin/gr-repo-list/gr-repo-list_test.ts +++ b/polygerrit-ui/app/elements/admin/gr-repo-list/gr-repo-list_test.ts @@ -464,7 +464,7 @@ suite('gr-repo-list tests', () => { }); }); - suite('list with less then 25 repos', () => { + suite('list with less than 25 repos', () => { setup(async () => { repos = createRepoList('test', 25); stubRestApi('getRepos').returns(Promise.resolve(repos)); diff --git a/polygerrit-ui/app/elements/change-list/gr-user-header/gr-user-header.ts b/polygerrit-ui/app/elements/change-list/gr-user-header/gr-user-header.ts index 57f9ee694a..3f9941624c 100644 --- a/polygerrit-ui/app/elements/change-list/gr-user-header/gr-user-header.ts +++ b/polygerrit-ui/app/elements/change-list/gr-user-header/gr-user-header.ts @@ -117,10 +117,12 @@ export class GrUserHeader extends LitElement { return; } - this.restApiService.getAccountDetails(userId).then(details => { - this._accountDetails = details ?? undefined; - this._status = details?.status ?? ''; - }); + this.restApiService + .getAccountDetails(userId, () => {}) + .then(details => { + this._accountDetails = details ?? undefined; + this._status = details?.status ?? ''; + }); } _computeDetail( diff --git a/polygerrit-ui/app/elements/change/gr-change-actions/gr-change-actions.ts b/polygerrit-ui/app/elements/change/gr-change-actions/gr-change-actions.ts index 6ca8b01f93..18fcce110d 100644 --- a/polygerrit-ui/app/elements/change/gr-change-actions/gr-change-actions.ts +++ b/polygerrit-ui/app/elements/change/gr-change-actions/gr-change-actions.ts @@ -1004,23 +1004,17 @@ export class GrChangeActions } private actionsChanged() { - this.hidden = - Object.keys(this.actions).length === 0 && - Object.keys(this.revisionActions).length === 0 && - this.additionalActions.length === 0; this.actionLoadingMessage = ''; this.disabledMenuActions = []; - if (Object.keys(this.revisionActions).length !== 0) { - if (!this.revisionActions.download) { - this.revisionActions = { - ...this.revisionActions, - download: DOWNLOAD_ACTION, - }; - fire(this, 'revision-actions-changed', { - value: this.revisionActions, - }); - } + if (!this.revisionActions.download) { + this.revisionActions = { + ...this.revisionActions, + download: DOWNLOAD_ACTION, + }; + fire(this, 'revision-actions-changed', { + value: this.revisionActions, + }); } if ( !this.actions.includedIn && diff --git a/polygerrit-ui/app/elements/change/gr-change-view/gr-change-view.ts b/polygerrit-ui/app/elements/change/gr-change-view/gr-change-view.ts index abd3ff0798..fc8bcb91d1 100644 --- a/polygerrit-ui/app/elements/change/gr-change-view/gr-change-view.ts +++ b/polygerrit-ui/app/elements/change/gr-change-view/gr-change-view.ts @@ -140,7 +140,7 @@ import {ShortcutController} from '../../lit/shortcut-controller'; import {FilesExpandedState} from '../gr-file-list-constants'; import {subscribe} from '../../lit/subscription-controller'; import {configModelToken} from '../../../models/config/config-model'; -import {getBaseUrl, prependOrigin} from '../../../utils/url-util'; +import {prependOrigin} from '../../../utils/url-util'; import {CopyLink, GrCopyLinks} from '../gr-copy-links/gr-copy-links'; import { ChangeChildView, @@ -1240,7 +1240,7 @@ export class GrChangeView extends LitElement { private renderCopyLinksDropdown() { const url = this.computeChangeUrl(); if (!url) return; - const changeURL = prependOrigin(getBaseUrl() + url); + const changeURL = prependOrigin(url); const links: CopyLink[] = [ { label: 'Change Number', diff --git a/polygerrit-ui/app/elements/change/gr-change-view/gr-change-view_test.ts b/polygerrit-ui/app/elements/change/gr-change-view/gr-change-view_test.ts index 533155821e..560006be36 100644 --- a/polygerrit-ui/app/elements/change/gr-change-view/gr-change-view_test.ts +++ b/polygerrit-ui/app/elements/change/gr-change-view/gr-change-view_test.ts @@ -21,6 +21,7 @@ import { mockPromise, pressKey, queryAndAssert, + stubBaseUrl, stubFlags, stubRestApi, waitUntil, @@ -390,7 +391,7 @@ suite('gr-change-view tests', () => { </gr-copy-clipboard> </div> <div class="commitActions"> - <gr-change-actions hidden="" id="actions"> </gr-change-actions> + <gr-change-actions id="actions"> </gr-change-actions> </div> </div> <h2 class="assistive-tech-only">Change metadata</h2> @@ -1649,4 +1650,36 @@ suite('gr-change-view tests', () => { copyLinksDialog.copyLinks.some(copyLink => copyLink.value === sha) ); }); + + test('copy links without a base URL', async () => { + element.change = createChangeViewChange(); + await element.updateComplete; + + const copyLinksDialog = queryAndAssert<GrCopyLinks>( + element, + 'gr-copy-links' + ); + assert.deepEqual(copyLinksDialog.copyLinks[1], { + label: 'Change URL', + shortcut: 'u', + value: 'http://localhost:9876/c/test-project/+/42', + }); + }); + + test('copy links with a base URL having a path', async () => { + stubBaseUrl('/review'); + element.change = createChangeViewChange(); + await element.updateComplete; + + const copyLinksDialog = queryAndAssert<GrCopyLinks>( + element, + 'gr-copy-links' + ); + + assert.deepEqual(copyLinksDialog.copyLinks[1], { + label: 'Change URL', + shortcut: 'u', + value: 'http://localhost:9876/review/c/test-project/+/42', + }); + }); }); diff --git a/polygerrit-ui/app/elements/checks/gr-checks-results.ts b/polygerrit-ui/app/elements/checks/gr-checks-results.ts index 65165c15f9..1ff76888e8 100644 --- a/polygerrit-ui/app/elements/checks/gr-checks-results.ts +++ b/polygerrit-ui/app/elements/checks/gr-checks-results.ts @@ -501,7 +501,7 @@ export class GrResultRow extends LitElement { return html` <div class="label ${status}"> <span>${label} ${valueStr}</span> - <paper-tooltip offset="5" ?fitToVisibleBounds=${true}> + <paper-tooltip offset="5" .fitToVisibleBounds=${true}> The check result has (probably) influenced this label vote. </paper-tooltip> </div> @@ -611,7 +611,7 @@ export class GrResultRow extends LitElement { @click=${(e: MouseEvent) => this.tagClick(e, tag.name)} > <span>${tag.name}</span> - <paper-tooltip offset="5" ?fitToVisibleBounds=${true}> + <paper-tooltip offset="5" .fitToVisibleBounds=${true}> ${tag.tooltip ?? 'A category tag for this check result. Click to filter.'} </paper-tooltip> diff --git a/polygerrit-ui/app/elements/checks/gr-checks-results_test.ts b/polygerrit-ui/app/elements/checks/gr-checks-results_test.ts index 385bde7db5..8affccbd43 100644 --- a/polygerrit-ui/app/elements/checks/gr-checks-results_test.ts +++ b/polygerrit-ui/app/elements/checks/gr-checks-results_test.ts @@ -45,12 +45,7 @@ suite('gr-result-row test', () => { /* HTML */ ` <div class="approved label"> <span> test-label +1 </span> - <paper-tooltip - fittovisiblebounds="" - offset="5" - role="tooltip" - tabindex="-1" - > + <paper-tooltip offset="5" role="tooltip" tabindex="-1"> The check result has (probably) influenced this label vote. </paper-tooltip> </div> @@ -92,7 +87,6 @@ suite('gr-result-row test', () => { <button class="tag"> <span> OBSOLETE </span> <paper-tooltip - fittovisiblebounds="" offset="5" role="tooltip" tabindex="-1" @@ -103,7 +97,6 @@ suite('gr-result-row test', () => { <button class="tag"> <span> E2E </span> <paper-tooltip - fittovisiblebounds="" offset="5" role="tooltip" tabindex="-1" diff --git a/polygerrit-ui/app/elements/core/gr-main-header/gr-main-header.ts b/polygerrit-ui/app/elements/core/gr-main-header/gr-main-header.ts index 4fee3952f8..2c6de04729 100644 --- a/polygerrit-ui/app/elements/core/gr-main-header/gr-main-header.ts +++ b/polygerrit-ui/app/elements/core/gr-main-header/gr-main-header.ts @@ -104,9 +104,6 @@ declare global { @customElement('gr-main-header') export class GrMainHeader extends LitElement { - @property({type: String}) - searchQuery = ''; - @property({type: Boolean, reflect: true}) loggedIn?: boolean; @@ -143,8 +140,6 @@ export class GrMainHeader extends LitElement { // private but used in test @state() feedbackURL = ''; - @state() private serverConfig?: ServerInfo; - private readonly restApiService = getAppContext().restApiService; private readonly getPluginLoader = resolve(this, pluginLoaderToken); @@ -172,7 +167,6 @@ export class GrMainHeader extends LitElement { this.subscriptions.push( this.getConfigModel().serverConfig$.subscribe(config => { if (!config) return; - this.serverConfig = config; this.retrieveFeedbackURL(config); this.retrieveRegisterURL(config); this.restApiService.getDocsBaseUrl(config).then(docBaseUrl => { @@ -378,12 +372,7 @@ export class GrMainHeader extends LitElement { class="hideOnMobile" name="header-small-banner" ></gr-endpoint-decorator> - <gr-smart-search - id="search" - label="Search for changes" - .searchQuery=${this.searchQuery} - .serverConfig=${this.serverConfig} - ></gr-smart-search> + <gr-smart-search id="search"></gr-smart-search> <gr-endpoint-decorator class="hideOnMobile" name="header-top-right" diff --git a/polygerrit-ui/app/elements/core/gr-main-header/gr-main-header_test.ts b/polygerrit-ui/app/elements/core/gr-main-header/gr-main-header_test.ts index 155ba10ce4..955846f67c 100644 --- a/polygerrit-ui/app/elements/core/gr-main-header/gr-main-header_test.ts +++ b/polygerrit-ui/app/elements/core/gr-main-header/gr-main-header_test.ts @@ -61,8 +61,7 @@ suite('gr-main-header tests', () => { name="header-small-banner" > </gr-endpoint-decorator> - <gr-smart-search id="search" label="Search for changes"> - </gr-smart-search> + <gr-smart-search id="search"> </gr-smart-search> <gr-endpoint-decorator class="hideOnMobile" name="header-top-right"> </gr-endpoint-decorator> <gr-endpoint-decorator @@ -159,7 +158,6 @@ suite('gr-main-header tests', () => { { name: 'Repos', url: '/repos', - noBaseUrl: true, view: undefined, }, ]; @@ -233,7 +231,6 @@ suite('gr-main-header tests', () => { { name: 'Repos', url: '/repos', - noBaseUrl: true, view: undefined, }, ]; @@ -280,7 +277,6 @@ suite('gr-main-header tests', () => { { name: 'Repos', url: '/repos', - noBaseUrl: true, view: undefined, }, ]; @@ -332,7 +328,6 @@ suite('gr-main-header tests', () => { { name: 'Repos', url: '/repos', - noBaseUrl: true, view: undefined, }, ]; @@ -494,7 +489,6 @@ suite('gr-main-header tests', () => { { name: 'Repos', url: '/repos', - noBaseUrl: true, view: undefined, }, ]; diff --git a/polygerrit-ui/app/elements/core/gr-search-bar/gr-search-bar.ts b/polygerrit-ui/app/elements/core/gr-search-bar/gr-search-bar.ts index a2974869d6..0856eedcaf 100644 --- a/polygerrit-ui/app/elements/core/gr-search-bar/gr-search-bar.ts +++ b/polygerrit-ui/app/elements/core/gr-search-bar/gr-search-bar.ts @@ -164,9 +164,6 @@ export class GrSearchBar extends LitElement { @state() mergeabilityComputationBehavior?: MergeabilityComputationBehavior; - @property({type: String}) - label = ''; - // private but used in test @state() inputVal = ''; @@ -224,7 +221,7 @@ export class GrSearchBar extends LitElement { <form> <gr-autocomplete id="searchInput" - .label=${this.label} + label="Search for changes" .text=${this.inputVal} .query=${this.query} allow-non-suggested-values diff --git a/polygerrit-ui/app/elements/core/gr-search-bar/gr-search-bar_test.ts b/polygerrit-ui/app/elements/core/gr-search-bar/gr-search-bar_test.ts index 6e348ce5a9..603ea7b202 100644 --- a/polygerrit-ui/app/elements/core/gr-search-bar/gr-search-bar_test.ts +++ b/polygerrit-ui/app/elements/core/gr-search-bar/gr-search-bar_test.ts @@ -70,6 +70,7 @@ suite('gr-search-bar tests', () => { <gr-autocomplete allow-non-suggested-values="" id="searchInput" + label="Search for changes" multi="" tab-complete="" > diff --git a/polygerrit-ui/app/elements/core/gr-smart-search/gr-smart-search.ts b/polygerrit-ui/app/elements/core/gr-smart-search/gr-smart-search.ts index b9c920ad86..8b4b52ffe0 100644 --- a/polygerrit-ui/app/elements/core/gr-smart-search/gr-smart-search.ts +++ b/polygerrit-ui/app/elements/core/gr-smart-search/gr-smart-search.ts @@ -14,11 +14,14 @@ import { import {AutocompleteSuggestion} from '../../shared/gr-autocomplete/gr-autocomplete'; import {getAppContext} from '../../../services/app-context'; import {LitElement, html} from 'lit'; -import {customElement, property, state} from 'lit/decorators.js'; +import {customElement, state} from 'lit/decorators.js'; import {subscribe} from '../../lit/subscription-controller'; import {resolve} from '../../../models/dependency'; import {configModelToken} from '../../../models/config/config-model'; -import {createSearchUrl} from '../../../models/views/search'; +import { + createSearchUrl, + searchViewModelToken, +} from '../../../models/views/search'; import {throwingErrorCallback} from '../../shared/gr-rest-api-interface/gr-rest-apis/gr-rest-api-helper'; const MAX_AUTOCOMPLETE_RESULTS = 10; @@ -36,29 +39,31 @@ declare global { @customElement('gr-smart-search') export class GrSmartSearch extends LitElement { - @property({type: String}) + @state() searchQuery = ''; @state() serverConfig?: ServerInfo; - @property({type: String}) - label = ''; - private readonly restApiService = getAppContext().restApiService; private readonly getConfigModel = resolve(this, configModelToken); private readonly getNavigation = resolve(this, navigationToken); + private readonly getSearchViewModel = resolve(this, searchViewModelToken); + constructor() { super(); subscribe( this, () => this.getConfigModel().serverConfig$, - config => { - this.serverConfig = config; - } + config => (this.serverConfig = config) + ); + subscribe( + this, + () => this.getSearchViewModel().query$, + query => (this.searchQuery = query ?? '') ); } @@ -72,7 +77,6 @@ export class GrSmartSearch extends LitElement { return html` <gr-search-bar id="search" - .label=${this.label} .value=${this.searchQuery} .projectSuggestions=${projectSuggestions} .groupSuggestions=${groupSuggestions} diff --git a/polygerrit-ui/app/elements/gr-app-element.ts b/polygerrit-ui/app/elements/gr-app-element.ts index a36f17af55..f81640de71 100644 --- a/polygerrit-ui/app/elements/gr-app-element.ts +++ b/polygerrit-ui/app/elements/gr-app-element.ts @@ -68,7 +68,7 @@ import {isDarkTheme, prefersDarkColorScheme} from '../utils/theme-util'; import {AppTheme} from '../constants/constants'; import {subscribe} from './lit/subscription-controller'; import {PluginViewState} from '../models/views/plugin'; -import {createSearchUrl, SearchViewState} from '../models/views/search'; +import {createSearchUrl} from '../models/views/search'; import {createSettingsUrl} from '../models/views/settings'; import {createDashboardUrl} from '../models/views/dashboard'; import {userModelToken} from '../models/user/user-model'; @@ -420,7 +420,6 @@ export class GrAppElement extends LitElement { return html` <gr-main-header id="mainHeader" - .searchQuery=${(this.params as SearchViewState)?.query} @mobile-search=${this.mobileSearchToggle} @show-keyboard-shortcuts=${this.showKeyboardShortcuts} .mobileSearchHidden=${!this.mobileSearch} @@ -464,14 +463,7 @@ export class GrAppElement extends LitElement { private renderMobileSearch() { if (!this.mobileSearch) return nothing; - return html` - <gr-smart-search - id="search" - label="Search for changes" - .searchQuery=${(this.params as SearchViewState)?.query} - > - </gr-smart-search> - `; + return html`<gr-smart-search id="search"></gr-smart-search>`; } private renderChangeListView() { diff --git a/polygerrit-ui/app/elements/gr-app_test.ts b/polygerrit-ui/app/elements/gr-app_test.ts index 6b97c85d08..15cfda68e9 100644 --- a/polygerrit-ui/app/elements/gr-app_test.ts +++ b/polygerrit-ui/app/elements/gr-app_test.ts @@ -30,7 +30,6 @@ suite('gr-app callback tests', () => { GrRouter.prototype, <any>'dispatchLocationChangeEvent' ); - setup(async () => { await fixture<GrApp>(html`<gr-app id="app"></gr-app>`); }); diff --git a/polygerrit-ui/app/elements/shared/gr-js-api-interface/gr-change-actions-js-api_test.ts b/polygerrit-ui/app/elements/shared/gr-js-api-interface/gr-change-actions-js-api_test.ts index 4977ec5f09..e3ef083e75 100644 --- a/polygerrit-ui/app/elements/shared/gr-js-api-interface/gr-change-actions-js-api_test.ts +++ b/polygerrit-ui/app/elements/shared/gr-js-api-interface/gr-change-actions-js-api_test.ts @@ -146,20 +146,17 @@ suite('gr-change-actions-js-api-interface tests', () => { test('move action button to overflow', async () => { const key = changeActions.add(ActionType.REVISION, 'Bork!'); await element.updateComplete; - assert.isTrue(queryAndAssert<GrDropdown>(element, '#moreActions').hidden); - assert.isOk( - queryAndAssert<GrButton>(element, `[data-action-key="${key}"]`) - ); + + let items = queryAndAssert<GrDropdown>(element, '#moreActions').items; + assert.isFalse(items?.some(item => item.name === 'Bork!')); + assert.isOk(query<GrButton>(element, `[data-action-key="${key}"]`)); + changeActions.setActionOverflow(ActionType.REVISION, key, true); await element.updateComplete; + + items = queryAndAssert<GrDropdown>(element, '#moreActions').items; + assert.isTrue(items?.some(item => item.name === 'Bork!')); assert.isNotOk(query<GrButton>(element, `[data-action-key="${key}"]`)); - assert.isFalse( - queryAndAssert<GrDropdown>(element, '#moreActions').hidden - ); - assert.strictEqual( - queryAndAssert<GrDropdown>(element, '#moreActions').items![0].name, - 'Bork!' - ); }); test('change actions priority', async () => { diff --git a/polygerrit-ui/app/models/views/admin.ts b/polygerrit-ui/app/models/views/admin.ts index 37b0c15a87..017470e93d 100644 --- a/polygerrit-ui/app/models/views/admin.ts +++ b/polygerrit-ui/app/models/views/admin.ts @@ -38,7 +38,6 @@ export interface AdminNavLinksOption { export interface NavLink { name: string; - noBaseUrl: boolean; url: string; view?: GerritView | AdminChildView; viewableToAll?: boolean; @@ -68,7 +67,6 @@ export enum AdminChildView { const ADMIN_LINKS: NavLink[] = [ { name: 'Repositories', - noBaseUrl: true, url: createAdminUrl({adminView: AdminChildView.REPOS}), view: 'gr-repo-list' as GerritView, viewableToAll: true, @@ -76,7 +74,6 @@ const ADMIN_LINKS: NavLink[] = [ { name: 'Groups', section: 'Groups', - noBaseUrl: true, url: createAdminUrl({adminView: AdminChildView.GROUPS}), view: 'gr-admin-group-list' as GerritView, }, @@ -84,7 +81,6 @@ const ADMIN_LINKS: NavLink[] = [ name: 'Plugins', capability: 'viewPlugins', section: 'Plugins', - noBaseUrl: true, url: createAdminUrl({adminView: AdminChildView.PLUGINS}), view: 'gr-plugin-list' as GerritView, }, @@ -94,7 +90,6 @@ export interface AdminLink { url: string; text: string; capability: string | null; - noBaseUrl: boolean; view: null; viewableToAll: boolean; target: '_blank' | null; @@ -142,7 +137,6 @@ function filterLinks( url: link.url, name: link.text, capability: link.capability || undefined, - noBaseUrl: !isExternalLink(link), view: undefined, viewableToAll: !link.capability, target: isExternalLink(link) ? '_blank' : null, diff --git a/polygerrit-ui/app/services/gr-reporting/gr-reporting_impl.ts b/polygerrit-ui/app/services/gr-reporting/gr-reporting_impl.ts index 55bb64cbd7..cc54c4a16d 100644 --- a/polygerrit-ui/app/services/gr-reporting/gr-reporting_impl.ts +++ b/polygerrit-ui/app/services/gr-reporting/gr-reporting_impl.ts @@ -676,6 +676,12 @@ export class GrReporting implements ReportingService, Finalizable { time(name: Timing) { this._baselines[name] = now(); window.performance.mark(`${name}-start`); + // When time(Timing.DASHBOARD_DISPLAYED) is called gr-dashboard-view + // we need to clean-up slowRpcList, otherwise it can accumulate to big size + if (name === Timing.DASHBOARD_DISPLAYED) { + this.slowRpcList = []; + this.hiddenDurationTimer.reset(); + } } /** diff --git a/polygerrit-ui/app/services/gr-rest-api/gr-rest-api-impl.ts b/polygerrit-ui/app/services/gr-rest-api/gr-rest-api-impl.ts index f19326455f..347e24c021 100644 --- a/polygerrit-ui/app/services/gr-rest-api/gr-rest-api-impl.ts +++ b/polygerrit-ui/app/services/gr-rest-api/gr-rest-api-impl.ts @@ -818,9 +818,9 @@ export class GrRestApiServiceImpl implements RestApiService, Finalizable { if (cachedEmails) { const emails = cachedEmails.map(entry => { if (entry.email === email) { - return {email, preferred: true}; + return {email: entry.email, preferred: true}; } else { - return {email}; + return {email: entry.email, preferred: false}; } }); this._cache.set('/accounts/self/emails', emails); @@ -1016,6 +1016,12 @@ export class GrRestApiServiceImpl implements RestApiService, Finalizable { }); } + /** + * Construct the uri to get list of changes. + * + * If options is undefined then default options (see _getChangesOptionsHex) is + * used. + */ getRequestForGetChanges( changesPerPage?: number, query?: string[] | string, @@ -1044,6 +1050,12 @@ export class GrRestApiServiceImpl implements RestApiService, Finalizable { return request; } + /** + * For every query fetches the matching changes. + * + * If options is undefined then default options (see _getChangesOptionsHex) is + * used. + */ getChangesForMultipleQueries( changesPerPage?: number, query?: string[], @@ -1085,6 +1097,12 @@ export class GrRestApiServiceImpl implements RestApiService, Finalizable { }); } + /** + * Fetches changes that match the query. + * + * If options is undefined then default options (see _getChangesOptionsHex) is + * used. + */ getChanges( changesPerPage?: number, query?: string, @@ -1189,6 +1207,7 @@ export class GrRestApiServiceImpl implements RestApiService, Finalizable { ListChangesOption.LABELS, ListChangesOption.DETAILED_ACCOUNTS, ListChangesOption.SUBMIT_REQUIREMENTS, + ListChangesOption.STAR, ]; return listChangesOptionsToHex(...options); @@ -3083,37 +3102,48 @@ export class GrRestApiServiceImpl implements RestApiService, Finalizable { }); } - async setInProjectLookup(changeNum: NumericChangeId, project: RepoName) { - const lookupProject = await this._projectLookup[changeNum]; - if (lookupProject && lookupProject !== project) { - console.warn( - 'Change set with multiple project nums.' + - 'One of them must be invalid.' - ); - } + /** + * This can be called by the router, if the project can be determined from + * the URL. Or when handling a dashabord or a search response. + * + * Then we don't need to make a dedicated REST API call or have a fallback, + * if that fails. + */ + setInProjectLookup(changeNum: NumericChangeId, project: RepoName) { this._projectLookup[changeNum] = Promise.resolve(project); } getFromProjectLookup( changeNum: NumericChangeId ): Promise<RepoName | undefined> { - const project = this._projectLookup[`${changeNum}`]; - if (project) { - return project; - } - - const onError = (response?: Response | null) => firePageError(response); - - const projectPromise = this.getChange(changeNum, onError).then(change => { - if (!change || !change.project) { - return; + // Hopefully setInProjectLookup() has already been called. Then we don't + // have to make a dedicated REST API call to look up the project. + let projectPromise = this._projectLookup[changeNum]; + if (projectPromise) return projectPromise; + + // Ignore errors, because we have some dedicated fallback logic, see below. + const onError = () => {}; + projectPromise = this.getChange(changeNum, onError).then(change => { + if (change?.project) return change.project; + + // In the very rare case that the change index cannot provide an answer + // (e.g. stale index) we should check, if the router has called + // setInProjectLookup() in the meantime. Then we can fall back to that. + const currentProjectPromise = this._projectLookup[changeNum]; + if (currentProjectPromise !== projectPromise) { + return currentProjectPromise; } - this.setInProjectLookup(changeNum, change.project); - return change.project; - }); + // No luck. Without knowing the project we cannot proceed at all. + firePageError( + new Response( + `Failed to lookup the repo for change number ${changeNum}`, + {status: 404} + ) + ); + return undefined; + }); this._projectLookup[changeNum] = projectPromise; - return projectPromise; } @@ -3303,10 +3333,6 @@ export class GrRestApiServiceImpl implements RestApiService, Finalizable { return this.getDocsBaseUrlCachedPromise; } - testOnly_clearDocsBaseUrlCache() { - this.getDocsBaseUrlCachedPromise = undefined; - } - getDocumentationSearches(filter: string): Promise<DocResult[] | undefined> { filter = filter.trim(); const encodedFilter = encodeURIComponent(filter); diff --git a/polygerrit-ui/app/services/gr-rest-api/gr-rest-api-impl_test.ts b/polygerrit-ui/app/services/gr-rest-api/gr-rest-api-impl_test.ts index 3ab5c54ba2..7abcb0d6d6 100644 --- a/polygerrit-ui/app/services/gr-rest-api/gr-rest-api-impl_test.ts +++ b/polygerrit-ui/app/services/gr-rest-api/gr-rest-api-impl_test.ts @@ -38,6 +38,7 @@ import { } from '../../constants/constants'; import { BasePatchSetNum, + ChangeInfo, ChangeMessageId, CommentInfo, DashboardId, @@ -62,6 +63,7 @@ import { import {assert} from '@open-wc/testing'; import {AuthService} from '../gr-auth/gr-auth'; import {GrAuthMock} from '../gr-auth/gr-auth_mock'; +import {getBaseUrl} from '../../utils/url-util'; const EXPECTED_QUERY_OPTIONS = listChangesOptionsToHex( ListChangesOption.CHANGE_ACTIONS, @@ -358,7 +360,7 @@ suite('gr-rest-api-service-impl tests', () => { assert.isTrue(fetchStub.calledOnce); assert.equal( fetchStub.firstCall.args[0].url, - 'test52/accounts/?o=DETAILS&q=%22bro%22' + `${getBaseUrl()}/accounts/?o=DETAILS&q=%22bro%22` ); }); @@ -367,7 +369,7 @@ suite('gr-rest-api-service-impl tests', () => { assert.isTrue(fetchStub.calledOnce); assert.equal( fetchStub.firstCall.args[0].url, - 'test53/accounts/?o=DETAILS&q=%22bro%22%20and%20cansee%3A341682' + `${getBaseUrl()}/accounts/?o=DETAILS&q=%22bro%22%20and%20cansee%3A341682` ); }); @@ -381,8 +383,7 @@ suite('gr-rest-api-service-impl tests', () => { assert.isTrue(fetchStub.calledOnce); assert.equal( fetchStub.firstCall.args[0].url, - 'test54/accounts/?o=DETAILS&q=%22bro%22%20and%20' + - 'cansee%3A341682%20and%20is%3Aactive' + `${getBaseUrl()}/accounts/?o=DETAILS&q=%22bro%22%20and%20cansee%3A341682%20and%20is%3Aactive` ); }); }); @@ -564,6 +565,29 @@ suite('gr-rest-api-service-impl tests', () => { assert.deepEqual(sendStub.lastCall.args[0].body, {token: 'foo'}); }); + test('setPreferredAccountEmail', async () => { + const email1 = 'email1@example.com'; + const email2 = 'email2@example.com'; + const encodedEmail = encodeURIComponent(email2); + const sendStub = sinon.stub(element._restApiHelper, 'send').resolves(); + element._cache.set('/accounts/self/emails', [ + {email: email1, preferred: true}, + {email: email2, preferred: false}, + ]); + + await element.setPreferredAccountEmail(email2); + assert.isTrue(sendStub.calledOnce); + assert.equal(sendStub.lastCall.args[0].method, HttpMethod.PUT); + assert.equal( + sendStub.lastCall.args[0].url, + `/accounts/self/emails/${encodedEmail}/preferred` + ); + assert.deepEqual(element._cache.get('/accounts/self/emails'), [ + {email: email1, preferred: false}, + {email: email2, preferred: true}, + ]); + }); + test('setAccountStatus', async () => { const sendStub = sinon .stub(element._restApiHelper, 'send') @@ -1156,32 +1180,46 @@ suite('gr-rest-api-service-impl tests', () => { }); test('setInProjectLookup', async () => { - await element.setInProjectLookup( - 555 as NumericChangeId, - 'project' as RepoName - ); + element.setInProjectLookup(555 as NumericChangeId, 'project' as RepoName); const project = await element.getFromProjectLookup(555 as NumericChangeId); assert.deepEqual(project, 'project' as RepoName); }); suite('getFromProjectLookup', () => { - test('getChange succeeds, no project', async () => { - sinon.stub(element, 'getChange').resolves(null); - const val = await element.getFromProjectLookup(555 as NumericChangeId); - assert.strictEqual(val, undefined); + const changeNum = 555 as NumericChangeId; + const repo = 'test-repo' as RepoName; + + test('getChange fails to yield a project', async () => { + const promise = mockPromise<null>(); + sinon.stub(element, 'getChange').returns(promise); + + const projectLookup = element.getFromProjectLookup(changeNum); + promise.resolve(null); + + assert.isUndefined(await projectLookup); }); test('getChange succeeds with project', async () => { - sinon - .stub(element, 'getChange') - .resolves({...createChange(), project: 'project' as RepoName}); - const projectLookup = element.getFromProjectLookup( - 555 as NumericChangeId - ); - const val = await projectLookup; - assert.equal(val, 'project' as RepoName); + const promise = mockPromise<null | ChangeInfo>(); + sinon.stub(element, 'getChange').returns(promise); + + const projectLookup = element.getFromProjectLookup(changeNum); + promise.resolve({...createChange(), project: repo}); + + assert.equal(await projectLookup, repo); assert.deepEqual(element._projectLookup, {'555': projectLookup}); }); + + test('getChange fails, but a setInProjectLookup() call is used as fallback', async () => { + const promise = mockPromise<null>(); + sinon.stub(element, 'getChange').returns(promise); + + const projectLookup = element.getFromProjectLookup(changeNum); + element.setInProjectLookup(changeNum, repo); + promise.resolve(null); + + assert.equal(await projectLookup, repo); + }); }); suite('getChanges populates _projectLookup', () => { @@ -1594,10 +1632,11 @@ suite('gr-rest-api-service-impl tests', () => { test('null config', async () => { const probePathMock = sinon.stub(element, 'probePath').resolves(true); const docsBaseUrl = await element.getDocsBaseUrl(undefined); - assert.isTrue( - probePathMock.calledWith('test91/Documentation/index.html') + assert.equal( + probePathMock.lastCall.args[0], + `${getBaseUrl()}/Documentation/index.html` ); - assert.equal(docsBaseUrl, 'test91/Documentation'); + assert.equal(docsBaseUrl, `${getBaseUrl()}/Documentation`); }); test('no doc config', async () => { @@ -1607,10 +1646,11 @@ suite('gr-rest-api-service-impl tests', () => { gerrit: createGerritInfo(), }; const docsBaseUrl = await element.getDocsBaseUrl(config); - assert.isTrue( - probePathMock.calledWith('test92/Documentation/index.html') + assert.equal( + probePathMock.lastCall.args[0], + `${getBaseUrl()}/Documentation/index.html` ); - assert.equal(docsBaseUrl, 'test92/Documentation'); + assert.equal(docsBaseUrl, `${getBaseUrl()}/Documentation`); }); test('has doc config', async () => { @@ -1627,8 +1667,9 @@ suite('gr-rest-api-service-impl tests', () => { test('no probe', async () => { const probePathMock = sinon.stub(element, 'probePath').resolves(false); const docsBaseUrl = await element.getDocsBaseUrl(undefined); - assert.isTrue( - probePathMock.calledWith('test94/Documentation/index.html') + assert.equal( + probePathMock.lastCall.args[0], + `${getBaseUrl()}/Documentation/index.html` ); assert.isNotOk(docsBaseUrl); }); diff --git a/polygerrit-ui/app/services/gr-rest-api/gr-rest-api.ts b/polygerrit-ui/app/services/gr-rest-api/gr-rest-api.ts index d70304e67c..d8bf276927 100644 --- a/polygerrit-ui/app/services/gr-rest-api/gr-rest-api.ts +++ b/polygerrit-ui/app/services/gr-rest-api/gr-rest-api.ts @@ -375,7 +375,6 @@ export interface RestApiService extends Finalizable { ): Promise<string>; getDocsBaseUrl(config?: ServerInfo): Promise<string | null>; - testOnly_clearDocsBaseUrlCache(): void; createChange( repo: RepoName, diff --git a/polygerrit-ui/app/test/mocks/gr-rest-api_mock.ts b/polygerrit-ui/app/test/mocks/gr-rest-api_mock.ts index f8e416066f..bfa881ecdd 100644 --- a/polygerrit-ui/app/test/mocks/gr-rest-api_mock.ts +++ b/polygerrit-ui/app/test/mocks/gr-rest-api_mock.ts @@ -321,9 +321,6 @@ export const grRestApiMock: RestApiService = { } return Promise.resolve(''); }, - testOnly_clearDocsBaseUrlCache() { - return; - }, getDocumentationSearches(): Promise<DocResult[] | undefined> { return Promise.resolve([]); }, diff --git a/polygerrit-ui/app/utils/change-util.ts b/polygerrit-ui/app/utils/change-util.ts index 4490afa7f0..fcc136103c 100644 --- a/polygerrit-ui/app/utils/change-util.ts +++ b/polygerrit-ui/app/utils/change-util.ts @@ -92,14 +92,16 @@ export const ListChangesOption = { // Skip mergeability data. SKIP_MERGEABLE: 22, - /** - * Skip diffstat computation that compute the insertions field (number of lines inserted) and - * deletions field (number of lines deleted) - */ + // Skip diffstat computation that compute the insertions field (number of lines inserted) and + // deletions field (number of lines deleted) SKIP_DIFFSTAT: 23, - /** Include the evaluated submit requirements for the caller. */ + // Include the evaluated submit requirements for the caller. SUBMIT_REQUIREMENTS: 24, + + // Include the 'starred' field, that is if the change is starred by the + // current user. + STAR: 25, }; export function listChangesOptionsToHex(...args: number[]) { diff --git a/polygerrit-ui/app/utils/url-util.ts b/polygerrit-ui/app/utils/url-util.ts index 826763808d..2f9bc0cca3 100644 --- a/polygerrit-ui/app/utils/url-util.ts +++ b/polygerrit-ui/app/utils/url-util.ts @@ -34,7 +34,7 @@ export function loginUrl(authConfig: AuthInfo | undefined): string { ) { return customLoginUrl.startsWith('http') ? customLoginUrl - : baseUrl + customLoginUrl; + : baseUrl + sanitizeRelativeUrl(customLoginUrl); } else { // Strip the canonical path from the path since needing canonical in // the path is unneeded and breaks the url. @@ -73,6 +73,10 @@ export function getPatchRangeExpression(params: PatchRangeParams) { return range; } +function sanitizeRelativeUrl(relativeUrl: string): string { + return relativeUrl.startsWith('/') ? relativeUrl : `/${relativeUrl}`; +} + export function prependOrigin(path: string): string { if (path.startsWith('http')) return path; if (path.startsWith('/')) return window.location.origin + path; diff --git a/polygerrit-ui/app/utils/url-util_test.ts b/polygerrit-ui/app/utils/url-util_test.ts index e36719dacd..e2ca617c7d 100644 --- a/polygerrit-ui/app/utils/url-util_test.ts +++ b/polygerrit-ui/app/utils/url-util_test.ts @@ -76,6 +76,12 @@ suite('url-util tests', () => { authConfig.auth_type = AuthType.HTTP_LDAP; assert.deepEqual(loginUrl(authConfig), customLoginUrl); }); + + test('auth.loginUrl is sanitized when defined as a relative url', () => { + authConfig.login_url = 'custom'; + authConfig.auth_type = AuthType.HTTP; + assert.deepEqual(loginUrl(authConfig), '/custom'); + }); }); suite('url encoding and decoding tests', () => { diff --git a/resources/com/google/gerrit/pgm/init/gerrit.sh b/resources/com/google/gerrit/pgm/init/gerrit.sh index 1399b15d22..b1f6ade1e8 100755 --- a/resources/com/google/gerrit/pgm/init/gerrit.sh +++ b/resources/com/google/gerrit/pgm/init/gerrit.sh @@ -388,7 +388,7 @@ ulimit -x >/dev/null 2>&1 && ulimit -x unlimited ; # file locks ##################################################### # Configure the maximum wait time for shutdown ##################################################### -EXTRA_STOP_TIMEOUT=30 +EXTRA_STOP_TIMEOUT=$(get_time_unit_sec "$(get_config --get container.shutdownTimeout || echo 30)") HTTPD_STOP_TIMEOUT=$(get_time_unit_sec "$(get_config --get httpd.gracefulStopTimeout || echo 0)") SSHD_STOP_TIMEOUT=$(get_time_unit_sec "$(get_config --get sshd.gracefulStopTimeout || echo 0)") diff --git a/resources/com/google/gerrit/server/commit-msg_test.sh b/resources/com/google/gerrit/server/commit-msg_test.sh index 0cc3da0368..a14109d3f4 100755 --- a/resources/com/google/gerrit/server/commit-msg_test.sh +++ b/resources/com/google/gerrit/server/commit-msg_test.sh @@ -2,7 +2,18 @@ set -eu -hook=$(pwd)/resources/com/google/gerrit/server/tools/root/hooks/commit-msg +readlink -f / &> /dev/null || readlink() { greadlink "$@" ; } # for MacOS +test_dir=$(dirname -- "$(readlink -f -- "$0")") +hook=$test_dir/tools/root/hooks/commit-msg + +if [ -z "${TEST_TMPDIR-}" ] ; then + TEST_TMPDIR=$(mktemp -d) + trap cleanup EXIT +fi + +function cleanup { + rm -rf "$TEST_TMPDIR" +} cd $TEST_TMPDIR diff --git a/tools/BUILD b/tools/BUILD index 70d431509f..b6a35726fb 100644 --- a/tools/BUILD +++ b/tools/BUILD @@ -76,7 +76,7 @@ java_package_configuration( "-Xep:BadImport:ERROR", "-Xep:BadInstanceof:ERROR", "-Xep:BadShiftAmount:ERROR", - "-Xep:BanJNDI:WARN", + "-Xep:BanJNDI:OFF", "-Xep:BanSerializableRead:ERROR", "-Xep:BigDecimalEquals:ERROR", "-Xep:BigDecimalLiteralDouble:ERROR", @@ -159,7 +159,7 @@ java_package_configuration( "-Xep:FloatingPointLiteralPrecision:ERROR", "-Xep:FloggerArgumentToString:ERROR", "-Xep:FloggerFormatString:ERROR", - "-Xep:FloggerLogString:WARN", + "-Xep:FloggerLogString:OFF", "-Xep:FloggerLogVarargs:ERROR", "-Xep:FloggerSplitLogStatement:ERROR", "-Xep:FloggerStringConcatenation:ERROR", @@ -189,7 +189,7 @@ java_package_configuration( "-Xep:ImmutableAnnotationChecker:ERROR", "-Xep:ImmutableEnumChecker:ERROR", "-Xep:ImmutableModification:ERROR", - "-Xep:ImpossibleNullComparison:WARN", + "-Xep:ImpossibleNullComparison:OFF", "-Xep:Incomparable:ERROR", "-Xep:IncompatibleArgumentType:ERROR", "-Xep:IncompatibleModifiers:ERROR", @@ -258,7 +258,7 @@ java_package_configuration( "-Xep:JodaWithDurationAddedLong:ERROR", "-Xep:LiteByteStringUtf8:ERROR", "-Xep:LiteEnumValueOf:ERROR", - "-Xep:LiteProtoToString:WARN", + "-Xep:LiteProtoToString:OFF", "-Xep:LocalDateTemporalAmount:ERROR", "-Xep:LockNotBeforeTry:ERROR", "-Xep:LockOnBoxedPrimitive:ERROR", @@ -290,7 +290,7 @@ java_package_configuration( "-Xep:MultipleParallelOrSequentialCalls:ERROR", "-Xep:MultipleUnaryOperatorsInMethodCall:ERROR", "-Xep:MustBeClosedChecker:ERROR", - "-Xep:MutableConstantField:WARN", + "-Xep:MutableConstantField:OFF", "-Xep:MutablePublicArray:ERROR", "-Xep:NCopiesOfChar:ERROR", "-Xep:NarrowingCompoundAssignment:ERROR", @@ -343,7 +343,7 @@ java_package_configuration( "-Xep:ProtoTimestampGetSecondsGetNano:ERROR", "-Xep:ProtoTruthMixedDescriptors:ERROR", "-Xep:ProtocolBufferOrdinal:ERROR", - "-Xep:ProvidesMethodOutsideOfModule:WARN", + "-Xep:ProvidesMethodOutsideOfModule:OFF", "-Xep:RandomCast:ERROR", "-Xep:RandomModInteger:ERROR", "-Xep:ReachabilityFenceUsage:ERROR", diff --git a/tools/bzl/asciidoc.bzl b/tools/bzl/asciidoc.bzl index 0858d6042f..28de4ec452 100644 --- a/tools/bzl/asciidoc.bzl +++ b/tools/bzl/asciidoc.bzl @@ -300,13 +300,14 @@ def genasciidoc_zip( backend = None, searchbox = True, resources = True, + webfonts = True, **kwargs): SUFFIX = "_htmlonly" _genasciidoc_htmlonly_zip( name = name + SUFFIX if resources else name, srcs = srcs, - attributes = attributes, + attributes = attributes + ([] if webfonts else ["webfonts!"]), backend = backend, searchbox = searchbox, **kwargs diff --git a/tools/bzl/javadoc.bzl b/tools/bzl/javadoc.bzl index 3add0258ce..abf3b7a15f 100644 --- a/tools/bzl/javadoc.bzl +++ b/tools/bzl/javadoc.bzl @@ -37,6 +37,7 @@ def _impl(ctx): "mkdir %s" % dir, " ".join([ "%s/bin/javadoc" % ctx.attr._jdk[java_common.JavaRuntimeInfo].java_home, + " ".join(["-J%s" % opt for opt in ctx.fragments.java.default_jvm_opts]), "-Xdoclint:-missing", "-protected", "-encoding UTF-8", @@ -75,4 +76,5 @@ java_doc = rule( }, outputs = {"zip": "%{name}.zip"}, implementation = _impl, + fragments = ["java"], ) diff --git a/tools/deps.bzl b/tools/deps.bzl index 7d4499a7c8..52e848f4ca 100644 --- a/tools/deps.bzl +++ b/tools/deps.bzl @@ -20,7 +20,7 @@ GITILES_REPO = GERRIT # When updating Bouncy Castle, also update it in bazlets. BC_VERS = "1.72" HTTPCOMP_VERS = "4.5.2" -JETTY_VERS = "9.4.51.v20230217" +JETTY_VERS = "9.4.53.v20231009" BYTE_BUDDY_VERSION = "1.10.7" def java_dependencies(): @@ -121,8 +121,8 @@ def java_dependencies(): # When upgrading commons-compress, also upgrade tukaani-xz maven_jar( name = "commons-compress", - artifact = "org.apache.commons:commons-compress:1.22", - sha1 = "691a8b4e6cf4248c3bc72c8b719337d5cb7359fa", + artifact = "org.apache.commons:commons-compress:1.25.0", + sha1 = "9d35aec423da6c8a7f93d7e9e1c6b1d9fe14bb5e", ) maven_jar( @@ -626,50 +626,50 @@ def java_dependencies(): maven_jar( name = "jetty-servlet", artifact = "org.eclipse.jetty:jetty-servlet:" + JETTY_VERS, - sha1 = "3ec1be0b1ca49b633dd7de0733d0054bb4763965", + sha1 = "6670d6a54cdcaedd8090e8cf420fd5dd7d08e859", ) maven_jar( name = "jetty-security", artifact = "org.eclipse.jetty:jetty-security:" + JETTY_VERS, - sha1 = "a3342214ce480cc5bb8e74fe7589dd0436a5d903", + sha1 = "6fbc8ebe9046954dc2f51d4ba69c8f8344b05f7f", ) maven_jar( name = "jetty-server", artifact = "org.eclipse.jetty:jetty-server:" + JETTY_VERS, - sha1 = "d0572c8460eb26adf8420e78535d95859c89a936", + sha1 = "8b0e761a0b359db59dae77c00b4213b0586cb994", ) maven_jar( name = "jetty-jmx", artifact = "org.eclipse.jetty:jetty-jmx:" + JETTY_VERS, - sha1 = "a69e9b0a223a5f661606f6fb36d3b3fcf6216432", + sha1 = "f0392f756b59f65ea7d6be41bf7a2f7b2c7c98d5", ) maven_jar( name = "jetty-http", artifact = "org.eclipse.jetty:jetty-http:" + JETTY_VERS, - sha1 = "fe37568aded59dd8e437e0f670fe5f809071fe8f", + sha1 = "87faf21eb322753f0527bcb88c43e67044786369", ) maven_jar( name = "jetty-io", artifact = "org.eclipse.jetty:jetty-io:" + JETTY_VERS, - sha1 = "a11a0713b17334a5b6e694602fbd1a9457cb5fdd", + sha1 = "70cf7649b27c964ad29bfddf58f3bfe0d30346cf", ) maven_jar( name = "jetty-util", artifact = "org.eclipse.jetty:jetty-util:" + JETTY_VERS, - sha1 = "a11df06530a3a28c9af7ff336730a2f8e18e7205", + sha1 = "f72bb4f687b4454052c6f06528ba9910714df947", ) maven_jar( name = "jetty-util-ajax", artifact = "org.eclipse.jetty:jetty-util-ajax:" + JETTY_VERS, - sha1 = "3b2a998a5ed1f93bc1878fa89d65e307d8b8ebaf", - src_sha1 = "027a15819d3fd1f18e1890bd1bf04b7d48cb3da4", + sha1 = "4d20f6206eb7747293697c5f64c2dc5bf4bd54a4", + src_sha1 = "1aed8017c3c8a449323901639de6b4eb3b1f02ea", ) maven_jar( diff --git a/tools/eclipse/project.py b/tools/eclipse/project.py index d574ecf7b7..dc136ed2cc 100755 --- a/tools/eclipse/project.py +++ b/tools/eclipse/project.py @@ -96,8 +96,7 @@ def _build_bazel_cmd(*args): build = True cmd.append(arg) if custom_java: - cmd.append('--host_java_toolchain=@bazel_tools//tools/jdk:toolchain_java%s' % custom_java) - cmd.append('--java_toolchain=@bazel_tools//tools/jdk:toolchain_java%s' % custom_java) + cmd.append('--config=java%s' % custom_java) return cmd diff --git a/tools/maven/gerrit-acceptance-framework_pom.xml b/tools/maven/gerrit-acceptance-framework_pom.xml index 95e5c72faa..192b612385 100644 --- a/tools/maven/gerrit-acceptance-framework_pom.xml +++ b/tools/maven/gerrit-acceptance-framework_pom.xml @@ -2,7 +2,7 @@ <modelVersion>4.0.0</modelVersion> <groupId>com.google.gerrit</groupId> <artifactId>gerrit-acceptance-framework</artifactId> - <version>3.8.2-SNAPSHOT</version> + <version>3.8.5-SNAPSHOT</version> <packaging>jar</packaging> <name>Gerrit Code Review - Acceptance Test Framework</name> <description>Framework for Gerrit's acceptance tests</description> diff --git a/tools/maven/gerrit-extension-api_pom.xml b/tools/maven/gerrit-extension-api_pom.xml index 2157c1ea0e..0745c3177a 100644 --- a/tools/maven/gerrit-extension-api_pom.xml +++ b/tools/maven/gerrit-extension-api_pom.xml @@ -2,7 +2,7 @@ <modelVersion>4.0.0</modelVersion> <groupId>com.google.gerrit</groupId> <artifactId>gerrit-extension-api</artifactId> - <version>3.8.2-SNAPSHOT</version> + <version>3.8.5-SNAPSHOT</version> <packaging>jar</packaging> <name>Gerrit Code Review - Extension API</name> <description>API for Gerrit Extensions</description> diff --git a/tools/maven/gerrit-plugin-api_pom.xml b/tools/maven/gerrit-plugin-api_pom.xml index c6e4effd27..894f31f46b 100644 --- a/tools/maven/gerrit-plugin-api_pom.xml +++ b/tools/maven/gerrit-plugin-api_pom.xml @@ -2,7 +2,7 @@ <modelVersion>4.0.0</modelVersion> <groupId>com.google.gerrit</groupId> <artifactId>gerrit-plugin-api</artifactId> - <version>3.8.2-SNAPSHOT</version> + <version>3.8.5-SNAPSHOT</version> <packaging>jar</packaging> <name>Gerrit Code Review - Plugin API</name> <description>API for Gerrit Plugins</description> diff --git a/tools/maven/gerrit-war_pom.xml b/tools/maven/gerrit-war_pom.xml index 6798eeccba..14c0d0cb49 100644 --- a/tools/maven/gerrit-war_pom.xml +++ b/tools/maven/gerrit-war_pom.xml @@ -2,7 +2,7 @@ <modelVersion>4.0.0</modelVersion> <groupId>com.google.gerrit</groupId> <artifactId>gerrit-war</artifactId> - <version>3.8.2-SNAPSHOT</version> + <version>3.8.5-SNAPSHOT</version> <packaging>war</packaging> <name>Gerrit Code Review - WAR</name> <description>Gerrit WAR</description> diff --git a/tools/maven/package.bzl b/tools/maven/package.bzl index eacb02b9f4..b25656da7f 100644 --- a/tools/maven/package.bzl +++ b/tools/maven/package.bzl @@ -40,7 +40,7 @@ def maven_package( src = {}, doc = {}, war = {}): - build_cmd = ["bazel_cmd", "build", "--java_toolchain=//tools:error_prone_warnings_toolchain_java11"] + build_cmd = ["bazel_cmd", "build"] mvn_cmd = ["python", "tools/maven/mvn.py", "-v", version] api_cmd = mvn_cmd[:] api_targets = [] diff --git a/tools/nongoogle.bzl b/tools/nongoogle.bzl index 7f26ef3493..dc6e6e0403 100644 --- a/tools/nongoogle.bzl +++ b/tools/nongoogle.bzl @@ -67,18 +67,18 @@ def declare_nongoogle_deps(): sha1 = "cb2f351bf4463751201f43bb99865235d5ba07ca", ) - SSHD_VERS = "2.9.2" + SSHD_VERS = "2.12.0" maven_jar( name = "sshd-osgi", artifact = "org.apache.sshd:sshd-osgi:" + SSHD_VERS, - sha1 = "bac0415734519b2fe433fea196017acf7ed32660", + sha1 = "32b8de1cbb722ba75bdf9898e0c41d42af00ce57", ) maven_jar( name = "sshd-sftp", artifact = "org.apache.sshd:sshd-sftp:" + SSHD_VERS, - sha1 = "7f9089c87b3b44f19998252fd3b68637e3322920", + sha1 = "0f96f00a07b186ea62838a6a4122e8f4cad44df6", ) maven_jar( @@ -96,7 +96,7 @@ def declare_nongoogle_deps(): maven_jar( name = "sshd-mina", artifact = "org.apache.sshd:sshd-mina:" + SSHD_VERS, - sha1 = "765dced3a2b4069bb0c550e18bda057bad8de26f", + sha1 = "8b202f7d4c0d7b714fd0c93a1352af52aa031149", ) maven_jar( diff --git a/tools/remote-bazelrc b/tools/remote-bazelrc index c9a83e47c2..d8da574151 100644 --- a/tools/remote-bazelrc +++ b/tools/remote-bazelrc @@ -25,41 +25,57 @@ # this higher can make builds faster by allowing more jobs to run in parallel. # Setting it too high can result in jobs that timeout, however, while waiting # for a remote machine to execute them. -build:remote --jobs=200 -build:remote --disk_cache= +build:remote_shared --jobs=200 +build:remote_shared --disk_cache= +build:remote_shared --remote_download_minimal # Set several flags related to specifying the platform, toolchain and java # properties. -build:remote --crosstool_top=@rbe_jdk11//cc:toolchain -build:remote --extra_toolchains=@rbe_jdk11//config:cc-toolchain -build:remote --extra_execution_platforms=@rbe_jdk11//config:platform -build:remote --host_platform=@rbe_jdk11//config:platform -build:remote --platforms=@rbe_jdk11//config:platform -build:remote --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 +build:remote_shared --crosstool_top=@rbe_jdk11//cc:toolchain +build:remote_shared --extra_toolchains=@rbe_jdk11//config:cc-toolchain +build:remote_shared --extra_execution_platforms=@rbe_jdk11//config:platform +build:remote_shared --host_platform=@rbe_jdk11//config:platform +build:remote_shared --platforms=@rbe_jdk11//config:platform +build:remote_shared --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 # Set various strategies so that all actions execute remotely. Mixing remote # and local execution will lead to errors unless the toolchain and remote # machine exactly match the host machine. -build:remote --define=EXECUTOR=remote +build:remote_shared --define=EXECUTOR=remote +# Set a higher timeout value, just in case. +build:remote_shared --remote_timeout=3600 + +# Configuration flags for remote settings in Google GCP RBE # Enable the remote cache so action results can be shared across machines, # developers, and workspaces. -build:remote --remote_cache=remotebuildexecution.googleapis.com +build:config_gcp --remote_cache=remotebuildexecution.googleapis.com # Enable remote execution so actions are performed on the remote systems. -build:remote --remote_executor=remotebuildexecution.googleapis.com - -# Set a higher timeout value, just in case. -build:remote --remote_timeout=3600 +build:config_gcp --remote_executor=remotebuildexecution.googleapis.com # Enable authentication. This will pick up application default credentials by # default. You can use --auth_credentials=some_file.json to use a service # account credential instead. -build:remote --google_default_credentials +build:config_gcp --google_default_credentials +build:config_gcp --config=remote_shared + +# Configuration flags for remote settings in BuildBuddy RBE +# Enable the remote cache so action results can be shared across machines, +# developers, and workspaces. +build:config_bb --remote_cache=grpcs://remote.buildbuddy.io + +# Enable remote execution so actions are performed on the remote systems. +build:config_bb --remote_executor=grpcs://remote.buildbuddy.io + +# The results from each Bazel command are viewable with BuildBuddy +build:config_bb --bes_results_url=https://app.buildbuddy.io/invocation/ + +# The results of a local build will be uploaded to the BuildBuddy server, +# providing visibility and collaboration features for the build. +build:config_bb --remote_upload_local_results -# The following flags enable the remote cache so action results can be shared -# across machines, developers, and workspaces. -build:remote-cache --remote_cache=remotebuildexecution.googleapis.com -build:remote-cache --tls_enabled=true -build:remote-cache --remote_timeout=3600 -build:remote-cache --auth_enabled=true +# Define the Build Event Service (BES) backend to use for remote caching and +# build event storage. +build:config_bb --bes_backend=grpcs://remote.buildbuddy.io +build:config_bb --config=remote_shared diff --git a/version.bzl b/version.bzl index 8eb41cf124..dbb4732b41 100644 --- a/version.bzl +++ b/version.bzl @@ -2,4 +2,4 @@ # Used by :api_install and :api_deploy targets # when talking to the destination repository. # -GERRIT_VERSION = "3.8.2-SNAPSHOT" +GERRIT_VERSION = "3.8.5-SNAPSHOT" |