From 0b2bf619e8b8e48fe4237cb6cdc29f0cd3d820c0 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Thu, 24 Dec 2020 00:15:32 +0100 Subject: Add query option allowing administrators to skip visibility filtering If an administrator wants to reindex changes which were created or updated in a given period based on a query for that period the query results are subject to visibility filtering. This can have the effect that e.g. private changes are missed. Add a query option "skip-visibility" to allow administrators to skip visibility filtering. Change-Id: I66c13659587b9459eb7cc585697c1655926ceac3 --- Documentation/rest-api-changes.txt | 6 ++ .../gerrit/server/restapi/change/QueryChanges.java | 24 +++++- .../acceptance/api/change/QueryChangesIT.java | 88 ++++++++++++++++++++++ 3 files changed, 117 insertions(+), 1 deletion(-) diff --git a/Documentation/rest-api-changes.txt b/Documentation/rest-api-changes.txt index 3cb40f6a0d..f67670b893 100644 --- a/Documentation/rest-api-changes.txt +++ b/Documentation/rest-api-changes.txt @@ -138,6 +138,12 @@ limit or a supplied `n` query parameter, the last change object has a The `S` or `start` query parameter can be supplied to skip a number of changes from the list. +Administrators can use the `skip-visibility` query parameter to skip visibility filtering. +This can be used to ensure that no changes are missed e.g. when querying for changes which +need to be reindexed. Without this parameter query results the user has no permission to read +are filtered out. REST requests with the skip-visibility option are rejected when the current +user doesn't have the ADMINISTRATE_SERVER capability. + Clients are allowed to specify more than one query by setting the `q` parameter multiple times. In this case the result is an array of arrays, one per query in the same order the queries were given in. diff --git a/java/com/google/gerrit/server/restapi/change/QueryChanges.java b/java/com/google/gerrit/server/restapi/change/QueryChanges.java index 6e5f554304..bf4d197147 100644 --- a/java/com/google/gerrit/server/restapi/change/QueryChanges.java +++ b/java/com/google/gerrit/server/restapi/change/QueryChanges.java @@ -27,13 +27,17 @@ import com.google.gerrit.extensions.restapi.TopLevelResource; import com.google.gerrit.index.query.QueryParseException; import com.google.gerrit.index.query.QueryRequiresAuthException; import com.google.gerrit.index.query.QueryResult; +import com.google.gerrit.server.CurrentUser; import com.google.gerrit.server.DynamicOptions; import com.google.gerrit.server.change.ChangeJson; +import com.google.gerrit.server.permissions.GlobalPermission; +import com.google.gerrit.server.permissions.PermissionBackend; import com.google.gerrit.server.permissions.PermissionBackendException; import com.google.gerrit.server.query.change.ChangeData; import com.google.gerrit.server.query.change.ChangeQueryBuilder; import com.google.gerrit.server.query.change.ChangeQueryProcessor; import com.google.inject.Inject; +import com.google.inject.Provider; import java.util.ArrayList; import java.util.Collections; import java.util.EnumSet; @@ -46,6 +50,8 @@ public class QueryChanges implements RestReadView, DynamicOpti private final ChangeJson.Factory json; private final ChangeQueryBuilder qb; private final ChangeQueryProcessor imp; + private final Provider userProvider; + private final PermissionBackend permissionBackend; private EnumSet options; @Option( @@ -88,16 +94,32 @@ public class QueryChanges implements RestReadView, DynamicOpti imp.setNoLimit(on); } + @Option(name = "--skip-visibility", usage = "Skip visibility check, only for administrators") + public void skipVisibility(boolean on) throws AuthException, PermissionBackendException { + if (on) { + CurrentUser user = userProvider.get(); + permissionBackend.user(user).check(GlobalPermission.ADMINISTRATE_SERVER); + imp.enforceVisibility(false); + } + } + @Override public void setDynamicBean(String plugin, DynamicOptions.DynamicBean dynamicBean) { imp.setDynamicBean(plugin, dynamicBean); } @Inject - QueryChanges(ChangeJson.Factory json, ChangeQueryBuilder qb, ChangeQueryProcessor qp) { + QueryChanges( + ChangeJson.Factory json, + ChangeQueryBuilder qb, + ChangeQueryProcessor qp, + Provider userProvider, + PermissionBackend permissionBackend) { this.json = json; this.qb = qb; this.imp = qp; + this.userProvider = userProvider; + this.permissionBackend = permissionBackend; options = EnumSet.noneOf(ListChangesOption.class); } diff --git a/javatests/com/google/gerrit/acceptance/api/change/QueryChangesIT.java b/javatests/com/google/gerrit/acceptance/api/change/QueryChangesIT.java index 78354d653b..6839b967fc 100644 --- a/javatests/com/google/gerrit/acceptance/api/change/QueryChangesIT.java +++ b/javatests/com/google/gerrit/acceptance/api/change/QueryChangesIT.java @@ -15,22 +15,32 @@ package com.google.gerrit.acceptance.api.change; import static com.google.common.truth.Truth.assertThat; +import static com.google.gerrit.testing.GerritJUnit.assertThrows; import com.google.common.collect.ImmutableList; import com.google.gerrit.acceptance.AbstractDaemonTest; import com.google.gerrit.acceptance.NoHttpd; +import com.google.gerrit.acceptance.PushOneCommit; +import com.google.gerrit.acceptance.testsuite.request.RequestScopeOperations; +import com.google.gerrit.common.data.Permission; import com.google.gerrit.extensions.common.ChangeInfo; +import com.google.gerrit.extensions.restapi.AuthException; import com.google.gerrit.extensions.restapi.TopLevelResource; +import com.google.gerrit.server.project.ProjectConfig; import com.google.gerrit.server.restapi.change.QueryChanges; import com.google.inject.Inject; import com.google.inject.Provider; +import java.util.Arrays; import java.util.List; +import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository; +import org.eclipse.jgit.junit.TestRepository; import org.junit.Test; @NoHttpd public class QueryChangesIT extends AbstractDaemonTest { @Inject private Provider queryChangesProvider; + @Inject private RequestScopeOperations requestScopeOperations; @Test @SuppressWarnings("unchecked") @@ -97,9 +107,87 @@ public class QueryChangesIT extends AbstractDaemonTest { assertThat(result2.get(1).get(0)._moreChanges).isTrue(); } + @Test + public void skipVisibility_rejectedForNonAdmin() throws Exception { + requestScopeOperations.setApiUser(user.id()); + final QueryChanges queryChanges = queryChangesProvider.get(); + String query = "is:open repo:" + project.get(); + queryChanges.addQuery(query); + AuthException thrown = + assertThrows(AuthException.class, () -> queryChanges.skipVisibility(true)); + assertThat(thrown).hasMessageThat().isEqualTo("administrate server not permitted"); + } + + @Test + @SuppressWarnings("unchecked") + public void skipVisibility_noReadPermission() throws Exception { + createChange().getChangeId(); + requestScopeOperations.setApiUser(admin.id()); + QueryChanges queryChanges = queryChangesProvider.get(); + + queryChanges.addQuery("is:open repo:" + project.get()); + List> result = + (List>) queryChanges.apply(TopLevelResource.INSTANCE).value(); + assertThat(result).hasSize(1); + + try (ProjectConfigUpdate u = updateProject(allProjects)) { + ProjectConfig cfg = u.getConfig(); + removeAllBranchPermissions(cfg, Permission.READ); + u.save(); + } + + queryChanges = queryChangesProvider.get(); + queryChanges.addQuery("is:open repo:" + project.get()); + List> result2 = + (List>) queryChanges.apply(TopLevelResource.INSTANCE).value(); + assertThat(result2).hasSize(0); + + queryChanges = queryChangesProvider.get(); + queryChanges.addQuery("is:open repo:" + project.get()); + queryChanges.skipVisibility(true); + List> result3 = + (List>) queryChanges.apply(TopLevelResource.INSTANCE).value(); + assertThat(result3).hasSize(1); + } + + @Test + @SuppressWarnings("unchecked") + public void skipVisibility_privateChange() throws Exception { + TestRepository userRepo = cloneProject(project, user); + PushOneCommit.Result result = + pushFactory.create(user.newIdent(), userRepo).to("refs/for/master"); + requestScopeOperations.setApiUser(user.id()); + gApi.changes().id(result.getChangeId()).setPrivate(true); + + requestScopeOperations.setApiUser(admin.id()); + QueryChanges queryChanges = queryChangesProvider.get(); + + queryChanges.addQuery("is:open repo:" + project.get()); + List> result2 = + (List>) queryChanges.apply(TopLevelResource.INSTANCE).value(); + assertThat(result2).hasSize(0); + + queryChanges = queryChangesProvider.get(); + queryChanges.addQuery("is:open repo:" + project.get()); + queryChanges.skipVisibility(true); + List> result3 = + (List>) queryChanges.apply(TopLevelResource.INSTANCE).value(); + assertThat(result3).hasSize(1); + } + private static void assertNoChangeHasMoreChangesSet(List results) { for (ChangeInfo info : results) { assertThat(info._moreChanges).isNull(); } } + + private static void removeAllBranchPermissions(ProjectConfig cfg, String... permissions) { + cfg.getAccessSections().stream() + .filter( + s -> + s.getName().startsWith("refs/heads/") + || s.getName().startsWith("refs/for/") + || s.getName().equals("refs/*")) + .forEach(s -> Arrays.stream(permissions).forEach(s::removePermission)); + } } -- cgit v1.2.3 From dfcb9bc90a25ec4fc7dd2c971cd3d20f9313ea98 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Thu, 24 Dec 2020 00:28:23 +0100 Subject: Add script for incremental reindexing during upgrade In order to shorten the downtime needed to reindex changes during a Gerrit upgrade the following strategy can be used: - index preparation - create a full consistent backup - note down the timestamp when the backup was created (backup-time) - create a complete copy of the production system from the backup - upgrade this copy to the new Gerrit version - online reindex this copy - upgrade of the production system - make system unavailable so that users can't reach it anymore e.g. by changing port numbers (downtime starts) - take a full backup - run $ ./reindex.py -u gerrit-url -s backup-time to write the list of changes which have been created or modified since the backup for the index preparation was created to a file "changes-to-reindex.list" - upgrade the production system to the new gerrit version skipping reindexing - copy the bulk of the new index from the copy system to the production system - run $ ./reindex.py -u gerrit-url this reindexes all changes which have been created or modified after the backup was taken reading these changes from the file "changes-to-reindex.list" - smoketest the system - make the production system available to the users again (downtime ends) Change-Id: Ie736e0dc32180329ca6ed31bcb49eb6b96bf2b91 --- contrib/reindex/.flake8 | 9 ++ contrib/reindex/.gitignore | 1 + contrib/reindex/Pipfile | 19 ++++ contrib/reindex/Pipfile.lock | 248 +++++++++++++++++++++++++++++++++++++++++++ contrib/reindex/README.md | 63 +++++++++++ contrib/reindex/reindex.py | 189 +++++++++++++++++++++++++++++++++ 6 files changed, 529 insertions(+) create mode 100644 contrib/reindex/.flake8 create mode 100644 contrib/reindex/.gitignore create mode 100644 contrib/reindex/Pipfile create mode 100644 contrib/reindex/Pipfile.lock create mode 100644 contrib/reindex/README.md create mode 100755 contrib/reindex/reindex.py diff --git a/contrib/reindex/.flake8 b/contrib/reindex/.flake8 new file mode 100644 index 0000000000..151557f247 --- /dev/null +++ b/contrib/reindex/.flake8 @@ -0,0 +1,9 @@ +[flake8] +max-line-length=100 +ignore= + # E203 whitespace before ':' + E203, + # W503: Line break before binary operator + W503, + # W504: Line break after binary operator + W504 diff --git a/contrib/reindex/.gitignore b/contrib/reindex/.gitignore new file mode 100644 index 0000000000..fd8c78f5e9 --- /dev/null +++ b/contrib/reindex/.gitignore @@ -0,0 +1 @@ +changes-to-reindex.list diff --git a/contrib/reindex/Pipfile b/contrib/reindex/Pipfile new file mode 100644 index 0000000000..21ffd90226 --- /dev/null +++ b/contrib/reindex/Pipfile @@ -0,0 +1,19 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +pygerrit2 = "*" +requests = "*" +tqdm = "*" + +[dev-packages] +flake8 = "*" +black = "*" + +[requires] +python_version = "3.9" + +[pipenv] +allow_prereleases = true diff --git a/contrib/reindex/Pipfile.lock b/contrib/reindex/Pipfile.lock new file mode 100644 index 0000000000..bb7cc2dc02 --- /dev/null +++ b/contrib/reindex/Pipfile.lock @@ -0,0 +1,248 @@ +{ + "_meta": { + "hash": { + "sha256": "37be5a74a22d0e084ebfe168bfdcd7bcaa87ad7b42be66b1d9fbff5e936ebe72" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.9" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "certifi": { + "hashes": [ + "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c", + "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830" + ], + "version": "==2020.12.5" + }, + "chardet": { + "hashes": [ + "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa", + "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==4.0.0" + }, + "idna": { + "hashes": [ + "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", + "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.10" + }, + "pbr": { + "hashes": [ + "sha256:5fad80b613c402d5b7df7bd84812548b2a61e9977387a80a5fc5c396492b13c9", + "sha256:b236cde0ac9a6aedd5e3c34517b423cd4fd97ef723849da6b0d2231142d89c00" + ], + "markers": "python_version >= '2.6'", + "version": "==5.5.1" + }, + "pygerrit2": { + "hashes": [ + "sha256:d12cff5cc514dd61281d997ea86771e7f818030c3d2ef230b25bb14dae7d3f86" + ], + "index": "pypi", + "version": "==2.0.14" + }, + "requests": { + "hashes": [ + "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804", + "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e" + ], + "index": "pypi", + "version": "==2.25.1" + }, + "tqdm": { + "hashes": [ + "sha256:38b658a3e4ecf9b4f6f8ff75ca16221ae3378b2e175d846b6b33ea3a20852cf5", + "sha256:d4f413aecb61c9779888c64ddf0c62910ad56dcbe857d8922bb505d4dbff0df1" + ], + "index": "pypi", + "version": "==4.54.1" + }, + "urllib3": { + "hashes": [ + "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08", + "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "version": "==1.26.2" + } + }, + "develop": { + "appdirs": { + "hashes": [ + "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41", + "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128" + ], + "version": "==1.4.4" + }, + "black": { + "hashes": [ + "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea" + ], + "index": "pypi", + "version": "==20.8b1" + }, + "click": { + "hashes": [ + "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", + "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==7.1.2" + }, + "flake8": { + "hashes": [ + "sha256:749dbbd6bfd0cf1318af27bf97a14e28e5ff548ef8e5b1566ccfb25a11e7c839", + "sha256:aadae8761ec651813c24be05c6f7b4680857ef6afaae4651a4eccaef97ce6c3b" + ], + "index": "pypi", + "version": "==3.8.4" + }, + "mccabe": { + "hashes": [ + "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", + "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + ], + "version": "==0.6.1" + }, + "mypy-extensions": { + "hashes": [ + "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d", + "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8" + ], + "version": "==0.4.3" + }, + "pathspec": { + "hashes": [ + "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd", + "sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d" + ], + "version": "==0.8.1" + }, + "pycodestyle": { + "hashes": [ + "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367", + "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.6.0" + }, + "pyflakes": { + "hashes": [ + "sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92", + "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.2.0" + }, + "regex": { + "hashes": [ + "sha256:02951b7dacb123d8ea6da44fe45ddd084aa6777d4b2454fa0da61d569c6fa538", + "sha256:0d08e71e70c0237883d0bef12cad5145b84c3705e9c6a588b2a9c7080e5af2a4", + "sha256:1862a9d9194fae76a7aaf0150d5f2a8ec1da89e8b55890b1786b8f88a0f619dc", + "sha256:1ab79fcb02b930de09c76d024d279686ec5d532eb814fd0ed1e0051eb8bd2daa", + "sha256:1fa7ee9c2a0e30405e21031d07d7ba8617bc590d391adfc2b7f1e8b99f46f444", + "sha256:262c6825b309e6485ec2493ffc7e62a13cf13fb2a8b6d212f72bd53ad34118f1", + "sha256:2a11a3e90bd9901d70a5b31d7dd85114755a581a5da3fc996abfefa48aee78af", + "sha256:2c99e97d388cd0a8d30f7c514d67887d8021541b875baf09791a3baad48bb4f8", + "sha256:3128e30d83f2e70b0bed9b2a34e92707d0877e460b402faca908c6667092ada9", + "sha256:38c8fd190db64f513fe4e1baa59fed086ae71fa45083b6936b52d34df8f86a88", + "sha256:3bddc701bdd1efa0d5264d2649588cbfda549b2899dc8d50417e47a82e1387ba", + "sha256:4902e6aa086cbb224241adbc2f06235927d5cdacffb2425c73e6570e8d862364", + "sha256:49cae022fa13f09be91b2c880e58e14b6da5d10639ed45ca69b85faf039f7a4e", + "sha256:56e01daca75eae420bce184edd8bb341c8eebb19dd3bce7266332258f9fb9dd7", + "sha256:5862975b45d451b6db51c2e654990c1820523a5b07100fc6903e9c86575202a0", + "sha256:6a8ce43923c518c24a2579fda49f093f1397dad5d18346211e46f134fc624e31", + "sha256:6c54ce4b5d61a7129bad5c5dc279e222afd00e721bf92f9ef09e4fae28755683", + "sha256:6e4b08c6f8daca7d8f07c8d24e4331ae7953333dbd09c648ed6ebd24db5a10ee", + "sha256:717881211f46de3ab130b58ec0908267961fadc06e44f974466d1887f865bd5b", + "sha256:749078d1eb89484db5f34b4012092ad14b327944ee7f1c4f74d6279a6e4d1884", + "sha256:7913bd25f4ab274ba37bc97ad0e21c31004224ccb02765ad984eef43e04acc6c", + "sha256:7a25fcbeae08f96a754b45bdc050e1fb94b95cab046bf56b016c25e9ab127b3e", + "sha256:83d6b356e116ca119db8e7c6fc2983289d87b27b3fac238cfe5dca529d884562", + "sha256:8b882a78c320478b12ff024e81dc7d43c1462aa4a3341c754ee65d857a521f85", + "sha256:8f6a2229e8ad946e36815f2a03386bb8353d4bde368fdf8ca5f0cb97264d3b5c", + "sha256:9801c4c1d9ae6a70aeb2128e5b4b68c45d4f0af0d1535500884d644fa9b768c6", + "sha256:a15f64ae3a027b64496a71ab1f722355e570c3fac5ba2801cafce846bf5af01d", + "sha256:a3d748383762e56337c39ab35c6ed4deb88df5326f97a38946ddd19028ecce6b", + "sha256:a63f1a07932c9686d2d416fb295ec2c01ab246e89b4d58e5fa468089cab44b70", + "sha256:b2b1a5ddae3677d89b686e5c625fc5547c6e492bd755b520de5332773a8af06b", + "sha256:b2f4007bff007c96a173e24dcda236e5e83bde4358a557f9ccf5e014439eae4b", + "sha256:baf378ba6151f6e272824b86a774326f692bc2ef4cc5ce8d5bc76e38c813a55f", + "sha256:bafb01b4688833e099d79e7efd23f99172f501a15c44f21ea2118681473fdba0", + "sha256:bba349276b126947b014e50ab3316c027cac1495992f10e5682dc677b3dfa0c5", + "sha256:c084582d4215593f2f1d28b65d2a2f3aceff8342aa85afd7be23a9cad74a0de5", + "sha256:d1ebb090a426db66dd80df8ca85adc4abfcbad8a7c2e9a5ec7513ede522e0a8f", + "sha256:d2d8ce12b7c12c87e41123997ebaf1a5767a5be3ec545f64675388970f415e2e", + "sha256:e32f5f3d1b1c663af7f9c4c1e72e6ffe9a78c03a31e149259f531e0fed826512", + "sha256:e3faaf10a0d1e8e23a9b51d1900b72e1635c2d5b0e1bea1c18022486a8e2e52d", + "sha256:f7d29a6fc4760300f86ae329e3b6ca28ea9c20823df123a2ea8693e967b29917", + "sha256:f8f295db00ef5f8bae530fc39af0b40486ca6068733fb860b42115052206466f" + ], + "version": "==2020.11.13" + }, + "toml": { + "hashes": [ + "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", + "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.10.2" + }, + "typed-ast": { + "hashes": [ + "sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355", + "sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919", + "sha256:0d8110d78a5736e16e26213114a38ca35cb15b6515d535413b090bd50951556d", + "sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa", + "sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652", + "sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75", + "sha256:3742b32cf1c6ef124d57f95be609c473d7ec4c14d0090e5a5e05a15269fb4d0c", + "sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01", + "sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d", + "sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1", + "sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907", + "sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c", + "sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3", + "sha256:7e4c9d7658aaa1fc80018593abdf8598bf91325af6af5cce4ce7c73bc45ea53d", + "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b", + "sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614", + "sha256:92c325624e304ebf0e025d1224b77dd4e6393f18aab8d829b5b7e04afe9b7a2c", + "sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb", + "sha256:b52ccf7cfe4ce2a1064b18594381bccf4179c2ecf7f513134ec2f993dd4ab395", + "sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b", + "sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41", + "sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6", + "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34", + "sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe", + "sha256:d648b8e3bf2fe648745c8ffcee3db3ff903d0817a01a12dd6a6ea7a8f4889072", + "sha256:f208eb7aff048f6bea9586e61af041ddf7f9ade7caed625742af423f6bae3298", + "sha256:fac11badff8313e23717f3dada86a15389d0708275bddf766cca67a84ead3e91", + "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4", + "sha256:fcf135e17cc74dbfbc05894ebca928ffeb23d9790b3167a674921db19082401f", + "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7" + ], + "version": "==1.4.1" + }, + "typing-extensions": { + "hashes": [ + "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918", + "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c", + "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f" + ], + "version": "==3.7.4.3" + } + } +} diff --git a/contrib/reindex/README.md b/contrib/reindex/README.md new file mode 100644 index 0000000000..acb958855d --- /dev/null +++ b/contrib/reindex/README.md @@ -0,0 +1,63 @@ +# Incremental reindexing during upgrade of large gerrit site + +In order to shorten the downtime needed to reindex changes during a +Gerrit upgrade the following strategy can be used: + +- index preparation + - create a full consistent backup + - note down the timestamp when the backup was created (backup-time) + - create a complete copy of the production system from the backup + - upgrade this copy to the new Gerrit version + - online reindex this copy +- upgrade of the production system + - make system unavailable so that users can't reach it anymore + e.g. by changing port numbers (downtime starts) + - take a full backup + - run + + ``` bash + ./reindex.py -u gerrit-url -s backup-time + ``` + + to write the list of changes which have been created or modified + since the backup for the index preparation was created to a file + "changes-to-reindex.list" + - upgrade the production system to the new gerrit version skipping + reindexing + - copy the bulk of the new index from the copy system to the + production system + - run + + ``` bash + ./reindex.py -u gerrit-url + ``` + + this reindexes all changes which have been created or modified after + the backup was taken reading these changes from the file + "changes-to-reindex.list" + - smoketest the system + - make the production system available to the users again + (downtime ends) + +## Online help + +For help on all available options run + +``` bash +./reindex -h +``` + +## Python environment + +Prerequisites: + +- python 3.9 +- pipenv + +Install virtual python environment and run the script + +``` bash +pipenv sync +pipenv shell +./reindex +``` diff --git a/contrib/reindex/reindex.py b/contrib/reindex/reindex.py new file mode 100755 index 0000000000..266f5ecc95 --- /dev/null +++ b/contrib/reindex/reindex.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +from argparse import ArgumentParser, RawTextHelpFormatter +from itertools import islice +import getpass +import logging +import os + +from pygerrit2 import GerritRestAPI, HTTPBasicAuth, HTTPBasicAuthFromNetrc +from tqdm import tqdm + +EPILOG = """\ +To query the list of changes which have been created or modified since the +given timestamp and write them to a file "changes-to-reindex.list" run +$ ./reindex.py -u gerrit-url -s timestamp + +To reindex the list of changes in file "changes-to-reindex.list" run +$ ./reindex.py -u gerrit-url +""" + + +def _parse_options(): + parser = ArgumentParser( + formatter_class=RawTextHelpFormatter, + epilog=EPILOG, + ) + parser.add_argument( + "-u", + "--url", + dest="url", + help="gerrit url", + ) + parser.add_argument( + "-s", + "--since", + dest="time", + help=( + "changes modified after the given 'TIME', inclusive. Must be in the\n" + "format '2006-01-02[ 15:04:05[.890][ -0700]]', omitting the time defaults\n" + "to 00:00:00 and omitting the timezone defaults to UTC." + ), + ) + parser.add_argument( + "-f", + "--file", + default="changes-to-reindex.list", + dest="file", + help=( + "file path to store list of changes if --since is given,\n" + "otherwise file path to read list of changes from" + ), + ) + parser.add_argument( + "-c", + "--chunk", + default=100, + dest="chunksize", + help="chunk size defining how many changes are reindexed per request", + type=int, + ) + parser.add_argument( + "--cert", + dest="cert", + type=str, + help="path to file containing custom ca certificates to trust", + ) + parser.add_argument( + "-v", + "--verbose", + dest="verbose", + action="store_true", + help="verbose debugging output", + ) + parser.add_argument( + "-n", + "--netrc", + default=True, + dest="netrc", + action="store_true", + help=( + "read credentials from .netrc, default to environment variables\n" + "USERNAME and PASSWORD, otherwise prompt for credentials interactively" + ), + ) + return parser.parse_args() + + +def _chunker(iterable, chunksize): + it = map(lambda s: s.strip(), iterable) + while True: + chunk = list(islice(it, chunksize)) + if not chunk: + return + yield chunk + + +class Reindexer: + """Class for reindexing Gerrit changes""" + + def __init__(self): + self.options = _parse_options() + self._init_logger() + credentials = self._authenticate() + if self.options.cert: + certs = os.path.expanduser(self.options.cert) + self.api = GerritRestAPI( + url=self.options.url, auth=credentials, verify=certs + ) + else: + self.api = GerritRestAPI(url=self.options.url, auth=credentials) + + def _init_logger(self): + self.logger = logging.getLogger("Reindexer") + self.logger.setLevel(logging.DEBUG) + h = logging.StreamHandler() + if self.options.verbose: + h.setLevel(logging.DEBUG) + else: + h.setLevel(logging.INFO) + formatter = logging.Formatter("%(message)s") + h.setFormatter(formatter) + self.logger.addHandler(h) + + def _authenticate(self): + username = password = None + if self.options.netrc: + auth = HTTPBasicAuthFromNetrc(url=self.options.url) + username = auth.username + password = auth.password + if not username: + username = os.environ.get("USERNAME") + if not password: + password = os.environ.get("PASSWORD") + while not username: + username = input("user: ") + while not password: + password = getpass.getpass("password: ") + auth = HTTPBasicAuth(username, password) + return auth + + def _query(self): + start = 0 + more_changes = True + while more_changes: + query = f"since:{self.options.time}&start={start}&skip-visibility" + for change in self.api.get(f"changes/?q={query}"): + more_changes = change.get("_more_changes") is not None + start += 1 + yield change.get("_number") + break + + def _query_to_file(self): + self.logger.debug( + f"writing changes since {self.options.time} to file {self.options.file}:" + ) + with open(self.options.file, "w") as output: + for id in self._query(): + self.logger.debug(id) + output.write(f"{id}\n") + + def _reindex_chunk(self, chunk): + self.logger.debug(f"indexing {chunk}") + response = self.api.post( + "/config/server/index.changes", + chunk, + ) + self.logger.debug(f"response: {response}") + + def _reindex(self): + self.logger.debug(f"indexing changes from file {self.options.file}") + with open(self.options.file, "r") as f: + with tqdm(unit="changes", desc="Indexed") as pbar: + for chunk in _chunker(f, self.options.chunksize): + self._reindex_chunk(chunk) + pbar.update(len(chunk)) + + def execute(self): + if self.options.time: + self._query_to_file() + else: + self._reindex() + + +def main(): + reindexer = Reindexer() + reindexer.execute() + + +if __name__ == "__main__": + main() -- cgit v1.2.3