diff options
author | Shawn O. Pearce <sop@google.com> | 2008-12-12 18:25:12 -0800 |
---|---|---|
committer | Shawn O. Pearce <sop@google.com> | 2008-12-12 18:25:12 -0800 |
commit | 38270b642ddb2c457104f07de2485d999ea60ae2 (patch) | |
tree | 0b6d07e59917c0cf1864efec5010528edd91dadc /webapp | |
parent | 583b353040c0568d545df2a8490535b03911925c (diff) |
Import all patch data again from Git to ensure it is accurate
We've made some schema changes about how patch data is stored and
represented in the Gerrit 2 database, so we need to import all of
the patch entities over again. This change includes a CLI tool
that can be run to re-import one or more patch set entities from
the corresponding Git commit.
Signed-off-by: Shawn O. Pearce <sop@google.com>
Diffstat (limited to 'webapp')
6 files changed, 426 insertions, 9 deletions
diff --git a/webapp/src/com/google/gerrit/client/reviewdb/Patch.java b/webapp/src/com/google/gerrit/client/reviewdb/Patch.java index 59406e09b1..4c63288126 100644 --- a/webapp/src/com/google/gerrit/client/reviewdb/Patch.java +++ b/webapp/src/com/google/gerrit/client/reviewdb/Patch.java @@ -132,10 +132,10 @@ public final class Patch { protected Patch() { } - public Patch(final Patch.Id newId, final ChangeType ct, final PatchType pt) { + public Patch(final Patch.Id newId) { key = newId; - setChangeType(ct); - setPatchType(pt); + setChangeType(ChangeType.MODIFIED); + setPatchType(PatchType.UNIFIED); } public Patch.Id getKey() { diff --git a/webapp/src/com/google/gerrit/client/reviewdb/PatchSet.java b/webapp/src/com/google/gerrit/client/reviewdb/PatchSet.java index 5e057d4473..88441adb10 100644 --- a/webapp/src/com/google/gerrit/client/reviewdb/PatchSet.java +++ b/webapp/src/com/google/gerrit/client/reviewdb/PatchSet.java @@ -59,11 +59,6 @@ public final class PatchSet { key = k; } - public PatchSet(final PatchSet.Id k, final RevId rev) { - this(k); - revision = rev; - } - public PatchSet.Id getKey() { return key; } @@ -75,4 +70,8 @@ public final class PatchSet { public RevId getRevision() { return revision; } + + public void setRevision(final RevId i) { + revision = i; + } } diff --git a/webapp/src/com/google/gerrit/client/reviewdb/PatchSetInfo.java b/webapp/src/com/google/gerrit/client/reviewdb/PatchSetInfo.java index e07e0b585b..a9b32d91dd 100644 --- a/webapp/src/com/google/gerrit/client/reviewdb/PatchSetInfo.java +++ b/webapp/src/com/google/gerrit/client/reviewdb/PatchSetInfo.java @@ -58,7 +58,11 @@ public final class PatchSetInfo { } public void setSubject(final String s) { - subject = s; + if (s != null && s.length() > 255) { + subject = s.substring(0, 255); + } else { + subject = s; + } } public String getMessage() { diff --git a/webapp/src/com/google/gerrit/git/GitMetaUtil.java b/webapp/src/com/google/gerrit/git/GitMetaUtil.java index 2bb3ee1331..1457cb3465 100644 --- a/webapp/src/com/google/gerrit/git/GitMetaUtil.java +++ b/webapp/src/com/google/gerrit/git/GitMetaUtil.java @@ -18,8 +18,31 @@ import org.spearce.jgit.lib.Repository; import java.io.File; import java.io.IOException; +import java.util.Random; public class GitMetaUtil { + private static final int SLEEP_MIN = 100; // milliseconds + private static final int SLEEP_MAX = 2 * 60 * 1000; // milliseconds + private static final ThreadLocal<Random> SLEEP_RNG = + new ThreadLocal<Random>() { + @Override + protected Random initialValue() { + return new Random(); + } + }; + + private static int waitTime() { + return SLEEP_MIN + SLEEP_RNG.get().nextInt(SLEEP_MAX - SLEEP_MIN); + } + + public static void randomSleep() { + try { + Thread.sleep(waitTime()); + } catch (InterruptedException ie) { + // Just let the thread continue anyway. + } + } + public static boolean isGitRepository(final File gitdir) { return new File(gitdir, "config").isFile() && new File(gitdir, "HEAD").isFile() diff --git a/webapp/src/com/google/gerrit/git/PatchSetImporter.java b/webapp/src/com/google/gerrit/git/PatchSetImporter.java new file mode 100644 index 0000000000..ef8ebb11ee --- /dev/null +++ b/webapp/src/com/google/gerrit/git/PatchSetImporter.java @@ -0,0 +1,274 @@ +// Copyright 2008 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.git; + +import com.google.gerrit.client.reviewdb.Patch; +import com.google.gerrit.client.reviewdb.PatchContent; +import com.google.gerrit.client.reviewdb.PatchSet; +import com.google.gerrit.client.reviewdb.PatchSetInfo; +import com.google.gerrit.client.reviewdb.RevId; +import com.google.gerrit.client.reviewdb.ReviewDb; +import com.google.gerrit.client.reviewdb.UserIdentity; +import com.google.gwtorm.client.OrmException; +import com.google.gwtorm.client.Transaction; + +import org.spearce.jgit.lib.Commit; +import org.spearce.jgit.lib.Constants; +import org.spearce.jgit.lib.ObjectId; +import org.spearce.jgit.lib.ObjectWriter; +import org.spearce.jgit.lib.PersonIdent; +import org.spearce.jgit.lib.Repository; +import org.spearce.jgit.lib.Tree; +import org.spearce.jgit.patch.CombinedFileHeader; +import org.spearce.jgit.patch.FileHeader; +import org.spearce.jgit.revwalk.RevCommit; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.security.MessageDigest; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +/** Imports a {@link PatchSet} from a {@link Commit}. */ +public class PatchSetImporter { + private static final int MAX_TRIES = 10; + private final ReviewDb db; + private final Repository repo; + private final RevCommit src; + private final PatchSet dst; + private final boolean isNew; + private org.spearce.jgit.patch.Patch gitpatch; + + private PatchSetInfo info; + private boolean infoIsNew; + + private final MessageDigest contentmd = Constants.newMessageDigest(); + private final Map<String, Patch> patchExisting = new HashMap<String, Patch>(); + private final List<Patch> patchInsert = new ArrayList<Patch>(); + private final List<Patch> patchUpdate = new ArrayList<Patch>(); + private final Map<PatchContent.Key, String> content = + new HashMap<PatchContent.Key, String>(); + + public PatchSetImporter(final ReviewDb dstDb, final Repository srcRepo, + final RevCommit srcCommit, final PatchSet dstPatchSet, + final boolean isNewPatchSet) { + db = dstDb; + repo = srcRepo; + src = srcCommit; + dst = dstPatchSet; + isNew = isNewPatchSet; + } + + public void run() throws IOException, OrmException { + gitpatch = readGitPatch(); + + dst.setRevision(new RevId(src.getId().name())); + + if (!isNew) { + // If we aren't a new patch set then we need to load the existing + // files so we can update or delete them if there are corrections. + // + info = db.patchSetInfo().get(dst.getKey()); + for (final Patch p : db.patches().byPatchSet(dst.getKey())) { + patchExisting.put(p.getFileName(), p); + } + } + + importInfo(); + for (final FileHeader fh : gitpatch.getFiles()) { + importFile(fh); + } + + // Ensure all content entities exist + // + putPatchContent(); + + final Transaction txn = db.beginTransaction(); + if (isNew) { + db.patchSets().insert(Collections.singleton(dst)); + } + if (infoIsNew) { + db.patchSetInfo().insert(Collections.singleton(info)); + } else { + db.patchSetInfo().update(Collections.singleton(info)); + } + db.patches().insert(patchInsert, txn); + if (!isNew) { + db.patches().update(patchUpdate, txn); + db.patches().delete(patchExisting.values(), txn); + } + txn.commit(); + } + + private void importInfo() { + if (info == null) { + info = new PatchSetInfo(dst.getKey()); + infoIsNew = true; + } + + info.setSubject(src.getShortMessage()); + info.setMessage(src.getFullMessage()); + info.setAuthor(toUserIdentity(src.getAuthorIdent())); + info.setCommitter(toUserIdentity(src.getCommitterIdent())); + } + + private UserIdentity toUserIdentity(final PersonIdent who) { + final UserIdentity u = new UserIdentity(); + u.setName(who.getName()); + u.setEmail(who.getEmailAddress()); + u.setDate(new Timestamp(who.getWhen().getTime())); + u.setTimeZone(who.getTimeZoneOffset()); + return u; + } + + private void importFile(final FileHeader fh) + throws UnsupportedEncodingException { + final String path; + if (fh.getChangeType() == FileHeader.ChangeType.DELETE) { + path = fh.getOldName(); + } else { + path = fh.getNewName(); + } + + Patch p = patchExisting.remove(path); + if (p == null) { + p = new Patch(new Patch.Id(dst.getKey(), path)); + patchInsert.add(p); + } else { + p.setSourceFileName(null); + patchUpdate.add(p); + } + + // Convert the ChangeType + // + if (fh.getChangeType() == FileHeader.ChangeType.ADD) { + p.setChangeType(Patch.ChangeType.ADD); + + } else if (fh.getChangeType() == FileHeader.ChangeType.MODIFY) { + p.setChangeType(Patch.ChangeType.MODIFIED); + + } else if (fh.getChangeType() == FileHeader.ChangeType.DELETE) { + p.setChangeType(Patch.ChangeType.DELETED); + + } else if (fh.getChangeType() == FileHeader.ChangeType.RENAME) { + p.setChangeType(Patch.ChangeType.RENAMED); + p.setSourceFileName(fh.getOldName()); + + } else if (fh.getChangeType() == FileHeader.ChangeType.COPY) { + p.setChangeType(Patch.ChangeType.COPIED); + p.setSourceFileName(fh.getOldName()); + } + + // Convert the PatchType + // + if (fh instanceof CombinedFileHeader) { + p.setPatchType(Patch.PatchType.N_WAY); + + } else if (fh.getPatchType() == FileHeader.PatchType.GIT_BINARY) { + p.setPatchType(Patch.PatchType.BINARY); + + } else if (fh.getPatchType() == FileHeader.PatchType.BINARY) { + p.setPatchType(Patch.PatchType.BINARY); + } + + // Hash the content. + // + final String contentStr = fh.getScriptText(); + contentmd.reset(); + contentmd.update(contentStr.getBytes("UTF-8")); + final PatchContent.Key contentKey = + new PatchContent.Key(ObjectId.fromRaw(contentmd.digest()).name()); + content.put(contentKey, contentStr); + p.setContent(contentKey); + } + + private void putPatchContent() throws OrmException { + OrmException contentPutError = null; + for (int attempts = 0; !content.isEmpty() && ++attempts < MAX_TRIES;) { + for (final PatchContent pc : db.patchContents().get(content.keySet())) { + content.remove(pc.getKey()); + } + + for (final Iterator<Map.Entry<PatchContent.Key, String>> i = + content.entrySet().iterator(); i.hasNext();) { + final Map.Entry<PatchContent.Key, String> e = i.next(); + final PatchContent pc = new PatchContent(e.getKey(), e.getValue()); + try { + db.patchContents().insert(Collections.singleton(pc)); + i.remove(); + } catch (OrmException err) { + contentPutError = err; + } + } + + if (!content.isEmpty()) { + GitMetaUtil.randomSleep(); + } + } + if (!content.isEmpty() && contentPutError != null) { + throw contentPutError; + } + } + + private org.spearce.jgit.patch.Patch readGitPatch() throws IOException { + final List<String> args = new ArrayList<String>(); + args.add("git"); + args.add("--git-dir=."); + args.add("diff-tree"); + args.add("-M"); + args.add("--full-index"); + + switch (src.getParentCount()) { + case 0: + args.add("--unified=5"); + args.add(new ObjectWriter(repo).writeTree(new Tree(repo)).name()); + args.add(src.getTree().getId().name()); + break; + case 1: + args.add("--unified=5"); + args.add(src.getParent(0).getId().name()); + args.add(src.getId().name()); + break; + default: + args.add("--cc"); + args.add(src.getId().name()); + break; + } + + final Process proc = + Runtime.getRuntime().exec(args.toArray(new String[args.size()]), null, + repo.getDirectory()); + try { + final org.spearce.jgit.patch.Patch p = new org.spearce.jgit.patch.Patch(); + proc.getOutputStream().close(); + proc.getErrorStream().close(); + p.parse(proc.getInputStream()); + proc.getInputStream().close(); + return p; + } finally { + try { + if (proc.waitFor() != 0) { + throw new IOException("git diff-tree exited abnormally"); + } + } catch (InterruptedException ie) { + } + } + } +} diff --git a/webapp/src/com/google/gerrit/pgm/ReimportPatchSets.java b/webapp/src/com/google/gerrit/pgm/ReimportPatchSets.java new file mode 100644 index 0000000000..fa40ddac8a --- /dev/null +++ b/webapp/src/com/google/gerrit/pgm/ReimportPatchSets.java @@ -0,0 +1,117 @@ +// Copyright 2008 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.pgm; + +import com.google.gerrit.client.reviewdb.Change; +import com.google.gerrit.client.reviewdb.PatchSet; +import com.google.gerrit.client.reviewdb.ReviewDb; +import com.google.gerrit.git.InvalidRepositoryException; +import com.google.gerrit.git.PatchSetImporter; +import com.google.gerrit.server.GerritServer; +import com.google.gwtjsonrpc.server.XsrfException; +import com.google.gwtorm.client.OrmException; + +import org.spearce.jgit.lib.ObjectId; +import org.spearce.jgit.lib.Repository; +import org.spearce.jgit.revwalk.RevCommit; +import org.spearce.jgit.revwalk.RevWalk; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.sql.SQLException; +import java.util.ArrayList; + +/** + * Recreates PatchSet and Patch entities for the changes supplied. + * <p> + * Takes on input strings of the form <code>change_id|patch_set_id</code>, such + * as might be created by the following PostgreSQL database dump: + * + * <pre> + * psql reviewdb -tAc 'select change_id,patch_set_id from patch_sets' + * </pre> + * <p> + * For each supplied PatchSet the info and patch entities are completely updated + * based on the data stored in Git. + */ +public class ReimportPatchSets { + public static void main(final String[] argv) throws OrmException, + XsrfException, IOException { + final ArrayList<PatchSet.Id> todo = new ArrayList<PatchSet.Id>(); + final BufferedReader br = + new BufferedReader(new InputStreamReader(System.in)); + String line; + while ((line = br.readLine()) != null) { + final String[] idstr = line.split("\\|"); + todo.add(new PatchSet.Id(Change.Id.fromString(idstr[0]), Integer + .parseInt(idstr[1]))); + } + + final GerritServer gs = GerritServer.getInstance(); + final ReviewDb db = gs.getDatabase().open(); + try { + for (int i = 0; i < todo.size(); i++) { + System.out.print("\rImport " + (i + 1) + " of " + todo.size() + " ..."); + System.out.flush(); + + final PatchSet.Id psid = todo.get(i); + final PatchSet ps = db.patchSets().get(psid); + if (ps == null) { + System.out.println(); + System.err.println("NotFound " + psid); + System.err.flush(); + continue; + } + + final Change c = db.changes().get(ps.getKey().getParentKey()); + if (c == null) { + System.out.println(); + System.err.println("Orphan " + psid); + System.err.flush(); + continue; + } + + final String projectName = c.getDest().getParentKey().get(); + final Repository repo; + try { + repo = gs.getRepositoryCache().get(projectName); + } catch (InvalidRepositoryException ie) { + System.out.println(); + System.err.println("NoProject " + psid); + System.err.println("NoProject " + ie.getMessage()); + System.err.flush(); + continue; + } + + final RevWalk rw = new RevWalk(repo); + final RevCommit src = + rw.parseCommit(ObjectId.fromString(ps.getRevision().get())); + new PatchSetImporter(db, repo, src, ps, false).run(); + } + } catch (OrmException e) { + e.printStackTrace(); + if (e.getCause() instanceof SQLException) { + final SQLException e2 = (SQLException) e.getCause(); + if (e2.getNextException() != null) { + e2.getNextException().printStackTrace(); + } + } + } finally { + System.out.println(); + db.close(); + } + } +} |