Prevent persistent task listing interruptions on IOExceptions

When iterating over the list of persisted tasks, it is possible for an
IOException to occur while reading a specific task. Prevent this
exception from breaking out of the iteration by catching and logging the
exception inside the loop instead of outside of it.

Also improve the logging by differentiating between failures that are
severe versus potentially related to other node actions since in a
multi-master scenario with shared storage, it is common for operations
on one node to "interfere" with task listing operations on another node
without causing a malfunction.  Specifically, improve the exception
handling so that the logging in these latter cases have a likely
explanation of the listing error, and do not consider these specific
filesystem errors operational errors.

Change-Id: Ia2ad431c20142ff0ce23dbace34aec837e3d8540

1 files changed, 15 insertions, 6 deletions

diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java
index 39158f3..19becdf 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java
@@ -26,6 +26,7 @@ import com.google.inject.Singleton;
 import java.io.IOException;
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
@@ -108,18 +109,26 @@ public class ReplicationTasksStorage {
   }
 
   public List<ReplicateRefUpdate> list() {
-    ArrayList<ReplicateRefUpdate> result = new ArrayList<>();
+    List<ReplicateRefUpdate> results = new ArrayList<>();
     try (DirectoryStream<Path> events = Files.newDirectoryStream(refUpdates())) {
-      for (Path e : events) {
-        if (Files.isRegularFile(e)) {
-          String json = new String(Files.readAllBytes(e), UTF_8);
-          result.add(GSON.fromJson(json, ReplicateRefUpdate.class));
+      for (Path path : events) {
+        if (Files.isRegularFile(path)) {
+          try {
+            String json = new String(Files.readAllBytes(path), UTF_8);
+            results.add(GSON.fromJson(json, ReplicateRefUpdate.class));
+          } catch (NoSuchFileException ex) {
+            logger.atFine().log(
+                "File %s not found while listing waiting tasks (likely in-flight or completed by another node)",
+                path);
+          } catch (IOException e) {
+            logger.atSevere().withCause(e).log("Error when firing pending event %s", path);
+          }
         }
       }
     } catch (IOException e) {
       logger.atSevere().withCause(e).log("Error when firing pending events");
     }
-    return result;
+    return results;
   }
 
   @SuppressWarnings("deprecation")