EventIndex: Handle null tokens in the crawler loop as well.

This is similar to 5eb510387c. But now the
checkpoint arrived during a crawl.

src/indexing/EventIndex.js

@@ -489,14 +489,20 @@ export default class EventIndex extends EventEmitter {
                 return object;
             });
 
-            // Create a new checkpoint so we can continue crawling the room for
+            let newCheckpoint;
-            // messages.
+
-            const newCheckpoint = {
+            // The token can be null for some reason. Don't create a checkpoint
-                roomId: checkpoint.roomId,
+            // in that case since adding it to the db will fail.
-                token: res.end,
+            if (res.end) {
-                fullCrawl: checkpoint.fullCrawl,
+                // Create a new checkpoint so we can continue crawling the room
-                direction: checkpoint.direction,
+                // for messages.
-            };
+                newCheckpoint = {
+                    roomId: checkpoint.roomId,
+                    token: res.end,
+                    fullCrawl: checkpoint.fullCrawl,
+                    direction: checkpoint.direction,
+                };
+            }
 
             try {
                 for (let i = 0; i < redactionEvents.length; i++) {
@@ -506,6 +512,15 @@ export default class EventIndex extends EventEmitter {
 
                 const eventsAlreadyAdded = await indexManager.addHistoricEvents(
                     events, newCheckpoint, checkpoint);
+
+                // We didn't get a valid new checkpoint from the server, nothing
+                // to do here anymore.
+                if (!newCheckpoint) {
+                    console.log("EventIndex: The server didn't return a valid ",
+                                "new checkpoint, not continuing the crawl.", checkpoint);
+                    continue
+                }
+
                 // If all events were already indexed we assume that we catched
                 // up with our index and don't need to crawl the room further.
                 // Let us delete the checkpoint in that case, otherwise push