if repo base has a 'prev', ignore block diff errors

backfill/backfill.go

···
 	// Producer routine
 	go func() {
 		defer close(recordQueue)
-		r.ForEach(ctx, b.NSIDFilter, func(recordPath string, nodeCid cid.Cid) error {
+		if err := r.ForEach(ctx, b.NSIDFilter, func(recordPath string, nodeCid cid.Cid) error {
 			numRecords++
 			recordQueue <- recordQueueItem{recordPath: recordPath, nodeCid: nodeCid}
 			return nil
-		})
+		}); err != nil {
+			log.Error("failed to iterated records in repo", "err", err)
+		}
 	}()
 
 	// Consumer routines

carstore/bs.go

···
 	return out
 }
 
-func BlockDiff(ctx context.Context, bs blockstore.Blockstore, oldroot cid.Cid, newcids map[cid.Cid]blockformat.Block) (map[cid.Cid]bool, error) {
+func BlockDiff(ctx context.Context, bs blockstore.Blockstore, oldroot cid.Cid, newcids map[cid.Cid]blockformat.Block, skipmissing bool) (map[cid.Cid]bool, error) {
 	ctx, span := otel.Tracer("repo").Start(ctx, "BlockDiff")
 	defer span.End()
 
···
 
 		oblk, err := bs.Get(ctx, c)
 		if err != nil {
-			return nil, fmt.Errorf("get failed in old tree: %w", err)
+			if skipmissing && ipld.IsNotFound(err) {
+				log.Warnw("missing block in old tree", "root", oldroot, "missing", c)
+			} else {
+				return nil, fmt.Errorf("get failed in old tree: %w", err)
+			}
 		}
 
 		if err := cbg.ScanForLinks(bytes.NewReader(oblk.RawData()), func(lnk cid.Cid) {
···
 		}
 	}
 
-	rmcids, err := BlockDiff(ctx, ds, ds.baseCid, ds.blks)
-	if err != nil {
-		return cid.Undef, nil, fmt.Errorf("block diff failed (base=%s,since=%v,rev=%s): %w", ds.baseCid, since, ds.lastRev, err)
-	}
-
-	ds.rmcids = rmcids
-
 	return carr.Header.Roots[0], ds, nil
 }
 
-func (ds *DeltaSession) CalcDiff(ctx context.Context, nroot cid.Cid) error {
-	rmcids, err := BlockDiff(ctx, ds, ds.baseCid, ds.blks)
+func (ds *DeltaSession) CalcDiff(ctx context.Context, skipmissing bool) error {
+	rmcids, err := BlockDiff(ctx, ds, ds.baseCid, ds.blks, skipmissing)
 	if err != nil {
-		return fmt.Errorf("block diff failed: %w", err)
+		return fmt.Errorf("block diff failed (base=%s,rev=%s): %w", ds.baseCid, ds.lastRev, err)
 	}
 
 	ds.rmcids = rmcids

carstore/repo_test.go

···
 			t.Fatal(err)
 		}
 
-		rr, err := repo.OpenRepo(ctx, ds, head, true)
+		rr, err := repo.OpenRepo(ctx, ds, head)
 		if err != nil {
 			t.Fatal(err)
 		}
···
 
 		rev = nrev
 
-		if err := ds.CalcDiff(ctx, nroot); err != nil {
+		if err := ds.CalcDiff(ctx, false); err != nil {
 			t.Fatal(err)
 		}
 
···
 				t.Fatal(err)
 			}
 
-			rr, err := repo.OpenRepo(ctx, ds, head, true)
+			rr, err := repo.OpenRepo(ctx, ds, head)
 			if err != nil {
 				t.Fatal(err)
 			}
···
 
 			rev = nrev
 
-			if err := ds.CalcDiff(ctx, nroot); err != nil {
+			if err := ds.CalcDiff(ctx, false); err != nil {
 				t.Fatal(err)
 			}
 
···
 			b.Fatal(err)
 		}
 
-		rr, err := repo.OpenRepo(ctx, ds, head, true)
+		rr, err := repo.OpenRepo(ctx, ds, head)
 		if err != nil {
 			b.Fatal(err)
 		}
···
 		}
 
 		rev = nrev
-		if err := ds.CalcDiff(ctx, nroot); err != nil {
+		if err := ds.CalcDiff(ctx, false); err != nil {
 			b.Fatal(err)
 		}
 
···
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 
-		rr, err := repo.OpenRepo(ctx, bs, head, true)
+		rr, err := repo.OpenRepo(ctx, bs, head)
 		if err != nil {
 			b.Fatal(err)
 		}
···
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 
-		rr, err := repo.OpenRepo(ctx, bs, head, true)
+		rr, err := repo.OpenRepo(ctx, bs, head)
 		if err != nil {
 			b.Fatal(err)
 		}

cmd/gosky/main.go

···
 		}
 	}
 
-	r, err := repo.OpenRepo(ctx, bstore, carr.Header.Roots[0], false)
+	r, err := repo.OpenRepo(ctx, bstore, carr.Header.Roots[0])
 	if err != nil {
 		return nil, err
 	}

repo/repo.go

···
 		return nil, err
 	}
 
-	return OpenRepo(ctx, bs, root, false)
+	return OpenRepo(ctx, bs, root)
 }
 
 func NewRepo(ctx context.Context, did string, bs blockstore.Blockstore) *Repo {
···
 	}
 }
 
-func OpenRepo(ctx context.Context, bs blockstore.Blockstore, root cid.Cid, fullRepo bool) (*Repo, error) {
+func OpenRepo(ctx context.Context, bs blockstore.Blockstore, root cid.Cid) (*Repo, error) {
 	cst := util.CborStore(bs)
 
 	var sc SignedCommit
···
 
 	var oldTree cid.Cid
 	if oldrepo.Defined() {
-		otherRepo, err := OpenRepo(ctx, r.bs, oldrepo, true)
+		otherRepo, err := OpenRepo(ctx, r.bs, oldrepo)
 		if err != nil {
 			return nil, err
 		}

repomgr/repomgr.go

···
 
 	head := ds.BaseCid()
 
-	r, err := repo.OpenRepo(ctx, ds, head, true)
+	r, err := repo.OpenRepo(ctx, ds, head)
 	if err != nil {
 		return "", cid.Undef, err
 	}
···
 	}
 
 	head := ds.BaseCid()
-	r, err := repo.OpenRepo(ctx, ds, head, true)
+	r, err := repo.OpenRepo(ctx, ds, head)
 	if err != nil {
 		return cid.Undef, err
 	}
···
 	}
 
 	head := ds.BaseCid()
-	r, err := repo.OpenRepo(ctx, ds, head, true)
+	r, err := repo.OpenRepo(ctx, ds, head)
 	if err != nil {
 		return err
 	}
···
 		return cid.Undef, nil, err
 	}
 
-	r, err := repo.OpenRepo(ctx, bs, head, true)
+	r, err := repo.OpenRepo(ctx, bs, head)
 	if err != nil {
 		return cid.Undef, nil, err
 	}
···
 		return cid.Undef, nil, err
 	}
 
-	r, err := repo.OpenRepo(ctx, bs, head, true)
+	r, err := repo.OpenRepo(ctx, bs, head)
 	if err != nil {
 		return cid.Undef, nil, err
 	}
···
 		return nil, err
 	}
 
-	r, err := repo.OpenRepo(ctx, bs, head, true)
+	r, err := repo.OpenRepo(ctx, bs, head)
 	if err != nil {
 		return nil, err
 	}
···
 		return fmt.Errorf("importing external carslice: %w", err)
 	}
 
-	r, err := repo.OpenRepo(ctx, ds, root, true)
+	r, err := repo.OpenRepo(ctx, ds, root)
 	if err != nil {
 		return fmt.Errorf("opening external user repo (%d, root=%s): %w", uid, root, err)
 	}
···
 		return err
 	}
 
+	var badPrev bool
+	if ds.BaseCid().Defined() {
+		oldrepo, err := repo.OpenRepo(ctx, ds, ds.BaseCid())
+		if err != nil {
+			return fmt.Errorf("failed to check data root in old repo: %w", err)
+		}
+
+		// if the old commit has a 'prev', CalcDiff will error out while trying
+		// to walk it. This is an old repo thing that is being deprecated.
+		// This check is a temporary workaround until all repos get migrated
+		// and this becomes no longer an issue
+		prev, _ := oldrepo.PrevCommit(ctx)
+		if prev != nil {
+			badPrev = true
+		}
+	}
+
+	if err := ds.CalcDiff(ctx, badPrev); err != nil {
+		return fmt.Errorf("failed while calculating mst diff (since=%v): %w", since, err)
+
+	}
+
 	var evtops []RepoOp
 
 	for _, op := range ops {
···
 	}
 
 	head := ds.BaseCid()
-	r, err := repo.OpenRepo(ctx, ds, head, true)
+	r, err := repo.OpenRepo(ctx, ds, head)
 	if err != nil {
 		return err
 	}
···
 	}
 
 	err = rm.processNewRepo(ctx, user, r, rev, func(ctx context.Context, root cid.Cid, finish func(context.Context, string) ([]byte, error), bs blockstore.Blockstore) error {
-		r, err := repo.OpenRepo(ctx, bs, root, true)
+		r, err := repo.OpenRepo(ctx, bs, root)
 		if err != nil {
 			return fmt.Errorf("opening new repo: %w", err)
 		}

testing/sig_test.go

···
 		t.Fatal(err)
 	}
 
-	r, err := repo.OpenRepo(ctx, bs, c, true)
+	r, err := repo.OpenRepo(ctx, bs, c)
 	if err != nil {
 		t.Fatal(err)
 	}