|
| 1 | +package memiavl |
| 2 | + |
| 3 | +import ( |
| 4 | + "bytes" |
| 5 | + "encoding/binary" |
| 6 | + "fmt" |
| 7 | + "io/ioutil" |
| 8 | + "os" |
| 9 | + "path/filepath" |
| 10 | + "unsafe" |
| 11 | + |
| 12 | + "github.com/tidwall/gjson" |
| 13 | + "github.com/tidwall/wal" |
| 14 | +) |
| 15 | + |
| 16 | +// OpenWAL opens the write ahead log, try to truncate the corrupted tail if there's any |
| 17 | +// TODO fix in upstream: https://github.com/tidwall/wal/pull/22 |
| 18 | +func OpenWAL(dir string, opts *wal.Options) (*wal.Log, error) { |
| 19 | + log, err := wal.Open(dir, opts) |
| 20 | + if err == wal.ErrCorrupt { |
| 21 | + // try to truncate corrupted tail |
| 22 | + var fis []os.DirEntry |
| 23 | + fis, err = os.ReadDir(dir) |
| 24 | + if err != nil { |
| 25 | + return nil, fmt.Errorf("read wal dir fail: %w", err) |
| 26 | + } |
| 27 | + var lastSeg string |
| 28 | + for _, fi := range fis { |
| 29 | + if fi.IsDir() || len(fi.Name()) < 20 { |
| 30 | + continue |
| 31 | + } |
| 32 | + lastSeg = fi.Name() |
| 33 | + } |
| 34 | + |
| 35 | + if len(lastSeg) == 0 { |
| 36 | + return nil, err |
| 37 | + } |
| 38 | + if err = truncateCorruptedTail(filepath.Join(dir, lastSeg), opts.LogFormat); err != nil { |
| 39 | + return nil, fmt.Errorf("truncate corrupted tail fail: %w", err) |
| 40 | + } |
| 41 | + |
| 42 | + // try again |
| 43 | + return wal.Open(dir, opts) |
| 44 | + } |
| 45 | + |
| 46 | + return log, err |
| 47 | +} |
| 48 | + |
| 49 | +func truncateCorruptedTail(path string, format wal.LogFormat) error { |
| 50 | + data, err := ioutil.ReadFile(path) |
| 51 | + if err != nil { |
| 52 | + return err |
| 53 | + } |
| 54 | + var pos int |
| 55 | + for len(data) > 0 { |
| 56 | + var n int |
| 57 | + if format == wal.JSON { |
| 58 | + n, err = loadNextJSONEntry(data) |
| 59 | + } else { |
| 60 | + n, err = loadNextBinaryEntry(data) |
| 61 | + } |
| 62 | + if err == wal.ErrCorrupt { |
| 63 | + break |
| 64 | + } |
| 65 | + if err != nil { |
| 66 | + return err |
| 67 | + } |
| 68 | + data = data[n:] |
| 69 | + pos += n |
| 70 | + } |
| 71 | + if pos != len(data) { |
| 72 | + return os.Truncate(path, int64(pos)) |
| 73 | + } |
| 74 | + return nil |
| 75 | +} |
| 76 | + |
| 77 | +func loadNextJSONEntry(data []byte) (n int, err error) { |
| 78 | + // {"index":number,"data":string} |
| 79 | + idx := bytes.IndexByte(data, '\n') |
| 80 | + if idx == -1 { |
| 81 | + return 0, wal.ErrCorrupt |
| 82 | + } |
| 83 | + line := data[:idx] |
| 84 | + dres := gjson.Get(*(*string)(unsafe.Pointer(&line)), "data") |
| 85 | + if dres.Type != gjson.String { |
| 86 | + return 0, wal.ErrCorrupt |
| 87 | + } |
| 88 | + return idx + 1, nil |
| 89 | +} |
| 90 | + |
| 91 | +func loadNextBinaryEntry(data []byte) (n int, err error) { |
| 92 | + // data_size + data |
| 93 | + size, n := binary.Uvarint(data) |
| 94 | + if n <= 0 { |
| 95 | + return 0, wal.ErrCorrupt |
| 96 | + } |
| 97 | + if uint64(len(data)-n) < size { |
| 98 | + return 0, wal.ErrCorrupt |
| 99 | + } |
| 100 | + return n + int(size), nil |
| 101 | +} |
0 commit comments