gitea/modules/queue/lqinternal/lqinternal.go
wxiaoguang 84c8ab9fd1
Help to recover from corrupted levelqueue (#24912)
gitea.com experienced the corrupted LevelQueue bug again.

I think the problem is clear now: if the keys in LevelDB went
out-of-sync, the LevelQueue itself doesn't have the ability to recover,
eg:

* LevelQueue.Len() reports 100
* LevelQueue.LPop() reports ErrNotFound = errors.New("no key found")

So it needs to dive into the LevelDB to remove all keys to recover the
corrupted LevelQueue.

More comments are in TestCorruptedLevelQueue.
2023-05-29 10:52:32 +08:00

49 lines
1.1 KiB
Go

// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package lqinternal
import (
"bytes"
"encoding/binary"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/opt"
)
func QueueItemIDBytes(id int64) []byte {
buf := make([]byte, 8)
binary.PutVarint(buf, id)
return buf
}
func QueueItemKeyBytes(prefix []byte, id int64) []byte {
key := make([]byte, len(prefix), len(prefix)+1+8)
copy(key, prefix)
key = append(key, '-')
return append(key, QueueItemIDBytes(id)...)
}
func RemoveLevelQueueKeys(db *leveldb.DB, namePrefix []byte) {
keyPrefix := make([]byte, len(namePrefix)+1)
copy(keyPrefix, namePrefix)
keyPrefix[len(namePrefix)] = '-'
it := db.NewIterator(nil, &opt.ReadOptions{Strict: opt.NoStrict})
defer it.Release()
for it.Next() {
if bytes.HasPrefix(it.Key(), keyPrefix) {
_ = db.Delete(it.Key(), nil)
}
}
}
func ListLevelQueueKeys(db *leveldb.DB) (res [][]byte) {
it := db.NewIterator(nil, &opt.ReadOptions{Strict: opt.NoStrict})
defer it.Release()
for it.Next() {
res = append(res, it.Key())
}
return res
}