Any simplified ManagedDB docs?

Users Badger
1

I’m still working on trying to implement Blevesearch to use BadgerDB; so far I’ve gotten all the tests passing by referring to Dgraph’s implementation of ManagedDB into a side library that bridges Blevesearch and BadgerDB.

The issue I’m having now is that I’m getting a memory leak somewhere and the application’s memory will grow exponentially leading to an out of memory panic eventually.

I’ve tried setting:

opt.TableLoadingMode = options.FileIO
opt.ValueLogLoadingMode = options.FileIO

Though doesn’t seem to fix anything; still grows pretty quickly. I have a feeling I might not be periodically calling something that’s doing garbage collection (or such)…

I already have goroutines handling the GC:

rv.vlogTicker = time.NewTicker(1 * time.Minute)
rv.mandatoryVlogTicker = time.NewTicker(5 * time.Minute)
go rv.runVlogGC()

runVlogGC() is pretty much the same as the Dgraph one:

func (s *Store) runVlogGC() {
	// Get initial size on start
	_, lastVlogSize := s.db.Size()

	runGC := func() {
		var err error
		for err == nil {
			// If a GC was successful, immediately run it again
			err = s.db.RunValueLogGC(0.7)
		}
		_, lastVlogSize = s.db.Size()
	}

	for {
		select {
		case <-s.vlogTicker.C:
			_, currentVlogSize := s.db.Size()
			if currentVlogSize < lastVlogSize+(100*MB) {
				continue
			}
			runGC()
		case <-s.mandatoryVlogTicker.C:
			runGC()
		}
	}
}
2 
package main

import (
    "fmt"
    "time"

    "github.com/dgraph-io/badger/v2"
    "github.com/dgraph-io/badger/v2/options"
)

const MB = 1 << 20 // 1 MB in bytes

// Store holds BadgerDB instance and related timers for GC
type Store struct {
    db                    *badger.DB
    vlogTicker            *time.Ticker
    mandatoryVlogTicker   *time.Ticker
}

// NewStore initializes a new Store with BadgerDB
func NewStore(dir string) (*Store, error) {
    opt := badger.DefaultOptions(dir)
    // These settings might help with memory usage; adjust based on your needs
    opt.TableLoadingMode = options.FileIO
    opt.ValueLogLoadingMode = options.FileIO

    db, err := badger.Open(opt)
    if err != nil {
        return nil, err
    }

    s := &Store{
        db:                    db,
        vlogTicker:            time.NewTicker(1 * time.Minute),
        mandatoryVlogTicker:   time.NewTicker(5 * time.Minute),
    }

    // Start the GC goroutine
    go s.runVlogGC()

    return s, nil
}

// runVlogGC runs periodic garbage collection on the value log
func (s *Store) runVlogGC() {
    // Get initial size on start
    _, lastVlogSize := s.db.Size()

    runGC := func() {
        var err error
        for err == nil {
            // If a GC was successful, immediately run it again
            err = s.db.RunValueLogGC(0.7)
        }
        _, lastVlogSize = s.db.Size()
    }

    for {
        select {
        case <-s.vlogTicker.C:
            _, currentVlogSize := s.db.Size()
            if currentVlogSize < lastVlogSize+(100*MB) {
                continue
            }
            runGC()
        case <-s.mandatoryVlogTicker.C:
            runGC()
        }
    }
}

// Close stops the timers and closes the BadgerDB instance
func (s *Store) Close() error {
    s.vlogTicker.Stop()
    s.mandatoryVlogTicker.Stop()
    return s.db.Close()
}

func main() {
    // Example usage
    store, err := NewStore("path/to/badgerdb")
    if err != nil {
        fmt.Println("Failed to initialize BadgerDB:", err)
        return
    }
    defer store.Close()

    // Simulate some work with BadgerDB
    err = store.db.Update(func(txn *badger.Txn) error {
        return txn.Set([]byte("key"), []byte("value"))
    })
    if err != nil {
        fmt.Println("Failed to set key:", err)
    }

    // Wait for a while to let GC run
    time.Sleep(10 * time.Minute)
    fmt.Println("Application exiting, BadgerDB closed.")
}