mrjn
February 28, 2018, 10:23pm
5
Your HDD is slow in terms of disk seeks. That’s why Go is hitting this limit. You can do a couple of things:
Increase the number of threads to say 10K. In bulk loader code main function, you can add this.
https://golang.org/pkg/runtime/debug/#SetMaxThreads
Modify the bulk loader codebase to set a higher ValueLogThreshold, say 2048 bytes here. So, that most of your values are stored in the LSM tree, and not in value log.
s.shufflePostings(shuffleInputChs, ci)
ci.wait()
thr.Done()
}(i, s.createBadger(i))
}
thr.Wait()
close(s.output)
}
func (s *shuffler) createBadger(i int) *badger.ManagedDB {
opt := badger.DefaultOptions
opt.SyncWrites = false
opt.TableLoadingMode = bo.MemoryMap
opt.Dir = s.opt.shardOutputDirs[i]
opt.ValueDir = opt.Dir
db, err := badger.OpenManaged(opt)
x.Check(err)
s.dbs = append(s.dbs, db)
return db
}
// How should value log be accessed
ValueLogLoadingMode options.FileLoadingMode
// 3. Flags that user might want to review
// ----------------------------------------
// The following affect all levels of LSM tree.
MaxTableSize int64 // Each table (or file) is at most this size.
LevelSizeMultiplier int // Equals SizeOf(Li+1)/SizeOf(Li).
MaxLevels int // Maximum number of levels of compaction.
// If value size >= this threshold, only store value offsets in tree.
ValueThreshold int
// Maximum number of tables to keep in memory, before stalling.
NumMemtables int
// The following affect how we handle LSM tree L0.
// Maximum number of Level 0 tables before we start compacting.
NumLevelZeroTables int
// If we hit this number of Level 0 tables, we will stall until L0 is
// compacted away.
NumLevelZeroTablesStall int
You could