janardhan1993 commented :
@lxwithgod : Thanks for sharing the code. One last thing can you please share the output of /debug/vars and heap profile when memory reaches 80% of available RAM.
https://docs.dgraph.io/master/howto/#retrieving-debug-information
lxwithgod commented :
{
“badger_blocked_puts_total”: 0,
“badger_disk_reads_total”: 12,
“badger_disk_writes_total”: 1385,
“badger_gets_total”: 8110019,
“badger_lsm_bloom_hits_total”: {“l0”: 12007729, “l1”: 641892},
“badger_lsm_level_gets_total”: {“l0”: 4896, “l1”: 375},
“badger_lsm_size_bytes”: {“p”: 152563008, “w”: 0},
“badger_memtable_gets_total”: 8186154,
“badger_pending_writes_total”: {“p”: 41, “w”: 0},
“badger_puts_total”: 7630116,
“badger_read_bytes”: 395857637,
“badger_vlog_size_bytes”: {“p”: 349571469, “w”: 161064996},
“badger_written_bytes”: 742364930,
“cmdline”: [“dgraph”,“server”,"–memory_mb",“10240”,"–zero",“localhost:5080”],
“dgraph-bulk-loader_badger_writes_pending”: 0,
“dgraph-bulk-loader_num_reducers_total”: 0,
“dgraph-bulk-loader_reduce_queue_size”: 0,
“dgraph_active_mutations_total”: 33699,
“dgraph_cache_hits_total”: 16074390,
“dgraph_cache_miss_total”: 8110014,
“dgraph_cache_race_total”: 0,
“dgraph_config”: {“allotted_memory”: 10240, “expand_edge”: 1, “max_pending_count”: 1000, “num_pending_proposals”: 2000, “posting_dir”: “p”, “posting_tables”: “loadtoram”, “tracing”: 0, “wal_dir”: “w”},
“dgraph_dirtymap_keys_total”: 0,
“dgraph_evicted_lists_total”: 1361459,
“dgraph_goroutines_total”: 3813,
“dgraph_heap_idle_bytes”: 3613687808,
“dgraph_lcache_capacity_bytes”: 457400000,
“dgraph_lcache_keys_total”: 6699833,
“dgraph_lcache_size_bytes”: 457646889,
“dgraph_max_list_bytes”: 67,
“dgraph_max_list_length”: 1,
“dgraph_memory_inuse_bytes”: 9634676736,
“dgraph_num_queries_total”: 0,
“dgraph_pending_proposals_total”: 84,
“dgraph_pending_queries_total”: 0,
“dgraph_posting_reads_total”: 0,
“dgraph_posting_writes_total”: 1414266,
“dgraph_predicate_stats”: {“predicate”: 2703283, “i.phone”: 2703018, “phone”: 2703283},
“dgraph_proc_memory_bytes”: 13555494912,
“dgraph_read_bytes_total”: 0,
“dgraph_server_health_status”: 1,
“dgraph_written_bytes_total”: 72839450,
“memstats”: {“Alloc”:6559036552,“TotalAlloc”:44173112280,“Sys”:14035085200,“Lookups”:451,“Mallocs”:701022117,“Frees”:616707233,“HeapAlloc”:6559036552,“HeapSys”:13121978368,“HeapIdle”:5493284864,“HeapInuse”:7628693504,“HeapReleased”:0,“HeapObjects”:84314884,“StackInuse”:125534208,“StackSys”:125534208,“MSpanInuse”:140982432,“MSpanSys”:193069056,“MCacheInuse”:222208,“MCacheSys”:245760,“BuckHashSys”:1657989,“GCSys”:542093312,“OtherSys”:50506507,“NextGC”:11857892560,“LastGC”:1512097902706964297,“PauseTotalNs”:38318744,“PauseNs”:[318155,159183,1535297,403163,255148,199212,147509,318886,249136,252670,489968,364717,260761,307410,299055,317935,456174,204879,483990,246440,519362,1726344,1305007,27498343,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],“PauseEnd”:[1512097514261310262,1512097514277459172,1512097514296045101,1512097634314702543,1512097664401341936,1512097665589561370,1512097666575678918,1512097667903413644,1512097669982800540,1512097672244883614,1512097674091234587,1512097679275080888,1512097682900170695,1512097687499888359,1512097695887723720,1512097701912343875,1512097711196394798,1512097726761633160,1512097740684397038,1512097755919927642,1512097774022130087,1512097822936959524,1512097853241033045,1512097902706964297,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],“NumGC”:24,“NumForcedGC”:0,“GCCPUFraction”:0.018990245108040225,“EnableGC”:true,“DebugGC”:false,“BySize”:[{“Size”:0,“Mallocs”:0,“Frees”:0},{“Size”:8,“Mallocs”:39254861,“Frees”:31593244},{“Size”:16,“Mallocs”:188453444,“Frees”:182234900},{“Size”:32,“Mallocs”:222379935,“Frees”:197069809},{“Size”:48,“Mallocs”:36974096,“Frees”:22657142},{“Size”:64,“Mallocs”:17577611,“Frees”:17292705},{“Size”:80,“Mallocs”:42953440,“Frees”:27481193},{“Size”:96,“Mallocs”:733679,“Frees”:586580},{“Size”:112,“Mallocs”:4084530,“Frees”:4046398},{“Size”:128,“Mallocs”:32726475,“Frees”:25311434},{“Size”:144,“Mallocs”:19212562,“Frees”:19005228},{“Size”:160,“Mallocs”:8113416,“Frees”:1364492},{“Size”:176,“Mallocs”:10834110,“Frees”:10754889},{“Size”:192,“Mallocs”:1531,“Frees”:1364},{“Size”:208,“Mallocs”:2077944,“Frees”:2024569},{“Size”:224,“Mallocs”:356,“Frees”:336},{“Size”:240,“Mallocs”:24907,“Frees”:22330},{“Size”:256,“Mallocs”:578657,“Frees”:577962},{“Size”:288,“Mallocs”:2793650,“Frees”:2793576},{“Size”:320,“Mallocs”:2711,“Frees”:2427},{“Size”:352,“Mallocs”:715,“Frees”:692},{“Size”:384,“Mallocs”:301655,“Frees”:62},{“Size”:416,“Mallocs”:283,“Frees”:267},{“Size”:448,“Mallocs”:74,“Frees”:71},{“Size”:480,“Mallocs”:80,“Frees”:80},{“Size”:512,“Mallocs”:1510,“Frees”:1274},{“Size”:576,“Mallocs”:723,“Frees”:664},{“Size”:640,“Mallocs”:1479,“Frees”:1216},{“Size”:704,“Mallocs”:314,“Frees”:303},{“Size”:768,“Mallocs”:69,“Frees”:67},{“Size”:896,“Mallocs”:396,“Frees”:256},{“Size”:1024,“Mallocs”:1452,“Frees”:1252},{“Size”:1152,“Mallocs”:624,“Frees”:560},{“Size”:1280,“Mallocs”:1162,“Frees”:843},{“Size”:1408,“Mallocs”:11,“Frees”:6},{“Size”:1536,“Mallocs”:137,“Frees”:136},{“Size”:1792,“Mallocs”:248,“Frees”:240},{“Size”:2048,“Mallocs”:1416105,“Frees”:1362619},{“Size”:2304,“Mallocs”:270,“Frees”:261},{“Size”:2688,“Mallocs”:1432,“Frees”:1236},{“Size”:3072,“Mallocs”:63,“Frees”:62},{“Size”:3200,“Mallocs”:3,“Frees”:0},{“Size”:3456,“Mallocs”:77,“Frees”:77},{“Size”:4096,“Mallocs”:1375,“Frees”:1209},{“Size”:4864,“Mallocs”:536,“Frees”:398},{“Size”:5376,“Mallocs”:1442,“Frees”:1324},{“Size”:6144,“Mallocs”:183,“Frees”:173},{“Size”:6528,“Mallocs”:10,“Frees”:0},{“Size”:6784,“Mallocs”:10,“Frees”:10},{“Size”:6912,“Mallocs”:0,“Frees”:0},{“Size”:8192,“Mallocs”:1221,“Frees”:1179},{“Size”:9472,“Mallocs”:142,“Frees”:138},{“Size”:9728,“Mallocs”:3,“Frees”:1},{“Size”:10240,“Mallocs”:814,“Frees”:758},{“Size”:10880,“Mallocs”:948,“Frees”:914},{“Size”:12288,“Mallocs”:173,“Frees”:158},{“Size”:13568,“Mallocs”:447,“Frees”:445},{“Size”:14336,“Mallocs”:862,“Frees”:844},{“Size”:16384,“Mallocs”:14267,“Frees”:14243},{“Size”:18432,“Mallocs”:879,“Frees”:870},{“Size”:19072,“Mallocs”:68,“Frees”:68}]}
}
lxwithgod commented :
{
“badger_blocked_puts_total”: 0,
“badger_disk_reads_total”: 21,
“badger_disk_writes_total”: 2971,
“badger_gets_total”: 10528815,
“badger_lsm_bloom_hits_total”: {“l0”: 17589935, “l1”: 3054179},
“badger_lsm_level_gets_total”: {“l0”: 7298, “l1”: 6884},
“badger_lsm_size_bytes”: {“p”: 757289956, “w”: 0},
“badger_memtable_gets_total”: 10606065,
“badger_pending_writes_total”: {“p”: 0, “w”: 0},
“badger_puts_total”: 15350093,
“badger_read_bytes”: 835373978,
“badger_vlog_size_bytes”: {“p”: 1015641222, “w”: 219164549},
“badger_written_bytes”: 1360298440,
“cmdline”: [“dgraph”,“server”,"–memory_mb",“10240”,"–zero",“localhost:5080”],
“dgraph-bulk-loader_badger_writes_pending”: 0,
“dgraph-bulk-loader_num_reducers_total”: 0,
“dgraph-bulk-loader_reduce_queue_size”: 0,
“dgraph_active_mutations_total”: 21230,
“dgraph_cache_hits_total”: 42963087,
“dgraph_cache_miss_total”: 10528810,
“dgraph_cache_race_total”: 0,
“dgraph_config”: {“allotted_memory”: 10240, “expand_edge”: 1, “max_pending_count”: 1000, “num_pending_proposals”: 2000, “posting_dir”: “p”, “posting_tables”: “loadtoram”, “tracing”: 0, “wal_dir”: “w”},
“dgraph_dirtymap_keys_total”: 0,
“dgraph_evicted_lists_total”: 3916984,
“dgraph_goroutines_total”: 1275,
“dgraph_heap_idle_bytes”: 3904512000,
“dgraph_lcache_capacity_bytes”: 457400000,
“dgraph_lcache_keys_total”: 6523016,
“dgraph_lcache_size_bytes”: 457287464,
“dgraph_max_list_bytes”: 67,
“dgraph_max_list_length”: 1,
“dgraph_memory_inuse_bytes”: 10043514880,
“dgraph_num_queries_total”: 0,
“dgraph_pending_proposals_total”: 102,
“dgraph_pending_queries_total”: 0,
“dgraph_posting_reads_total”: 0,
“dgraph_posting_writes_total”: 4134594,
“dgraph_predicate_stats”: {“predicate”: 3509484, “i.phone”: 3509286, “phone”: 3509529},
“dgraph_proc_memory_bytes”: 14479511552,
“dgraph_read_bytes_total”: 0,
“dgraph_server_health_status”: 1,
“dgraph_written_bytes_total”: 213872122,
“memstats”: {“Alloc”:8786298216,“TotalAlloc”:81209651120,“Sys”:14759917456,“Lookups”:659,“Mallocs”:1259602311,“Frees”:1151595233,“HeapAlloc”:8786298216,“HeapSys”:13607108608,“HeapIdle”:4150550528,“HeapInuse”:9456558080,“HeapReleased”:0,“HeapObjects”:108007078,“StackInuse”:340918272,“StackSys”:340918272,“MSpanInuse”:146170648,“MSpanSys”:193069056,“MCacheInuse”:222208,“MCacheSys”:245760,“BuckHashSys”:1686525,“GCSys”:564903936,“OtherSys”:51985299,“NextGC”:11854313952,“LastGC”:1512098133036066693,“PauseTotalNs”:671362948,“PauseNs”:[318155,159183,1535297,403163,255148,199212,147509,318886,249136,252670,489968,364717,260761,307410,299055,317935,456174,204879,483990,246440,519362,1726344,1305007,27498343,19292043,11857593,15123489,9630876,564572493,12567710,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],“PauseEnd”:[1512097514261310262,1512097514277459172,1512097514296045101,1512097634314702543,1512097664401341936,1512097665589561370,1512097666575678918,1512097667903413644,1512097669982800540,1512097672244883614,1512097674091234587,1512097679275080888,1512097682900170695,1512097687499888359,1512097695887723720,1512097701912343875,1512097711196394798,1512097726761633160,1512097740684397038,1512097755919927642,1512097774022130087,1512097822936959524,1512097853241033045,1512097902706964297,1512097945581486380,1512097982641146184,1512098026130239179,1512098062738689170,1512098092942670091,1512098133036066693,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],“NumGC”:30,“NumForcedGC”:0,“GCCPUFraction”:0.020783923356994482,“EnableGC”:true,“DebugGC”:false,“BySize”:[{“Size”:0,“Mallocs”:0,“Frees”:0},{“Size”:8,“Mallocs”:61749724,“Frees”:52602623},{“Size”:16,“Mallocs”:320713688,“Frees”:304085758},{“Size”:32,“Mallocs”:384608909,“Frees”:353070100},{“Size”:48,“Mallocs”:68216339,“Frees”:53284590},{“Size”:64,“Mallocs”:55909167,“Frees”:54662856},{“Size”:80,“Mallocs”:85915506,“Frees”:69754736},{“Size”:96,“Mallocs”:2200577,“Frees”:1976353},{“Size”:112,“Mallocs”:7500968,“Frees”:7257550},{“Size”:128,“Mallocs”:51542763,“Frees”:43085172},{“Size”:144,“Mallocs”:49229643,“Frees”:47964031},{“Size”:160,“Mallocs”:10535510,“Frees”:3860378},{“Size”:176,“Mallocs”:18484471,“Frees”:17904960},{“Size”:192,“Mallocs”:2394,“Frees”:2261},{“Size”:208,“Mallocs”:4939937,“Frees”:4689122},{“Size”:224,“Mallocs”:505,“Frees”:471},{“Size”:240,“Mallocs”:31099,“Frees”:29206},{“Size”:256,“Mallocs”:2344070,“Frees”:2343677},{“Size”:288,“Mallocs”:3616113,“Frees”:3492878},{“Size”:320,“Mallocs”:5886,“Frees”:5524},{“Size”:352,“Mallocs”:1005,“Frees”:956},{“Size”:384,“Mallocs”:301687,“Frees”:86},{“Size”:416,“Mallocs”:392,“Frees”:366},{“Size”:448,“Mallocs”:95,“Frees”:89},{“Size”:480,“Mallocs”:123,“Frees”:116},{“Size”:512,“Mallocs”:2650,“Frees”:2462},{“Size”:576,“Mallocs”:1694,“Frees”:1670},{“Size”:640,“Mallocs”:3987,“Frees”:3741},{“Size”:704,“Mallocs”:429,“Frees”:407},{“Size”:768,“Mallocs”:100,“Frees”:90},{“Size”:896,“Mallocs”:499,“Frees”:344},{“Size”:1024,“Mallocs”:2479,“Frees”:2302},{“Size”:1152,“Mallocs”:1580,“Frees”:1554},{“Size”:1280,“Mallocs”:3103,“Frees”:2886},{“Size”:1408,“Mallocs”:11,“Frees”:6},{“Size”:1536,“Mallocs”:194,“Frees”:182},{“Size”:1792,“Mallocs”:346,“Frees”:324},{“Size”:2048,“Mallocs”:4136851,“Frees”:3910729},{“Size”:2304,“Mallocs”:378,“Frees”:359},{“Size”:2688,“Mallocs”:4011,“Frees”:3813},{“Size”:3072,“Mallocs”:96,“Frees”:87},{“Size”:3200,“Mallocs”:3,“Frees”:0},{“Size”:3456,“Mallocs”:106,“Frees”:102},{“Size”:4096,“Mallocs”:2183,“Frees”:2025},{“Size”:4864,“Mallocs”:644,“Frees”:498},{“Size”:5376,“Mallocs”:3927,“Frees”:3743},{“Size”:6144,“Mallocs”:260,“Frees”:238},{“Size”:6528,“Mallocs”:10,“Frees”:0},{“Size”:6784,“Mallocs”:23,“Frees”:22},{“Size”:6912,“Mallocs”:0,“Frees”:0},{“Size”:8192,“Mallocs”:1884,“Frees”:1783},{“Size”:9472,“Mallocs”:223,“Frees”:211},{“Size”:9728,“Mallocs”:3,“Frees”:1},{“Size”:10240,“Mallocs”:1830,“Frees”:1792},{“Size”:10880,“Mallocs”:2435,“Frees”:2288},{“Size”:12288,“Mallocs”:235,“Frees”:208},{“Size”:13568,“Mallocs”:620,“Frees”:589},{“Size”:14336,“Mallocs”:2259,“Frees”:2122},{“Size”:16384,“Mallocs”:18751,“Frees”:18084},{“Size”:18432,“Mallocs”:2068,“Frees”:1958},{“Size”:19072,“Mallocs”:88,“Frees”:85}]}
}
janardhan1993 commented :
@lxwithgod: Everything looks good to me, can you please try loading with --memory_mb 5120
janardhan1993 commented :
@lxwithgod: As you can see dgraph RES size is 7987 MB only. The rest is virtual memory
lxwithgod commented :
fatal error: runtime: out of memory
runtime stack:
runtime.throw(0x13106cc, 0x16)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/runtime/panic.go:605 +0x95
runtime.sysMap(0xc466230000, 0x37400000, 0x0, 0x1aa3af8)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/runtime/mem_linux.go:216 +0x1d0
runtime.(*mheap).sysAlloc(0x1a8a5a0, 0x37400000, 0x7f54f47f4bc0)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/runtime/malloc.go:470 +0xd7
runtime.(*mheap).grow(0x1a8a5a0, 0x1ba00, 0x0)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/runtime/mheap.go:887 +0x60
runtime.(*mheap).allocSpanLocked(0x1a8a5a0, 0x1ba00, 0x1aa3b08, 0x7f558800e3a8)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/runtime/mheap.go:800 +0x334
runtime.(*mheap).alloc_m(0x1a8a5a0, 0x1ba00, 0x7f54737f0100, 0x8c370c)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/runtime/mheap.go:666 +0x118
runtime.(*mheap).alloc.func1()
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/runtime/mheap.go:733 +0x4d
runtime.systemstack(0x7f54737fde10)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/runtime/asm_amd64.s:360 +0xab
runtime.(*mheap).alloc(0x1a8a5a0, 0x1ba00, 0x7f5473010100, 0x8c2d74)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/runtime/mheap.go:732 +0xa1
runtime.largeAlloc(0x37400000, 0x7f5594030001, 0xc438629a1c)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/runtime/malloc.go:827 +0x98
runtime.mallocgc.func1()
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/runtime/malloc.go:722 +0x46
runtime.systemstack(0xc420402600)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/runtime/asm_amd64.s:344 +0x79
runtime.mstart()
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/runtime/proc.go:1135
goroutine 3397 [running]:
runtime.systemstack_switch()
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/runtime/asm_amd64.s:298 fp=0xc4381035c8 sp=0xc4381035c0 pc=0x9093c0
runtime.mallocgc(0x37400000, 0x126ff80, 0xc438103701, 0xc4381036b8)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/runtime/malloc.go:721 +0x7ae fp=0xc438103670 sp=0xc4381035c8 pc=0x8bf3ee
runtime.newarray(0x126ff80, 0x440000, 0xc43863a410)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/runtime/malloc.go:853 +0x60 fp=0xc4381036a0 sp=0xc438103670 pc=0x8bf750
runtime.makeBucketArray(0x1207880, 0x8d9416, 0x133ec68, 0xc438103748)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/runtime/hashmap.go:927 +0xf5 fp=0xc4381036e8 sp=0xc4381036a0 pc=0x8b8695
runtime.hashGrow(0x1207880, 0xc42057e270)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/runtime/hashmap.go:951 +0xa3 fp=0xc438103748 sp=0xc4381036e8 pc=0x8b8853
runtime.mapassign_faststr(0x1207880, 0xc42057e270, 0xc43900db40, 0xd, 0xc45b0552b8)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/runtime/hashmap_fast.go:666 +0x28d fp=0xc4381037c8 sp=0xc438103748 pc=0x8bb39d
github.com/dgraph-io/dgraph/dgraph/cmd/zero.(*Oracle).commit(0xc42057c000, 0xc4361d7440, 0x0, 0x0)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/dgraph/cmd/zero/oracle.go:118 +0xe1 fp=0xc438103818 sp=0xc4381037c8 pc=0x1118181
github.com/dgraph-io/dgraph/dgraph/cmd/zero.(*Server).commit(0xc420578000, 0x1a092c0, 0xc42057f4a0, 0xc4361d7440, 0x12879a0, 0xc420180901)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/dgraph/cmd/zero/oracle.go:293 +0x12d fp=0xc4381038b8 sp=0xc438103818 pc=0x111958d
github.com/dgraph-io/dgraph/dgraph/cmd/zero.(*Server).CommitOrAbort(0xc420578000, 0x1a092c0, 0xc42057f4a0, 0xc4361d7440, 0xc420578000, 0x100, 0xff)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/dgraph/cmd/zero/oracle.go:310 +0x8a fp=0xc438103900 sp=0xc4381038b8 pc=0x11197fa
github.com/dgraph-io/dgraph/protos/intern._Zero_CommitOrAbort_Handler(0x12ecaa0, 0xc420578000, 0x1a092c0, 0xc42057f4a0, 0xc420180930, 0x0, 0x0, 0x0, 0x7f54f47df408, 0xc42024c000)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/protos/intern/internal.pb.go:2273 +0x276 fp=0xc438103978 sp=0xc438103900 pc=0xe8b4c6
google.golang.org/grpc.(*Server).processUnaryRPC(0xc420072d80, 0x1a0c760, 0xc42056b200, 0xc4389112c0, 0xc420162210, 0x19f2cc0, 0x0, 0x0, 0x0)
/home/travis/gopath/src/google.golang.org/grpc/server.go:900 +0x9d3 fp=0xc438103d90 sp=0xc438103978 pc=0xe54383
google.golang.org/grpc.(*Server).handleStream(0xc420072d80, 0x1a0c760, 0xc42056b200, 0xc4389112c0, 0x0)
/home/travis/gopath/src/google.golang.org/grpc/server.go:1122 +0x1528 fp=0xc438103f80 sp=0xc438103d90 pc=0xe58498
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc420161ad0, 0xc420072d80, 0x1a0c760, 0xc42056b200, 0xc4389112c0)
/home/travis/gopath/src/google.golang.org/grpc/server.go:617 +0x9f fp=0xc438103fb8 sp=0xc438103f80 pc=0xe5ed7f
runtime.goexit()
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc438103fc0 sp=0xc438103fb8 pc=0x90bff1
created by google.golang.org/grpc.(*Server).serveStreams.func1
/home/travis/gopath/src/google.golang.org/grpc/server.go:615 +0xa1
goroutine 1 [semacquire, 68 minutes]:
sync.runtime_Semacquire(0xc42031fa4c)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/runtime/sema.go:56 +0x39
sync.(*WaitGroup).Wait(0xc42031fa40)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/sync/waitgroup.go:131 +0x72
github.com/dgraph-io/dgraph/dgraph/cmd/zero.run()
/home/travis/gopath/src/github.com/dgraph-io/dgraph/dgraph/cmd/zero/run.go:271 +0x91b
github.com/dgraph-io/dgraph/dgraph/cmd/zero.init.0.func1(0xc4201a3b00, 0xc4202c13e0, 0x0, 0x2)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/dgraph/cmd/zero/run.go:70 +0x52
github.com/dgraph-io/dgraph/vendor/github.com/spf13/cobra.(*Command).execute(0xc4201a3b00, 0xc4202c13a0, 0x2, 0x2, 0xc4201a3b00, 0xc4202c13a0)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/vendor/github.com/spf13/cobra/command.go:702 +0x2c6
github.com/dgraph-io/dgraph/vendor/github.com/spf13/cobra.(*Command).ExecuteC(0x19f4c40, 0xc42017dee8, 0x8ee210, 0x19fc310)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/vendor/github.com/spf13/cobra/command.go:783 +0x30e
github.com/dgraph-io/dgraph/vendor/github.com/spf13/cobra.(*Command).Execute(0x19f4c40, 0x19fc310, 0x14fcf5a02b827db5)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/vendor/github.com/spf13/cobra/command.go:736 +0x2b
github.com/dgraph-io/dgraph/dgraph/cmd.Execute()
/home/travis/gopath/src/github.com/dgraph-io/dgraph/dgraph/cmd/root.go:51 +0x31
main.main()
/home/travis/gopath/src/github.com/dgraph-io/dgraph/dgraph/main.go:34 +0xad
goroutine 18 [syscall, 68 minutes]:
os/signal.signal_recv(0x8e2e08)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/runtime/sigqueue.go:131 +0xa6
os/signal.loop()
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/os/signal/signal_unix.go:22 +0x22
created by os/signal.init.0
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/os/signal/signal_unix.go:28 +0x41
goroutine 19 [chan receive]:
github.com/dgraph-io/dgraph/x.init.1.func1()
/home/travis/gopath/src/github.com/dgraph-io/dgraph/x/metrics.go:96 +0x7e
created by github.com/dgraph-io/dgraph/x.init.1
/home/travis/gopath/src/github.com/dgraph-io/dgraph/x/metrics.go:91 +0x545
goroutine 25 [chan receive, 68 minutes]:
github.com/dgraph-io/dgraph/x.(*WaterMark).process(0xc4202573e0)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/x/watermark.go:184 +0x28e
created by github.com/dgraph-io/dgraph/x.(*WaterMark).Init
/home/travis/gopath/src/github.com/dgraph-io/dgraph/x/watermark.go:77 +0xc8
goroutine 104 [chan receive]:
github.com/dgraph-io/dgraph/x.(*WaterMark).process(0xc42024c150)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/x/watermark.go:184 +0x28e
created by github.com/dgraph-io/dgraph/x.(*WaterMark).Init
/home/travis/gopath/src/github.com/dgraph-io/dgraph/x/watermark.go:77 +0xc8
goroutine 178 [chan receive]:
github.com/dgraph-io/dgraph/x.(*WaterMark).process(0xc42057c068)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/x/watermark.go:184 +0x28e
created by github.com/dgraph-io/dgraph/x.(*WaterMark).Init
/home/travis/gopath/src/github.com/dgraph-io/dgraph/x/watermark.go:77 +0xc8
goroutine 179 [chan receive]:
github.com/dgraph-io/dgraph/dgraph/cmd/zero.(*Oracle).sendDeltasToSubscribers(0xc42057c000)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/dgraph/cmd/zero/oracle.go:165 +0x4a1
created by github.com/dgraph-io/dgraph/dgraph/cmd/zero.(*Oracle).Init
/home/travis/gopath/src/github.com/dgraph-io/dgraph/dgraph/cmd/zero/oracle.go:60 +0x1b3
goroutine 180 [select, 5 minutes]:
github.com/dgraph-io/dgraph/dgraph/cmd/zero.(*Server).rebalanceTablets(0xc420578000)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/dgraph/cmd/zero/tablet.go:59 +0x138
created by github.com/dgraph-io/dgraph/dgraph/cmd/zero.(*Server).Init
/home/travis/gopath/src/github.com/dgraph-io/dgraph/dgraph/cmd/zero/zero.go:85 +0x1ff
goroutine 181 [chan receive]:
github.com/dgraph-io/dgraph/dgraph/cmd/zero.(*Server).purgeOracle(0xc420578000)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/dgraph/cmd/zero/oracle.go:367 +0x39c
created by github.com/dgraph-io/dgraph/dgraph/cmd/zero.(*Server).Init
/home/travis/gopath/src/github.com/dgraph-io/dgraph/dgraph/cmd/zero/zero.go:86 +0x221
goroutine 182 [IO wait, 68 minutes]:
internal/poll.runtime_pollWait(0x7f5593ff6f70, 0x72, 0xffffffffffffffff)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc420168598, 0x72, 0xc420045c00, 0x0, 0x0)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/internal/poll/fd_poll_runtime.go:85 +0xae
internal/poll.(*pollDesc).waitRead(0xc420168598, 0xffffffffffffff00, 0x0, 0x0)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Accept(0xc420168580, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/internal/poll/fd_unix.go:335 +0x1e2
net.(*netFD).accept(0xc420168580, 0xc4201cc020, 0x0, 0x0)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc42014a2e8, 0xe51fd2, 0x908730, 0xc420045e48)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/net/tcpsock_posix.go:136 +0x2e
net.(*TCPListener).Accept(0xc42014a2e8, 0x133dfa8, 0xc420072d80, 0x1a0cdc0, 0xc4201cc020)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/net/tcpsock.go:247 +0x49
google.golang.org/grpc.(*Server).Serve(0xc420072d80, 0x1a08300, 0xc42014a2e8, 0x0, 0x0)
/home/travis/gopath/src/google.golang.org/grpc/server.go:495 +0x196
github.com/dgraph-io/dgraph/dgraph/cmd/zero.(*state).serveGRPC.func1(0xc42031fa40, 0xc420072d80, 0x1a08300, 0xc42014a2e8)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/dgraph/cmd/zero/run.go:121 +0x6d
created by github.com/dgraph-io/dgraph/dgraph/cmd/zero.(*state).serveGRPC
/home/travis/gopath/src/github.com/dgraph-io/dgraph/dgraph/cmd/zero/run.go:119 +0x31b
goroutine 183 [IO wait, 68 minutes]:
internal/poll.runtime_pollWait(0x7f5593ff6eb0, 0x72, 0xffffffffffffffff)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc420168618, 0x72, 0xc420101c00, 0x0, 0x0)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/internal/poll/fd_poll_runtime.go:85 +0xae
internal/poll.(*pollDesc).waitRead(0xc420168618, 0xffffffffffffff00, 0x0, 0x0)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Accept(0xc420168600, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/internal/poll/fd_unix.go:335 +0x1e2
net.(*netFD).accept(0xc420168600, 0x8bf678, 0x30, 0x1286580)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc42014a2f0, 0xa34efa, 0x1286580, 0xc42057e150)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/net/tcpsock_posix.go:136 +0x2e
net.(*TCPListener).Accept(0xc42014a2f0, 0xc4200140c8, 0x11fb8a0, 0x19eabd0, 0x12df240)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/net/tcpsock.go:247 +0x49
net/http.(*Server).Serve(0xc420158410, 0x1a08300, 0xc42014a2f0, 0x0, 0x0)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/net/http/server.go:2695 +0x1b2
github.com/dgraph-io/dgraph/dgraph/cmd/zero.(*state).serveHTTP.func1(0xc42031fa40, 0xc420158410, 0x1a08300, 0xc42014a2f0)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/dgraph/cmd/zero/run.go:136 +0x87
created by github.com/dgraph-io/dgraph/dgraph/cmd/zero.(*state).serveHTTP
/home/travis/gopath/src/github.com/dgraph-io/dgraph/dgraph/cmd/zero/run.go:134 +0x9b
goroutine 184 [select, 3 minutes]:
github.com/dgraph-io/dgraph/vendor/github.com/dgraph-io/badger.(*DB).updateSize(0xc4200fc000, 0xc42000c120)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/vendor/github.com/dgraph-io/badger/db.go:852 +0x1cc
created by github.com/dgraph-io/dgraph/vendor/github.com/dgraph-io/badger.Open
/home/travis/gopath/src/github.com/dgraph-io/dgraph/vendor/github.com/dgraph-io/badger/db.go:241 +0x988
goroutine 165 [select]:
github.com/dgraph-io/dgraph/vendor/github.com/dgraph-io/badger.(*levelsController).runWorker(0xc42012c000, 0xc4201c6120)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/vendor/github.com/dgraph-io/badger/levels.go:188 +0x1e2
created by github.com/dgraph-io/dgraph/vendor/github.com/dgraph-io/badger.(*levelsController).startCompact
/home/travis/gopath/src/github.com/dgraph-io/dgraph/vendor/github.com/dgraph-io/badger/levels.go:173 +0x82
goroutine 166 [select]:
github.com/dgraph-io/dgraph/vendor/github.com/dgraph-io/badger.(*levelsController).runWorker(0xc42012c000, 0xc4201c6120)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/vendor/github.com/dgraph-io/badger/levels.go:188 +0x1e2
created by github.com/dgraph-io/dgraph/vendor/github.com/dgraph-io/badger.(*levelsController).startCompact
/home/travis/gopath/src/github.com/dgraph-io/dgraph/vendor/github.com/dgraph-io/badger/levels.go:173 +0x82
goroutine 167 [select]:
github.com/dgraph-io/dgraph/vendor/github.com/dgraph-io/badger.(*levelsController).runWorker(0xc42012c000, 0xc4201c6120)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/vendor/github.com/dgraph-io/badger/levels.go:188 +0x1e2
created by github.com/dgraph-io/dgraph/vendor/github.com/dgraph-io/badger.(*levelsController).startCompact
/home/travis/gopath/src/github.com/dgraph-io/dgraph/vendor/github.com/dgraph-io/badger/levels.go:173 +0x82
goroutine 168 [chan receive, 68 minutes]:
github.com/dgraph-io/dgraph/vendor/github.com/dgraph-io/badger.(*DB).flushMemtable(0xc4200fc000, 0xc4201c6140, 0x0, 0x0)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/vendor/github.com/dgraph-io/badger/db.go:747 +0xfa
created by github.com/dgraph-io/dgraph/vendor/github.com/dgraph-io/badger.Open
/home/travis/gopath/src/github.com/dgraph-io/dgraph/vendor/github.com/dgraph-io/badger/db.go:253 +0xb3a
goroutine 105 [select]:
github.com/dgraph-io/dgraph/vendor/github.com/dgraph-io/badger.(*DB).doWrites(0xc4200fc000, 0xc4205680c0)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/vendor/github.com/dgraph-io/badger/db.go:606 +0x47a
created by github.com/dgraph-io/dgraph/vendor/github.com/dgraph-io/badger.Open
/home/travis/gopath/src/github.com/dgraph-io/dgraph/vendor/github.com/dgraph-io/badger/db.go:297 +0xfe8
goroutine 106 [chan receive, 68 minutes]:
github.com/dgraph-io/dgraph/vendor/github.com/dgraph-io/badger.(*valueLog).waitOnGC(0xc4200fc128, 0xc4205680e0)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/vendor/github.com/dgraph-io/badger/value.go:981 +0x61
created by github.com/dgraph-io/dgraph/vendor/github.com/dgraph-io/badger.Open
/home/travis/gopath/src/github.com/dgraph-io/dgraph/vendor/github.com/dgraph-io/badger/db.go:300 +0x104e
goroutine 211 [select]:
github.com/dgraph-io/dgraph/vendor/github.com/coreos/etcd/raft.(*node).run(0xc4205802a0, 0xc42015ef00)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/vendor/github.com/coreos/etcd/raft/node.go:313 +0x5f3
created by github.com/dgraph-io/dgraph/vendor/github.com/coreos/etcd/raft.StartNode
/home/travis/gopath/src/github.com/dgraph-io/dgraph/vendor/github.com/coreos/etcd/raft/node.go:210 +0x610
goroutine 212 [runnable]:
github.com/dgraph-io/dgraph/dgraph/cmd/zero.(*node).Run(0xc420574000)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/dgraph/cmd/zero/raft.go:471 +0xa93
created by github.com/dgraph-io/dgraph/dgraph/cmd/zero.(*node).initAndStartNode
/home/travis/gopath/src/github.com/dgraph-io/dgraph/dgraph/cmd/zero/raft.go:422 +0x58c
goroutine 213 [chan receive, 68 minutes]:
github.com/dgraph-io/dgraph/conn.(*Node).BatchAndSendMessages(0xc42024c0c0)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/conn/node.go:276 +0x4d0
created by github.com/dgraph-io/dgraph/dgraph/cmd/zero.(*node).initAndStartNode
/home/travis/gopath/src/github.com/dgraph-io/dgraph/dgraph/cmd/zero/raft.go:423 +0x5b4
goroutine 214 [select, 68 minutes, locked to thread]:
runtime.gopark(0x133eb20, 0x0, 0x12feb13, 0x6, 0x18, 0x1)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/runtime/proc.go:287 +0x12c
runtime.selectgo(0xc42055df50, 0xc42019e4e0)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/runtime/select.go:395 +0x1149
runtime.ensureSigM.func1()
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/runtime/signal_unix.go:511 +0x220
runtime.goexit()
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/runtime/asm_amd64.s:2337 +0x1
goroutine 215 [chan receive, 68 minutes]:
github.com/dgraph-io/dgraph/dgraph/cmd/zero.run.func1(0xc42031fa40, 0xc420580300, 0x1a08300, 0xc42014a2f0, 0x1a08300, 0xc42014a2e8, 0xc4202c1480)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/dgraph/cmd/zero/run.go:262 +0x62
created by github.com/dgraph-io/dgraph/dgraph/cmd/zero.run
/home/travis/gopath/src/github.com/dgraph-io/dgraph/dgraph/cmd/zero/run.go:260 +0x8b1
goroutine 109 [IO wait]:
internal/poll.runtime_pollWait(0x7f5593ff6df0, 0x72, 0x0)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc420168118, 0x72, 0xffffffffffffff00, 0x1a01fc0, 0x19fc678)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/internal/poll/fd_poll_runtime.go:85 +0xae
internal/poll.(*pollDesc).waitRead(0xc420168118, 0xc425af0000, 0x8000, 0x8000)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc420168100, 0xc425af0000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/internal/poll/fd_unix.go:126 +0x18a
net.(*netFD).Read(0xc420168100, 0xc425af0000, 0x8000, 0x8000, 0x8b25b0, 0xc4374de298, 0xc42057f3e0)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/net/fd_unix.go:202 +0x52
net.(*conn).Read(0xc4201cc020, 0xc425af0000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/net/net.go:176 +0x6d
bufio.(*Reader).Read(0xc42015cea0, 0xc4203c81f8, 0x9, 0x9, 0x9, 0xc420176c80, 0x7ffd333fdbd6)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/bufio/bufio.go:213 +0x30b
io.ReadAtLeast(0x19fe000, 0xc42015cea0, 0xc4203c81f8, 0x9, 0x9, 0x9, 0x3522e11a20176cf0, 0x5a24bf78, 0xc420176cf8)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/io/io.go:309 +0x86
io.ReadFull(0x19fe000, 0xc42015cea0, 0xc4203c81f8, 0x9, 0x9, 0x3bcdf7476f9, 0x1a81e60, 0xbe814dbe34193a5c)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/io/io.go:327 +0x58
golang.org/x/net/http2.readFrameHeader(0xc4203c81f8, 0x9, 0x9, 0x19fe000, 0xc42015cea0, 0x0, 0x7070e0900000000, 0xc42016cec8, 0xc420176e20)
/home/travis/gopath/src/golang.org/x/net/http2/frame.go:237 +0x7b
golang.org/x/net/http2.(*Framer).ReadFrame(0xc4203c81c0, 0xc4374bf320, 0xc4374bf320, 0x0, 0x0)
/home/travis/gopath/src/golang.org/x/net/http2/frame.go:492 +0xa4
google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc42056b200, 0xc420162ab0, 0x133dfe0)
/home/travis/gopath/src/google.golang.org/grpc/transport/http2_server.go:390 +0x3a
google.golang.org/grpc.(*Server).serveStreams(0xc420072d80, 0x1a0c760, 0xc42056b200)
/home/travis/gopath/src/google.golang.org/grpc/server.go:613 +0x140
google.golang.org/grpc.(*Server).handleRawConn(0xc420072d80, 0x1a0cdc0, 0xc4201cc020)
/home/travis/gopath/src/google.golang.org/grpc/server.go:574 +0x5e0
created by google.golang.org/grpc.(*Server).Serve
/home/travis/gopath/src/google.golang.org/grpc/server.go:535 +0x632
goroutine 110 [select]:
google.golang.org/grpc/transport.loopyWriter(0x1a09200, 0xc420010580, 0xc420162a20, 0xc420046fb8)
/home/travis/gopath/src/google.golang.org/grpc/transport/transport.go:719 +0x3c6
google.golang.org/grpc/transport.newHTTP2Server.func1(0xc42056b200)
/home/travis/gopath/src/google.golang.org/grpc/transport/http2_server.go:256 +0x5e
created by google.golang.org/grpc/transport.newHTTP2Server
/home/travis/gopath/src/google.golang.org/grpc/transport/http2_server.go:255 +0xe3f
goroutine 111 [select, 68 minutes]:
google.golang.org/grpc/transport.(*http2Server).keepalive(0xc42056b200)
/home/travis/gopath/src/google.golang.org/grpc/transport/http2_server.go:951 +0x264
created by google.golang.org/grpc/transport.newHTTP2Server
/home/travis/gopath/src/google.golang.org/grpc/transport/http2_server.go:259 +0xe64
goroutine 175 [select]:
google.golang.org/grpc/transport.loopyWriter(0x1a09200, 0xc4201ca640, 0xc42057e6f0, 0xc420047fb8)
/home/travis/gopath/src/google.golang.org/grpc/transport/transport.go:719 +0x3c6
google.golang.org/grpc/transport.newHTTP2Client.func3(0xc420520c00)
/home/travis/gopath/src/google.golang.org/grpc/transport/http2_client.go:302 +0x5e
created by google.golang.org/grpc/transport.newHTTP2Client
/home/travis/gopath/src/google.golang.org/grpc/transport/http2_client.go:301 +0xde7
goroutine 217 [select, 68 minutes]:
google.golang.org/grpc.(*ccResolverWrapper).watcher(0xc420256e10)
/home/travis/gopath/src/google.golang.org/grpc/resolver_conn_wrapper.go:108 +0x1ac
created by google.golang.org/grpc.(*ccResolverWrapper).start
/home/travis/gopath/src/google.golang.org/grpc/resolver_conn_wrapper.go:94 +0x3f
goroutine 218 [select, 68 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc4201aa780)
/home/travis/gopath/src/google.golang.org/grpc/balancer_conn_wrappers.go:122 +0x156
created by google.golang.org/grpc.newCCBalancerWrapper
/home/travis/gopath/src/google.golang.org/grpc/balancer_conn_wrappers.go:113 +0x197
goroutine 219 [select, 68 minutes]:
google.golang.org/grpc.(*addrConn).transportMonitor(0xc420322000)
/home/travis/gopath/src/google.golang.org/grpc/clientconn.go:1077 +0x166
google.golang.org/grpc.(*addrConn).connect.func1(0xc420322000)
/home/travis/gopath/src/google.golang.org/grpc/clientconn.go:777 +0x1ce
created by google.golang.org/grpc.(*addrConn).connect
/home/travis/gopath/src/google.golang.org/grpc/clientconn.go:768 +0xeb
goroutine 174 [IO wait]:
internal/poll.runtime_pollWait(0x7f5593ff6d30, 0x72, 0x0)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc420578398, 0x72, 0xffffffffffffff00, 0x1a01fc0, 0x19fc678)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/internal/poll/fd_poll_runtime.go:85 +0xae
internal/poll.(*pollDesc).waitRead(0xc420578398, 0xc425b08000, 0x8000, 0x8000)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc420578380, 0xc425b08000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/internal/poll/fd_unix.go:126 +0x18a
net.(*netFD).Read(0xc420578380, 0xc425b08000, 0x8000, 0x8000, 0x11, 0x0, 0x0)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/net/fd_unix.go:202 +0x52
net.(*conn).Read(0xc42014a158, 0xc425b08000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/net/net.go:176 +0x6d
bufio.(*Reader).Read(0xc420066cc0, 0xc420326038, 0x9, 0x9, 0x9, 0x0, 0x0)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/bufio/bufio.go:213 +0x30b
io.ReadAtLeast(0x19fe000, 0xc420066cc0, 0xc420326038, 0x9, 0x9, 0x9, 0xc420559de0, 0x7ffd333fdbd6, 0xc420559df0)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/io/io.go:309 +0x86
io.ReadFull(0x19fe000, 0xc420066cc0, 0xc420326038, 0x9, 0x9, 0x8fb598, 0x22af71afb7afe, 0xc42f5e76e3)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/io/io.go:327 +0x58
golang.org/x/net/http2.readFrameHeader(0xc420326038, 0x9, 0x9, 0x19fe000, 0xc420066cc0, 0x0, 0x3bb00000000, 0x1a81e60, 0xbe814dbcaf5d776e)
/home/travis/gopath/src/golang.org/x/net/http2/frame.go:237 +0x7b
golang.org/x/net/http2.(*Framer).ReadFrame(0xc420326000, 0xc4374bf2c0, 0xc4374bf2c0, 0x0, 0x0)
/home/travis/gopath/src/golang.org/x/net/http2/frame.go:492 +0xa4
google.golang.org/grpc/transport.(*http2Client).reader(0xc420520c00)
/home/travis/gopath/src/google.golang.org/grpc/transport/http2_client.go:1167 +0xc3
created by google.golang.org/grpc/transport.newHTTP2Client
/home/travis/gopath/src/google.golang.org/grpc/transport/http2_client.go:270 +0xc96
goroutine 113 [chan receive]:
github.com/dgraph-io/dgraph/conn.(*Pool).MonitorHealth(0xc4201aa690)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/conn/pool.go:177 +0xb5
created by github.com/dgraph-io/dgraph/conn.NewPool
/home/travis/gopath/src/github.com/dgraph-io/dgraph/conn/pool.go:136 +0x26e
goroutine 223 [select]:
github.com/dgraph-io/dgraph/dgraph/cmd/zero.(*Server).Oracle(0xc420578000, 0xc4201c6600, 0x1a0c2e0, 0xc4203c2260, 0x0, 0x0)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/dgraph/cmd/zero/oracle.go:324 +0x1e9
github.com/dgraph-io/dgraph/protos/intern._Zero_Oracle_Handler(0x12ecaa0, 0xc420578000, 0x1a0b5c0, 0xc42016eb40, 0xb8, 0xc4202c9e80)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/protos/intern/internal.pb.go:2197 +0x10e
google.golang.org/grpc.(*Server).processStreamingRPC(0xc420072d80, 0x1a0c760, 0xc42056b200, 0xc4202fef00, 0xc420162210, 0x19f11a0, 0x0, 0x0, 0x0)
/home/travis/gopath/src/google.golang.org/grpc/server.go:1032 +0xa3e
google.golang.org/grpc.(*Server).handleStream(0xc420072d80, 0x1a0c760, 0xc42056b200, 0xc4202fef00, 0x0)
/home/travis/gopath/src/google.golang.org/grpc/server.go:1126 +0x14c1
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc420161ad0, 0xc420072d80, 0x1a0c760, 0xc42056b200, 0xc4202fef00)
/home/travis/gopath/src/google.golang.org/grpc/server.go:617 +0x9f
created by google.golang.org/grpc.(*Server).serveStreams.func1
/home/travis/gopath/src/google.golang.org/grpc/server.go:615 +0xa1
goroutine 224 [select]:
github.com/dgraph-io/dgraph/dgraph/cmd/zero.(*Server).Update(0xc420578000, 0x1a0cc40, 0xc4202ac7a0, 0x0, 0x0)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/dgraph/cmd/zero/zero.go:503 +0x338
github.com/dgraph-io/dgraph/protos/intern._Zero_Update_Handler(0x12ecaa0, 0xc420578000, 0x1a0b5c0, 0xc4202e6500, 0xc42019e538, 0x0)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/protos/intern/internal.pb.go:2167 +0xb2
google.golang.org/grpc.(*Server).processStreamingRPC(0xc420072d80, 0x1a0c760, 0xc42056b200, 0xc4202ff180, 0xc420162210, 0x19f1180, 0x0, 0x0, 0x0)
/home/travis/gopath/src/google.golang.org/grpc/server.go:1032 +0xa3e
google.golang.org/grpc.(*Server).handleStream(0xc420072d80, 0x1a0c760, 0xc42056b200, 0xc4202ff180, 0x0)
/home/travis/gopath/src/google.golang.org/grpc/server.go:1126 +0x14c1
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc420161ad0, 0xc420072d80, 0x1a0c760, 0xc42056b200, 0xc4202ff180)
/home/travis/gopath/src/google.golang.org/grpc/server.go:617 +0x9f
created by google.golang.org/grpc.(*Server).serveStreams.func1
/home/travis/gopath/src/google.golang.org/grpc/server.go:615 +0xa1
goroutine 225 [select, 3 minutes]:
google.golang.org/grpc/transport.(*recvBufferReader).read(0xc4201aae10, 0xc420148e30, 0x5, 0x5, 0x8bf678, 0xc42017db08, 0x8d9198)
/home/travis/gopath/src/google.golang.org/grpc/transport/transport.go:133 +0x28b
google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4201aae10, 0xc420148e30, 0x5, 0x5, 0x3, 0xc420027300, 0xc420027370)
/home/travis/gopath/src/google.golang.org/grpc/transport/transport.go:122 +0x67
google.golang.org/grpc/transport.(*transportReader).Read(0xc4203bb020, 0xc420148e30, 0x5, 0x5, 0x91846e, 0xc4202ff278, 0xc4200109c0)
/home/travis/gopath/src/google.golang.org/grpc/transport/transport.go:387 +0x55
io.ReadAtLeast(0x19ff300, 0xc4203bb020, 0xc420148e30, 0x5, 0x5, 0x5, 0xc42056b200, 0xc4202ff180, 0x7f5500000005)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/io/io.go:309 +0x86
io.ReadFull(0x19ff300, 0xc4203bb020, 0xc420148e30, 0x5, 0x5, 0x0, 0x122a140, 0x1291be0)
/home/travis/.gimme/versions/go1.9.2.linux.amd64/src/io/io.go:327 +0x58
google.golang.org/grpc/transport.(*Stream).Read(0xc4202ff180, 0xc420148e30, 0x5, 0x5, 0x12bda00, 0xc42017dcd0, 0x8d9198)
/home/travis/gopath/src/google.golang.org/grpc/transport/transport.go:371 +0xbf
google.golang.org/grpc.(*parser).recvMsg(0xc420148e20, 0x10000000, 0xc420543380, 0x133da30, 0xc42017dd20, 0xd549c5, 0xc420578000, 0xc42017deb8)
/home/travis/gopath/src/google.golang.org/grpc/rpc_util.go:285 +0x65
google.golang.org/grpc.recv(0xc420148e20, 0x1a08ac0, 0x1aa1fd8, 0xc4202ff180, 0x0, 0x0, 0x1291be0, 0xc42015b370, 0x10000000, 0x0, …)
/home/travis/gopath/src/google.golang.org/grpc/rpc_util.go:394 +0x4d
google.golang.org/grpc.(*serverStream).RecvMsg(0xc4202e6500, 0x1291be0, 0xc42015b370, 0x0, 0x0)
/home/travis/gopath/src/google.golang.org/grpc/stream.go:698 +0x12e
github.com/dgraph-io/dgraph/protos/intern.(*zeroUpdateServer).Recv(0xc4202ac7a0, 0xc42017df48, 0xc42019f620, 0xc420578000)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/protos/intern/internal.pb.go:2186 +0x62
github.com/dgraph-io/dgraph/dgraph/cmd/zero.(*Server).receiveUpdates(0xc420578000, 0x1a0cc40, 0xc4202ac7a0, 0xc42056b200, 0xc4202ff180)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/dgraph/cmd/zero/zero.go:441 +0x16f
github.com/dgraph-io/dgraph/dgraph/cmd/zero.(*Server).Update.func1(0xc4205810e0, 0xc420578000, 0x1a0cc40, 0xc4202ac7a0)
/home/travis/gopath/src/github.com/dgraph-io/dgraph/dgraph/cmd/zero/zero.go:478 +0x3f
created by github.com/dgraph-io/dgraph/dgraph/cmd/zero.(*Server).Update
/home/travis/gopath/src/github.com/dgraph-io/dgraph/dgraph/cmd/zero/zero.go:477 +0xac
deepakjois commented :
This has been open for too long. Dgraph v1.0 has been released in the meanwhile and we have made several improvements. Hopefully that will resolve any issues.
