Hi!
I’ve followed the setup guide for Prometheus here Get started with Dgraph for my cluster using Docker.
When inside my prometheus instance I can run wget on the /debug/vars routes on each node and get the proper data but prometheus is complaining with the following errors when trying to get the proper information itself.
I can confirm those ports and hosts are correct, and like I said, running wget from inside the prometheus docker container returns the proper data.
The JSON below is for wget http://cluster0-node0:8080/debug/vars inside the prometheus container.
{
"badger_blocked_puts_total": 0,
"badger_disk_reads_total": 1064575,
"badger_disk_writes_total": 116,
"badger_gets_total": 1117864,
"badger_lsm_bloom_hits_total": {
"l1": 1098407,
"l2": 23114
},
"badger_lsm_level_gets_total": {
"l1": 18019,
"l2": 1082983
},
"badger_lsm_size_bytes": {
"p": 1496051084,
"w": 0
},
"badger_memtable_gets_total": 1117864,
"badger_pending_writes_total": {
"p": 0,
"w": 0
},
"badger_puts_total": 1096,
"badger_read_bytes": 137997238,
"badger_vlog_size_bytes": {
"p": 5642580220,
"w": 13675098
},
"badger_written_bytes": 2074341,
"cmdline": ["dgraph", "server", "--my=cluster0-node0:7080", "--lru_mb=2048", "--zero=zero:5080", "--logtostderr"],
"dgraph-bulk-loader_badger_writes_pending": 0,
"dgraph-bulk-loader_num_reducers_total": 0,
"dgraph-bulk-loader_reduce_queue_size": 0,
"dgraph_active_mutations_total": 0,
"dgraph_cache_hits_total": 1902817,
"dgraph_cache_miss_total": 1067111,
"dgraph_cache_race_total": 28654,
"dgraph_config": {
"allotted_memory": 2048,
"badger.options": "ssd",
"badger.tables": "mmap",
"badger.vlog": "none",
"expand_edge": 1,
"num_pending_proposals": 2000,
"posting_dir": "p",
"tracing": 0,
"wal_dir": "w"
},
"dgraph_dirtymap_keys_total": 0,
"dgraph_evicted_lists_total": 105,
"dgraph_goroutines_total": 74,
"dgraph_heap_idle_bytes": 755744768,
"dgraph_lcache_capacity_bytes": 523294079,
"dgraph_lcache_keys_total": 1038352,
"dgraph_lcache_size_bytes": 94015224,
"dgraph_max_list_bytes": 618584,
"dgraph_max_list_length": 228404,
"dgraph_memory_inuse_bytes": 1194377216,
"dgraph_num_queries_total": 16,
"dgraph_pending_proposals_total": 0,
"dgraph_pending_queries_total": 0,
"dgraph_posting_reads_total": 0,
"dgraph_posting_writes_total": 105,
"dgraph_predicate_stats": {
"_predicate_": 116,
"i.title": 196,
"owner": 58,
"title": 58
},
"dgraph_proc_memory_bytes": 4395511808,
"dgraph_read_bytes_total": 98922234,
"dgraph_server_health_status": 1,
"dgraph_written_bytes_total": 2005382,
"memstats": {
"Alloc": 1239695928,
"TotalAlloc": 177574711032,
"Sys": 2530163320,
"Lookups": 914,
"Mallocs": 4767348384,
"Frees": 4748950292,
"HeapAlloc": 1239695928,
"HeapSys": 2359754752,
"HeapIdle": 1051148288,
"HeapInuse": 1308606464,
"HeapReleased": 418414592,
"HeapObjects": 18398092,
"StackInuse": 8781824,
"StackSys": 8781824,
"MSpanInuse": 21917792,
"MSpanSys": 35651584,
"MCacheInuse": 222208,
"MCacheSys": 245760,
"BuckHashSys": 1654890,
"GCSys": 107626496,
"OtherSys": 16448014,
"NextGC": 1936136304,
"LastGC": 1536938292219744852,
"PauseTotalNs": 279310601,
"PauseNs": [348792, 439089, 454105, 411806, 437041, 413281, 466367, 416011, 369595, 424851, 327838, 424396, 427492, 372663, 362784, 453969, 426148, 425706, 383552, 411830, 447820, 478444, 463119, 411859, 474188, 417187, 482967, 486462, 358002, 468915, 469251, 431000, 397727, 397239, 357391, 502029, 398894, 397345, 440376, 413801, 381664, 470892, 437657, 442130, 458768, 402247, 408048, 382568, 426628, 570662, 351513, 472765, 417862, 375821, 420010, 405434, 524565, 441821, 422115, 558517, 403648, 455866, 418720, 457738, 393626, 430934, 411285, 281169, 426401, 345315, 388810, 385143, 421137, 448291, 455226, 422950, 474362, 401266, 433595, 445580, 395095, 522386, 448557, 456802, 466369, 370176, 437314, 353102, 437441, 446282, 425983, 417644, 399756, 425912, 472185, 409103, 472069, 413924, 420880, 434620, 478421, 453232, 408410, 531287, 343117, 450826, 399392, 541524, 392977, 511088, 399720, 398849, 446230, 453919, 390437, 405137, 404270, 454861, 450108, 404411, 431940, 508367, 326480, 378604, 441843, 466707, 446444, 448211, 408704, 392296, 443931, 444620, 419193, 451022, 477118, 443456, 471738, 397620, 335680, 453691, 412102, 482311, 411681, 540638, 378926, 402343, 369740, 361154, 475584, 392415, 471754, 387812, 515702, 400362, 324964, 400525, 398584, 504470, 409834, 534652, 693842, 379255, 440677, 393587, 418263, 407752, 398688, 347379, 411855, 371562, 376496, 433751, 433956, 425977, 387935, 479380, 351586, 387642, 476956, 601770, 446883, 421603, 426932, 426560, 414440, 537517, 450648, 1689752, 2240940, 1369029, 9186739, 522666, 467312, 484815, 452626, 442588, 418420, 468883, 382241, 440929, 447884, 526529, 476923, 412929, 492351, 400153, 567434, 589491, 606443, 517621, 513291, 516975, 480219, 22034351, 506659, 498193, 434035, 465115, 480864, 425887, 387225, 441869, 509542, 570729, 421080, 430631, 491334, 473295, 547482, 428401, 604291, 506883, 542619, 510823, 432291, 417709, 563845, 445325, 405714, 499297, 490124, 539741, 485223, 446460, 525058, 521902, 424947, 568984, 415516, 552713, 462143, 545125, 474775, 532387, 432901, 457800],
"PauseEnd": [1536938289641632005, 1536938292219744852, 1536934390752013902, 1536934391515785884, 1536934396621138991, 1536934516637831164, 1536934636659416429, 1536934673986570492, 1536934674777698577, 1536934675559110213, 1536934676356044449, 1536934677173922374, 1536934678038825227, 1536934678846304969, 1536934679658144336, 1536934680495377260, 1536934681318500494, 1536934682128390819, 1536934682954384985, 1536934683766445996, 1536934684573107101, 1536934685381299411, 1536934686191264871, 1536934686971235821, 1536934687748740778, 1536934688524032157, 1536934689310174588, 1536934690094384887, 1536934690875618605, 1536934691596662456, 1536934737919383465, 1536934857932788992, 1536934973566502838, 1536934974369426817, 1536934975175698403, 1536934975958601593, 1536934976746018823, 1536934977597073730, 1536934978397109977, 1536934979207217457, 1536934980015622130, 1536934980838581867, 1536934981653728766, 1536934982470503423, 1536934983294451293, 1536934984088720962, 1536934984896120313, 1536934985716431585, 1536934986488588234, 1536934987270812718, 1536934988058215566, 1536934988837822483, 1536934989626006283, 1536934990410846781, 1536934991188749797, 1536934991867149660, 1536935103918524047, 1536935223936136631, 1536935273747060791, 1536935274609322019, 1536935275398329557, 1536935276187254357, 1536935276996876560, 1536935277874222500, 1536935278692029136, 1536935279514083594, 1536935280336632513, 1536935281164364173, 1536935281974031314, 1536935282776860916, 1536935283595647124, 1536935284416038783, 1536935285235685095, 1536935286061883293, 1536935286817595224, 1536935287599133254, 1536935288374117311, 1536935289144866161, 1536935289929216630, 1536935290714810313, 1536935291463573382, 1536935292138621733, 1536935412157616533, 1536935532178395297, 1536935573976773048, 1536935574794699682, 1536935575588017087, 1536935576396130742, 1536935577216908550, 1536935578077910710, 1536935578895517376, 1536935579698160993, 1536935580516385399, 1536935581337505993, 1536935582151593324, 1536935582993351306, 1536935583800963314, 1536935584610554344, 1536935585433975087, 1536935586234057941, 1536935587012361167, 1536935587794383164, 1536935588586427816, 1536935589356623390, 1536935590134184030, 1536935590906749342, 1536935591630417597, 1536935631620824754, 1536935751641097974, 1536935871659728062, 1536935874184425396, 1536935874985592514, 1536935875774563963, 1536935876586992909, 1536935877433447737, 1536935878250379099, 1536935879076355278, 1536935879894712956, 1536935880725394080, 1536935881551257884, 1536935882374107029, 1536935883202202846, 1536935884007059810, 1536935884800667107, 1536935885618118731, 1536935886412705154, 1536935887213404956, 1536935888006569098, 1536935888806857746, 1536935889616596997, 1536935890413641667, 1536935891196433844, 1536935891912871345, 1536935973620521084, 1536936093638891056, 1536936173763550961, 1536936174582699892, 1536936175376071925, 1536936176156424019, 1536936176958534974, 1536936177826319518, 1536936178618755992, 1536936179416273374, 1536936180230188364, 1536936181064316099, 1536936181896349624, 1536936182718481338, 1536936183556065920, 1536936184378009824, 1536936185184866683, 1536936186012770119, 1536936186802006673, 1536936187578349068, 1536936188370195483, 1536936189155352363, 1536936189935903597, 1536936190714169226, 1536936191481892813, 1536936192155841269, 1536936312178307123, 1536936432197832716, 1536936473641956191, 1536936474466875155, 1536936475263527183, 1536936476063906695, 1536936476860007683, 1536936477730248486, 1536936478545713396, 1536936479367258949, 1536936480171538655, 1536936480997833010, 1536936481813774015, 1536936482630843618, 1536936483472569724, 1536936484278156703, 1536936485094454723, 1536936485916484137, 1536936486702906453, 1536936487494821435, 1536936488277354953, 1536936489077640561, 1536936489877198571, 1536936490669124976, 1536936491455361090, 1536936492135939060, 1536936612160621598, 1536936681936530875, 1536936684839654958, 1536936685007862408, 1536936685235656618, 1536936685489925463, 1536936707671935808, 1536936708038654071, 1536936708574178049, 1536936709213151593, 1536936709954909986, 1536936710777953046, 1536936711569441221, 1536936712686294379, 1536936713939858386, 1536936715478633727, 1536936717410325845, 1536936719236219428, 1536936720047862902, 1536936721062477302, 1536936757033314766, 1536936776030526322, 1536936778868863032, 1536936781726570873, 1536936784539523056, 1536936787277389207, 1536936789937481339, 1536936792558679447, 1536936851501858815, 1536936971592510149, 1536937075730247618, 1536937078532212707, 1536937081313568053, 1536937084118881445, 1536937086867124095, 1536937089498895838, 1536937092104868015, 1536937212206337948, 1536937332303779545, 1536937375892646356, 1536937378722454392, 1536937381517898196, 1536937384333676242, 1536937387100252008, 1536937389788848492, 1536937392443226396, 1536937512552013543, 1536937632629296174, 1536937675623320441, 1536937678433302665, 1536937681211934245, 1536937684046104151, 1536937686785145695, 1536937689467800975, 1536937692106589839, 1536937812190034773, 1536937932268152982, 1536937975941906914, 1536937978783958612, 1536937981540086296, 1536937984334636190, 1536937987058915473, 1536937989659993596, 1536937992298706722, 1536938112394033892, 1536938232473055486, 1536938275895011434, 1536938278725569416, 1536938281524619383, 1536938284290503227, 1536938287028834872],
"NumGC": 514,
"NumForcedGC": 0,
"GCCPUFraction": 0.0001535942208432091,
"EnableGC": true,
"DebugGC": false,
"BySize": [{
"Size": 0,
"Mallocs": 0,
"Frees": 0
}, {
"Size": 8,
"Mallocs": 10216049,
"Frees": 9178792
}, {
"Size": 16,
"Mallocs": 575464954,
"Frees": 573759972
}, {
"Size": 32,
"Mallocs": 2746978691,
"Frees": 2739147046
}, {
"Size": 48,
"Mallocs": 27556797,
"Frees": 24379189
}, {
"Size": 64,
"Mallocs": 898874398,
"Frees": 897515653
}, {
"Size": 80,
"Mallocs": 21214546,
"Frees": 20110126
}, {
"Size": 96,
"Mallocs": 9034027,
"Frees": 8979605
}, {
"Size": 112,
"Mallocs": 1339269,
"Frees": 1335007
}, {
"Size": 128,
"Mallocs": 7528804,
"Frees": 5457042
}, {
"Size": 144,
"Mallocs": 14435697,
"Frees": 14415335
}, {
"Size": 160,
"Mallocs": 1998234,
"Frees": 1992064
}, {
"Size": 176,
"Mallocs": 2427105,
"Frees": 2423003
}, {
"Size": 192,
"Mallocs": 1677,
"Frees": 1652
}, {
"Size": 208,
"Mallocs": 712956,
"Frees": 710864
}, {
"Size": 224,
"Mallocs": 772,
"Frees": 724
}, {
"Size": 240,
"Mallocs": 79,
"Frees": 54
}, {
"Size": 256,
"Mallocs": 1349679,
"Frees": 1345466
}, {
"Size": 288,
"Mallocs": 1018441,
"Frees": 1015410
}, {
"Size": 320,
"Mallocs": 667055,
"Frees": 664960
}, {
"Size": 352,
"Mallocs": 1333758,
"Frees": 1329672
}, {
"Size": 384,
"Mallocs": 4669,
"Frees": 2182
}, {
"Size": 416,
"Mallocs": 714,
"Frees": 658
}, {
"Size": 448,
"Mallocs": 795,
"Frees": 771
}, {
"Size": 480,
"Mallocs": 1339,
"Frees": 1333
}, {
"Size": 512,
"Mallocs": 16816,
"Frees": 16807
}, {
"Size": 576,
"Mallocs": 996286,
"Frees": 993260
}, {
"Size": 640,
"Mallocs": 112,
"Frees": 97
}, {
"Size": 704,
"Mallocs": 1293,
"Frees": 1275
}, {
"Size": 768,
"Mallocs": 301,
"Frees": 294
}, {
"Size": 896,
"Mallocs": 251,
"Frees": 230
}, {
"Size": 1024,
"Mallocs": 15967,
"Frees": 15808
}, {
"Size": 1152,
"Mallocs": 721,
"Frees": 699
}, {
"Size": 1280,
"Mallocs": 79,
"Frees": 72
}, {
"Size": 1408,
"Mallocs": 52,
"Frees": 31
}, {
"Size": 1536,
"Mallocs": 170,
"Frees": 154
}, {
"Size": 1792,
"Mallocs": 370,
"Frees": 334
}, {
"Size": 2048,
"Mallocs": 3785855,
"Frees": 3785787
}, {
"Size": 2304,
"Mallocs": 665,
"Frees": 639
}, {
"Size": 2688,
"Mallocs": 75,
"Frees": 64
}, {
"Size": 3072,
"Mallocs": 122,
"Frees": 95
}, {
"Size": 3200,
"Mallocs": 19,
"Frees": 7
}, {
"Size": 3456,
"Mallocs": 19,
"Frees": 9
}, {
"Size": 4096,
"Mallocs": 13155,
"Frees": 13107
}, {
"Size": 4864,
"Mallocs": 884,
"Frees": 835
}, {
"Size": 5376,
"Mallocs": 124,
"Frees": 96
}, {
"Size": 6144,
"Mallocs": 501267,
"Frees": 501221
}, {
"Size": 6528,
"Mallocs": 32,
"Frees": 7
}, {
"Size": 6784,
"Mallocs": 19,
"Frees": 14
}, {
"Size": 6912,
"Mallocs": 3,
"Frees": 1
}, {
"Size": 8192,
"Mallocs": 10723,
"Frees": 10679
}, {
"Size": 9472,
"Mallocs": 797,
"Frees": 626
}, {
"Size": 9728,
"Mallocs": 9,
"Frees": 5
}, {
"Size": 10240,
"Mallocs": 7296,
"Frees": 7273
}, {
"Size": 10880,
"Mallocs": 42,
"Frees": 34
}, {
"Size": 12288,
"Mallocs": 48,
"Frees": 22
}, {
"Size": 13568,
"Mallocs": 5740,
"Frees": 5734
}, {
"Size": 14336,
"Mallocs": 9,
"Frees": 7
}, {
"Size": 16384,
"Mallocs": 822,
"Frees": 798
}, {
"Size": 18432,
"Mallocs": 4757,
"Frees": 4735
}, {
"Size": 19072,
"Mallocs": 22,
"Frees": 22
}]
}
}
My prometheus logs are being spammed with:
monitor_1 | level=warn ts=2018-09-14T15:34:59.919314679Z caller=scrape.go:804 component="scrape manager" scrape_pool=dgraph target=http://cluster0-node1:8081/debug/vars msg="append failed" err="\"INVALID\" is not a valid start token"
monitor_1 | level=warn ts=2018-09-14T15:35:00.172148576Z caller=scrape.go:804 component="scrape manager" scrape_pool=dgraph target=http://cluster0-node0:8080/debug/vars msg="append failed" err="\"INVALID\" is not a valid start token"
monitor_1 | level=warn ts=2018-09-14T15:35:00.211198498Z caller=scrape.go:804 component="scrape manager" scrape_pool=dgraph target=http://zero:6080/debug/vars msg="append failed" err="\"INVALID\" is not a valid start token"
monitor_1 | level=warn ts=2018-09-14T15:35:01.486844385Z caller=scrape.go:804 component="scrape manager" scrape_pool=dgraph target=http://cluster0-node2:8082/debug/vars msg="append failed" err="\"INVALID\" is not a valid start token"
monitor_1 | level=warn ts=2018-09-14T15:35:01.919575602Z caller=scrape.go:804 component="scrape manager" scrape_pool=dgraph target=http://cluster0-node1:8081/debug/vars msg="append failed" err="\"INVALID\" is not a valid start token"
monitor_1 | level=warn ts=2018-09-14T15:35:02.171994766Z caller=scrape.go:804 component="scrape manager" scrape_pool=dgraph target=http://cluster0-node0:8080/debug/vars msg="append failed" err="\"INVALID\" is not a valid start token"
monitor_1 | level=warn ts=2018-09-14T15:35:02.21152201Z caller=scrape.go:804 component="scrape manager" scrape_pool=dgraph target=http://zero:6080/debug/vars msg="append failed" err="\"INVALID\" is not a valid start token"
monitor_1 | level=warn ts=2018-09-14T15:35:03.486762735Z caller=scrape.go:804 component="scrape manager" scrape_pool=dgraph target=http://cluster0-node2:8082/debug/vars msg="append failed" err="\"INVALID\" is not a valid start token"
monitor_1 | level=warn ts=2018-09-14T15:35:03.919575689Z caller=scrape.go:804 component="scrape manager" scrape_pool=dgraph target=http://cluster0-node1:8081/debug/vars msg="append failed" err="\"INVALID\" is not a valid start token"
monitor_1 | level=warn ts=2018-09-14T15:35:04.171939035Z caller=scrape.go:804 component="scrape manager" scrape_pool=dgraph target=http://cluster0-node0:8080/debug/vars msg="append failed" err="\"INVALID\" is not a valid start token"
monitor_1 | level=warn ts=2018-09-14T15:35:04.211553545Z caller=scrape.go:804 component="scrape manager" scrape_pool=dgraph target=http://zero:6080/debug/vars msg="append failed" err="\"INVALID\" is not a valid start token"
monitor_1 | level=warn ts=2018-09-14T15:35:05.486810769Z caller=scrape.go:804 component="scrape manager" scrape_pool=dgraph target=http://cluster0-node2:8082/debug/vars msg="append failed" err="\"INVALID\" is not a valid start token"
monitor_1 | level=warn ts=2018-09-14T15:35:05.919468184Z caller=scrape.go:804 component="scrape manager" scrape_pool=dgraph target=http://cluster0-node1:8081/debug/vars msg="append failed" err="\"INVALID\" is not a valid start token"
monitor_1 | level=warn ts=2018-09-14T15:35:06.172312584Z caller=scrape.go:804 component="scrape manager" scrape_pool=dgraph target=http://cluster0-node0:8080/debug/vars msg="append failed" err="\"INVALID\" is not a valid start token"
monitor_1 | level=warn ts=2018-09-14T15:35:06.211265409Z caller=scrape.go:804 component="scrape manager" scrape_pool=dgraph target=http://zero:6080/debug/vars msg="append failed" err="\"INVALID\" is not a valid start token"
monitor_1 | level=warn ts=2018-09-14T15:35:07.48678432Z caller=scrape.go:804 component="scrape manager" scrape_pool=dgraph target=http://cluster0-node2:8082/debug/vars msg="append failed" err="\"INVALID\" is not a valid start token"
monitor_1 | level=warn ts=2018-09-14T15:35:07.919484644Z caller=scrape.go:804 component="scrape manager" scrape_pool=dgraph target=http://cluster0-node1:8081/debug/vars msg="append failed" err="\"INVALID\" is not a valid start token"
monitor_1 | level=warn ts=2018-09-14T15:35:08.172268535Z caller=scrape.go:804 component="scrape manager" scrape_pool=dgraph target=http://cluster0-node0:8080/debug/vars msg="append failed" err="\"INVALID\" is not a valid start token"
monitor_1 | level=warn ts=2018-09-14T15:35:08.211560802Z caller=scrape.go:804 component="scrape manager" scrape_pool=dgraph target=http://zero:6080/debug/vars msg="append failed" err="\"INVALID\" is not a valid start token"