Optimize Dgraph Memory Usage

(pprof) list processTask

         .     1.91GB    887:	out, err := qs.helpProcessTask(ctx, q, gid)
         .          .    888:	if err != nil {
    4.50MB     4.45GB    889:		return &pb.Result{}, err // Create this object once, use it repeatedly.
         .          .    890:	}
         .     4.97GB    891:	return out, nil
         .          .    892:}

         .          .    437:			//
         .          .    438:			// The variable __dgraph_0__ will -
         .          .    439:			//      * be empty if the condition is true
         .          .    440:			//      * have 1 UID (the 0 UID) if the condition is false
    1.44GB     1.44GB    441:			upsertQuery += qc.condVars[i] + ` as var(func: uid(0)) ` + cond + `
         .          .    442:			 `
         .          .    443:		}
         .          .    444:	}
    6.58MB     6.58MB    445:	upsertQuery += `}`

Potentially use strings.Builder here.

CC: @ashishgoswami , @harshil_goel

(pprof) list Uids.func1
Total: 11.56GB
ROUTINE ======================== github.com/dgraph-io/dgraph/posting.(*List).Uids.func1 in github.com/dgraph-io/dgraph/posting/list.go
    8.59GB     8.59GB (flat, cum) 74.31% of Total
         .          .   1001:		return out, nil
         .          .   1002:	}
         .          .   1003:
         .          .   1004:	err := l.iterate(opt.ReadTs, opt.AfterUid, func(p *pb.Posting) error {
         .          .   1005:		if p.PostingType == pb.Posting_REF {
    8.59GB     8.59GB   1006:			res = append(res, p.Uid)
         .          .   1007:		}
         .          .   1008:		return nil
         .          .   1009:	})
         .          .   1010:	l.RUnlock()
         .          .   1011:	if err != nil {
(pprof) list algo.MergeSorted
Total: 11.56GB
ROUTINE ======================== github.com/dgraph-io/dgraph/algo.MergeSorted in github.com/dgraph-io/dgraph/algo/uidlist.go
    1.74GB     1.74GB (flat, cum) 15.05% of Total
         .          .    383:			}
         .          .    384:		}
         .          .    385:	}
         .          .    386:
         .          .    387:	// Our final output. Give it an approximate capacity as copies are expensive.
    1.74GB     1.74GB    388:	output := make([]uint64, 0, maxSz)
         .          .    389:	// idx[i] is the element we are looking at for lists[i].
         .          .    390:	idx := make([]int, len(lists))
         .          .    391:	var last uint64   // Last element added to sorted / final output.
         .          .    392:	for h.Len() > 0 { // While heap is not empty.
         .          .    393:		me := (*h)[0] // Peek at the top element in heap.
         .          .    394:		if len(output) == 0 || me.val != last {
         .          .    395:			output = append(output, me.val) // Add if unique.
         .          .    396:			last = me.val
         .          .    397:		}
         .          .    398:		l := lists[me.listIdx]
         .          .    399:		if idx[me.listIdx] >= len(l.Uids)-1 {
         .          .    400:			heap.Pop(h)
         .          .    401:		} else {
         .          .    402:			idx[me.listIdx]++
         .          .    403:			val := l.Uids[idx[me.listIdx]]
         .          .    404:			(*h)[0].val = val
         .          .    405:			heap.Fix(h, 0) // Faster than Pop() followed by Push().
         .          .    406:		}
         .          .    407:	}
       1MB        1MB    408:	return &pb.List{Uids: output}
         .          .    409:}
         .          .    410:
         .          .    411:// IndexOf performs a binary search on the uids slice and returns the index at
         .          .    412:// which it finds the uid, else returns -1
         .          .    413:func IndexOf(u *pb.List, uid uint64) int {

This is another reason to switch to roaring bitmaps.

PR Worker: Return nil on error by jarifibrahim · Pull Request #5414 · dgraph-io/dgraph · GitHub reduces the memory used by

(pprof) list processTask

         .     1.91GB    887:	out, err := qs.helpProcessTask(ctx, q, gid)
         .          .    888:	if err != nil {
    4.50MB     4.45GB    889:		return &pb.Result{}, err // Create this object once, use it repeatedly.
         .          .    890:	}
         .     4.97GB    891:	return out, nil
         .          .    892:}