From ae28975038cb25655da968e3f043210749ba382b Mon Sep 17 00:00:00 2001 From: Abhinav Dangeti Date: Wed, 17 Feb 2021 13:45:09 -0700 Subject: [PATCH] MB-44216: Race in closing moss store and obtaining stats + There appears to be racey code path in obtaining stats for a moss store that has since been closed. + Fetch histograms from moss within lock to avoid this panic .. http: panic serving 127.0.0.1:36252: runtime error: invalid memory address or nil pointer dereference goroutine 1069 [running]: net/http.(*conn).serve.func1(0xc0004301e0) /home/couchbase/.cbdepscache/exploded/x86_64/go-1.13.7/go/src/net/http/server.go:1767 +0x139 panic(0xffeba0, 0x1cb22b0) /home/couchbase/.cbdepscache/exploded/x86_64/go-1.13.7/go/src/runtime/panic.go:679 +0x1b2 github.com/couchbase/moss.(*Store).Histograms(...) /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/gopkg/go-1.13.7/pkg/mod/github.com/couchbase/moss@v0.1.0/store_stats.go:89 github.com/blevesearch/bleve/v2/index/upsidedown/store/moss.(*stats).statsMap(0xc0000110c0, 0x7faa69bc5a70) /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/gopkg/go-1.13.7/pkg/mod/github.com/blevesearch/bleve/v2@v2.0.1/index/upsidedown/store/moss/stats.go:48 +0x27b github.com/blevesearch/bleve/v2/index/upsidedown/store/moss.(*Store).StatsMap(0xc0020f2820, 0x13431c0) /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/gopkg/go-1.13.7/pkg/mod/github.com/blevesearch/bleve/v2@v2.0.1/index/upsidedown/store/moss/store.go:214 +0x2f github.com/blevesearch/bleve/v2/index/upsidedown.(*indexStat).statsMap(0xc0020f2780, 0xc0040ba780) /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/gopkg/go-1.13.7/pkg/mod/github.com/blevesearch/bleve/v2@v2.0.1/index/upsidedown/stats.go:46 +0x478 github.com/blevesearch/bleve/v2/index/upsidedown.(*UpsideDownCouch).StatsMap(0xc0020f0480, 0x6) /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/gopkg/go-1.13.7/pkg/mod/github.com/blevesearch/bleve/v2@v2.0.1/index/upsidedown/upsidedown.go:1022 +0x2f github.com/blevesearch/bleve/v2.(*IndexStat).statsMap(0xc0020efb40, 0xc0040ba750) /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/gopkg/go-1.13.7/pkg/mod/github.com/blevesearch/bleve/v2@v2.0.1/index_stats.go:31 +0x4a github.com/blevesearch/bleve/v2.(*indexImpl).StatsMap(0xc0002c8690, 0x0) /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/gopkg/go-1.13.7/pkg/mod/github.com/blevesearch/bleve/v2@v2.0.1/index_impl.go:791 +0x2f github.com/couchbase/cbft.(*BleveDest).StatsMap(0xc0020ddad0, 0xf930e0, 0xc003eba100, 0xc00225e270) /home/couchbase/jenkins/workspace/couchbase-server-unix/cbft/pindex_bleve.go:1911 +0xb2 github.com/couchbase/cbft.addPIndexStats(0xc001cd1300, 0xc003ed9b00, 0x11a15dd, 0x13) /home/couchbase/jenkins/workspace/couchbase-server-unix/cbft/ns_server.go:596 +0x73 github.com/couchbase/cbft.gatherIndexStats(0xc000422e00, 0xc001fdf000, 0x1, 0xc0003d5a40, 0x1c, 0xc0044c5460) /home/couchbase/jenkins/workspace/couchbase-server-unix/cbft/ns_server.go:350 +0x165a github.com/couchbase/cbft.(*PrometheusHighMetricsHandler).ServeHTTP(0xc0003e7fe0, 0x1343c40, 0xc003aa0760, 0xc0039dad00) /home/couchbase/jenkins/workspace/couchbase-server-unix/cbft/prometheus.go:113 +0x144 github.com/couchbase/cbgt/rest.(*HandlerWithRESTMeta).ServeHTTP(0xc00050e480, 0x13455c0, 0xc000518ee0, 0xc0039dad00) /home/couchbase/jenkins/workspace/couchbase-server-unix/cbgt/rest/rest.go:254 +0x10c github.com/couchbase/cbft.(*AuthVersionHandler).ServeHTTP(0xc0004cd700, 0x13455c0, 0xc000518ee0, 0xc0039dad00) /home/couchbase/jenkins/workspace/couchbase-server-unix/cbft/rest.go:174 +0x13d main.exportMuxRoutesToHttprouter.func1.1(0x13455c0, 0xc000518ee0, 0xc0039dac00, 0x0, 0x0, 0x0) cbft/cmd/cbft/main.go:729 +0xa1 github.com/julienschmidt/httprouter.(*Router).ServeHTTP(0xc0002a3240, 0x13455c0, 0xc000518ee0, 0xc0039dac00) /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/gopkg/go-1.13.7/pkg/mod/github.com/julienschmidt/httprouter@v1.1.1-0.20170430222011-975b5c4c7c21/router.go:344 +0x932 net/http.serverHandler.ServeHTTP(0xc000214460, 0x13455c0, 0xc000518ee0, 0xc0039dac00) /home/couchbase/.cbdepscache/exploded/x86_64/go-1.13.7/go/src/net/http/server.go:2802 +0xa4 net/http.(*conn).serve(0xc0004301e0, 0x1349ec0, 0xc001a2e200) /home/couchbase/.cbdepscache/exploded/x86_64/go-1.13.7/go/src/net/http/server.go:1890 +0x875 created by net/http.(*Server).Serve /home/couchbase/.cbdepscache/exploded/x86_64/go-1.13.7/go/src/net/http/server.go:2928 +0x384 --- index/upsidedown/store/moss/lower.go | 10 ++++++++++ index/upsidedown/store/moss/stats.go | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/index/upsidedown/store/moss/lower.go b/index/upsidedown/store/moss/lower.go index b0f004c09..4e5d40d60 100644 --- a/index/upsidedown/store/moss/lower.go +++ b/index/upsidedown/store/moss/lower.go @@ -553,3 +553,13 @@ func (w *mossStoreWrapper) Actual() *moss.Store { w.m.Unlock() return rv } + +func (w *mossStoreWrapper) histograms() string { + var rv string + w.m.Lock() + if w.s != nil { + rv = w.s.Histograms().String() + } + w.m.Unlock() + return rv +} diff --git a/index/upsidedown/store/moss/stats.go b/index/upsidedown/store/moss/stats.go index c39fc4b99..5ebc4a93f 100644 --- a/index/upsidedown/store/moss/stats.go +++ b/index/upsidedown/store/moss/stats.go @@ -45,7 +45,7 @@ func (s *stats) statsMap() map[string]interface{} { } if msw, ok := s.s.llstore.(*mossStoreWrapper); ok { - ms["store_histograms"] = msw.Actual().Histograms().String() + ms["store_histograms"] = msw.histograms() } ms["coll_histograms"] = s.s.ms.Histograms().String()