1
0

synapse.json 292 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106610761086109611061116112611361146115611661176118611961206121612261236124612561266127612861296130613161326133613461356136613761386139614061416142614361446145614661476148614961506151615261536154615561566157615861596160616161626163616461656166616761686169617061716172617361746175617661776178617961806181618261836184618561866187618861896190619161926193619461956196619761986199620062016202620362046205620662076208620962106211621262136214621562166217621862196220622162226223622462256226622762286229623062316232623362346235623662376238623962406241624262436244624562466247624862496250625162526253625462556256625762586259626062616262626362646265626662676268626962706271627262736274627562766277627862796280628162826283628462856286628762886289629062916292629362946295629662976298629963006301630263036304630563066307630863096310631163126313631463156316631763186319632063216322632363246325632663276328632963306331633263336334633563366337633863396340634163426343634463456346634763486349635063516352635363546355635663576358635963606361636263636364636563666367636863696370637163726373637463756376637763786379638063816382638363846385638663876388638963906391639263936394639563966397639863996400640164026403640464056406640764086409641064116412641364146415641664176418641964206421642264236424642564266427642864296430643164326433643464356436643764386439644064416442644364446445644664476448644964506451645264536454645564566457645864596460646164626463646464656466646764686469647064716472647364746475647664776478647964806481648264836484648564866487648864896490649164926493649464956496649764986499650065016502650365046505650665076508650965106511651265136514651565166517651865196520652165226523652465256526652765286529653065316532653365346535653665376538653965406541654265436544654565466547654865496550655165526553655465556556655765586559656065616562656365646565656665676568656965706571657265736574657565766577657865796580658165826583658465856586658765886589659065916592659365946595659665976598659966006601660266036604660566066607660866096610661166126613661466156616661766186619662066216622662366246625662666276628662966306631663266336634663566366637663866396640664166426643664466456646664766486649665066516652665366546655665666576658665966606661666266636664666566666667666866696670667166726673667466756676667766786679668066816682668366846685668666876688668966906691669266936694669566966697669866996700670167026703670467056706670767086709671067116712671367146715671667176718671967206721672267236724672567266727672867296730673167326733673467356736673767386739674067416742674367446745674667476748674967506751675267536754675567566757675867596760676167626763676467656766676767686769677067716772677367746775677667776778677967806781678267836784678567866787678867896790679167926793679467956796679767986799680068016802680368046805680668076808680968106811681268136814681568166817681868196820682168226823682468256826682768286829683068316832683368346835683668376838683968406841684268436844684568466847684868496850685168526853685468556856685768586859686068616862686368646865686668676868686968706871687268736874687568766877687868796880688168826883688468856886688768886889689068916892689368946895689668976898689969006901690269036904690569066907690869096910691169126913691469156916691769186919692069216922692369246925692669276928692969306931693269336934693569366937693869396940694169426943694469456946694769486949695069516952695369546955695669576958695969606961696269636964696569666967696869696970697169726973697469756976697769786979698069816982698369846985698669876988698969906991699269936994699569966997699869997000700170027003700470057006700770087009701070117012701370147015701670177018701970207021702270237024702570267027702870297030703170327033703470357036703770387039704070417042704370447045704670477048704970507051705270537054705570567057705870597060706170627063706470657066706770687069707070717072707370747075707670777078707970807081708270837084708570867087708870897090709170927093709470957096709770987099710071017102710371047105710671077108710971107111711271137114711571167117711871197120712171227123712471257126712771287129713071317132713371347135713671377138713971407141714271437144714571467147714871497150715171527153715471557156715771587159716071617162716371647165716671677168716971707171717271737174717571767177717871797180718171827183718471857186718771887189719071917192719371947195719671977198719972007201720272037204720572067207720872097210721172127213721472157216721772187219722072217222722372247225722672277228722972307231723272337234723572367237723872397240724172427243724472457246724772487249725072517252725372547255725672577258725972607261726272637264726572667267726872697270727172727273727472757276727772787279728072817282728372847285728672877288728972907291729272937294729572967297729872997300730173027303730473057306730773087309731073117312731373147315731673177318731973207321732273237324732573267327732873297330733173327333733473357336733773387339734073417342734373447345734673477348734973507351735273537354735573567357735873597360736173627363736473657366736773687369737073717372737373747375737673777378737973807381738273837384738573867387738873897390739173927393739473957396739773987399740074017402740374047405740674077408740974107411741274137414741574167417741874197420742174227423742474257426742774287429743074317432743374347435743674377438743974407441744274437444744574467447744874497450745174527453745474557456745774587459746074617462746374647465746674677468746974707471747274737474747574767477747874797480748174827483748474857486748774887489749074917492749374947495749674977498749975007501750275037504750575067507750875097510751175127513751475157516751775187519752075217522752375247525752675277528752975307531753275337534753575367537753875397540754175427543754475457546754775487549755075517552755375547555755675577558755975607561756275637564756575667567756875697570757175727573757475757576757775787579758075817582758375847585758675877588758975907591759275937594759575967597759875997600760176027603760476057606760776087609761076117612761376147615761676177618761976207621762276237624762576267627762876297630763176327633763476357636763776387639764076417642764376447645764676477648764976507651765276537654765576567657765876597660766176627663766476657666766776687669767076717672767376747675767676777678767976807681768276837684768576867687768876897690769176927693769476957696769776987699770077017702770377047705770677077708770977107711771277137714771577167717771877197720772177227723772477257726772777287729773077317732773377347735773677377738773977407741774277437744774577467747774877497750775177527753775477557756775777587759776077617762776377647765776677677768776977707771777277737774777577767777777877797780778177827783778477857786778777887789779077917792779377947795779677977798779978007801780278037804780578067807780878097810781178127813781478157816781778187819782078217822782378247825782678277828782978307831783278337834783578367837783878397840784178427843784478457846784778487849785078517852785378547855785678577858785978607861786278637864786578667867786878697870787178727873787478757876787778787879788078817882788378847885788678877888788978907891789278937894789578967897789878997900790179027903790479057906790779087909791079117912791379147915791679177918791979207921792279237924792579267927792879297930793179327933793479357936793779387939794079417942794379447945794679477948794979507951795279537954795579567957795879597960796179627963796479657966796779687969797079717972797379747975797679777978797979807981798279837984798579867987798879897990799179927993799479957996799779987999800080018002800380048005800680078008800980108011801280138014801580168017801880198020802180228023802480258026802780288029803080318032803380348035803680378038803980408041804280438044804580468047804880498050805180528053805480558056805780588059806080618062806380648065806680678068806980708071807280738074807580768077807880798080808180828083808480858086808780888089809080918092809380948095809680978098809981008101810281038104810581068107810881098110811181128113811481158116811781188119812081218122812381248125812681278128812981308131813281338134813581368137813881398140814181428143814481458146814781488149815081518152815381548155815681578158815981608161816281638164816581668167816881698170817181728173817481758176817781788179818081818182818381848185818681878188818981908191819281938194819581968197819881998200820182028203820482058206820782088209821082118212821382148215821682178218821982208221822282238224822582268227822882298230823182328233823482358236823782388239824082418242824382448245824682478248824982508251825282538254825582568257825882598260826182628263826482658266826782688269827082718272827382748275827682778278827982808281828282838284828582868287828882898290829182928293829482958296829782988299830083018302830383048305830683078308830983108311831283138314831583168317831883198320832183228323832483258326832783288329833083318332833383348335833683378338833983408341834283438344834583468347834883498350835183528353835483558356835783588359836083618362836383648365836683678368836983708371837283738374837583768377837883798380838183828383838483858386838783888389839083918392839383948395839683978398839984008401840284038404840584068407840884098410841184128413841484158416841784188419842084218422842384248425842684278428842984308431843284338434843584368437843884398440844184428443844484458446844784488449845084518452845384548455845684578458845984608461846284638464846584668467846884698470847184728473847484758476847784788479848084818482848384848485848684878488848984908491849284938494849584968497849884998500850185028503850485058506850785088509851085118512851385148515851685178518851985208521852285238524852585268527852885298530853185328533853485358536853785388539854085418542854385448545854685478548854985508551855285538554855585568557855885598560856185628563856485658566856785688569857085718572857385748575857685778578857985808581858285838584858585868587858885898590859185928593859485958596859785988599860086018602860386048605860686078608860986108611861286138614861586168617861886198620862186228623862486258626862786288629863086318632863386348635863686378638863986408641864286438644864586468647864886498650865186528653865486558656865786588659866086618662866386648665866686678668866986708671867286738674867586768677867886798680868186828683868486858686868786888689869086918692869386948695869686978698869987008701870287038704870587068707870887098710871187128713871487158716871787188719872087218722872387248725872687278728872987308731873287338734873587368737873887398740874187428743874487458746874787488749875087518752875387548755875687578758875987608761876287638764876587668767876887698770877187728773877487758776877787788779878087818782878387848785878687878788878987908791879287938794879587968797879887998800880188028803880488058806880788088809881088118812881388148815881688178818881988208821882288238824882588268827882888298830883188328833883488358836883788388839884088418842884388448845884688478848884988508851885288538854885588568857885888598860886188628863886488658866886788688869887088718872887388748875887688778878887988808881888288838884888588868887888888898890889188928893889488958896889788988899890089018902890389048905890689078908890989108911891289138914891589168917891889198920892189228923892489258926892789288929893089318932893389348935893689378938893989408941894289438944894589468947894889498950895189528953895489558956895789588959896089618962896389648965896689678968896989708971897289738974897589768977897889798980898189828983898489858986898789888989899089918992899389948995899689978998899990009001900290039004900590069007900890099010901190129013901490159016901790189019902090219022902390249025902690279028902990309031903290339034903590369037903890399040904190429043904490459046904790489049905090519052905390549055905690579058905990609061906290639064906590669067906890699070907190729073907490759076907790789079908090819082908390849085908690879088908990909091909290939094909590969097909890999100910191029103910491059106910791089109911091119112911391149115911691179118911991209121912291239124912591269127912891299130913191329133913491359136913791389139914091419142914391449145914691479148914991509151915291539154915591569157915891599160916191629163916491659166916791689169917091719172917391749175917691779178917991809181918291839184918591869187918891899190919191929193919491959196919791989199920092019202920392049205920692079208920992109211921292139214921592169217921892199220922192229223922492259226922792289229923092319232923392349235923692379238923992409241924292439244924592469247924892499250925192529253925492559256925792589259926092619262926392649265926692679268926992709271927292739274927592769277927892799280928192829283928492859286928792889289929092919292929392949295929692979298929993009301930293039304930593069307930893099310931193129313931493159316931793189319932093219322932393249325932693279328932993309331933293339334933593369337933893399340934193429343934493459346934793489349935093519352935393549355935693579358935993609361936293639364936593669367936893699370937193729373937493759376937793789379938093819382938393849385938693879388938993909391939293939394939593969397939893999400940194029403940494059406940794089409941094119412941394149415941694179418941994209421942294239424942594269427942894299430943194329433943494359436943794389439944094419442944394449445944694479448944994509451945294539454945594569457945894599460946194629463946494659466946794689469947094719472947394749475947694779478947994809481948294839484948594869487948894899490949194929493949494959496949794989499950095019502950395049505950695079508950995109511951295139514951595169517951895199520952195229523952495259526952795289529953095319532953395349535953695379538953995409541954295439544954595469547954895499550955195529553955495559556955795589559956095619562956395649565956695679568956995709571957295739574957595769577957895799580958195829583958495859586958795889589959095919592959395949595959695979598959996009601960296039604960596069607960896099610961196129613961496159616961796189619962096219622962396249625962696279628962996309631963296339634963596369637963896399640964196429643964496459646964796489649965096519652965396549655965696579658965996609661966296639664966596669667966896699670967196729673967496759676967796789679968096819682968396849685968696879688968996909691969296939694969596969697969896999700970197029703970497059706970797089709971097119712971397149715971697179718971997209721972297239724972597269727972897299730973197329733973497359736973797389739974097419742974397449745974697479748974997509751975297539754975597569757975897599760976197629763976497659766976797689769977097719772977397749775977697779778977997809781978297839784978597869787978897899790979197929793979497959796979797989799980098019802980398049805980698079808980998109811981298139814981598169817981898199820982198229823982498259826982798289829983098319832983398349835983698379838983998409841984298439844984598469847984898499850985198529853985498559856985798589859986098619862986398649865986698679868986998709871987298739874987598769877987898799880988198829883988498859886988798889889989098919892989398949895989698979898989999009901990299039904990599069907990899099910991199129913991499159916991799189919992099219922992399249925992699279928992999309931993299339934993599369937993899399940994199429943994499459946994799489949995099519952995399549955995699579958995999609961996299639964996599669967996899699970997199729973997499759976997799789979998099819982998399849985998699879988998999909991999299939994999599969997999899991000010001100021000310004100051000610007100081000910010100111001210013100141001510016100171001810019100201002110022100231002410025100261002710028100291003010031100321003310034100351003610037100381003910040100411004210043100441004510046100471004810049100501005110052100531005410055100561005710058100591006010061100621006310064100651006610067100681006910070100711007210073100741007510076100771007810079100801008110082100831008410085100861008710088100891009010091100921009310094100951009610097100981009910100101011010210103101041010510106101071010810109101101011110112101131011410115101161011710118101191012010121101221012310124101251012610127101281012910130101311013210133101341013510136101371013810139101401014110142101431014410145101461014710148101491015010151101521015310154101551015610157101581015910160101611016210163101641016510166101671016810169101701017110172101731017410175101761017710178101791018010181101821018310184101851018610187101881018910190101911019210193101941019510196101971019810199102001020110202102031020410205102061020710208102091021010211102121021310214102151021610217102181021910220102211022210223102241022510226102271022810229102301023110232102331023410235102361023710238102391024010241102421024310244102451024610247102481024910250102511025210253102541025510256102571025810259102601026110262102631026410265102661026710268102691027010271102721027310274102751027610277102781027910280102811028210283102841028510286102871028810289102901029110292102931029410295102961029710298102991030010301103021030310304103051030610307103081030910310103111031210313103141031510316103171031810319103201032110322103231032410325103261032710328103291033010331103321033310334103351033610337103381033910340103411034210343103441034510346103471034810349103501035110352103531035410355103561035710358103591036010361103621036310364103651036610367103681036910370103711037210373103741037510376103771037810379103801038110382103831038410385103861038710388103891039010391103921039310394103951039610397103981039910400104011040210403104041040510406104071040810409104101041110412104131041410415104161041710418104191042010421104221042310424104251042610427104281042910430104311043210433104341043510436104371043810439104401044110442104431044410445104461044710448104491045010451104521045310454104551045610457104581045910460104611046210463104641046510466104671046810469104701047110472104731047410475104761047710478104791048010481104821048310484104851048610487104881048910490104911049210493104941049510496104971049810499105001050110502105031050410505105061050710508105091051010511105121051310514105151051610517105181051910520105211052210523105241052510526105271052810529105301053110532105331053410535105361053710538105391054010541105421054310544105451054610547105481054910550105511055210553105541055510556105571055810559105601056110562105631056410565105661056710568105691057010571105721057310574105751057610577105781057910580105811058210583105841058510586105871058810589105901059110592105931059410595105961059710598105991060010601106021060310604106051060610607106081060910610106111061210613106141061510616106171061810619106201062110622106231062410625106261062710628106291063010631106321063310634106351063610637106381063910640106411064210643106441064510646106471064810649106501065110652106531065410655106561065710658106591066010661106621066310664106651066610667106681066910670106711067210673106741067510676106771067810679106801068110682106831068410685106861068710688106891069010691106921069310694106951069610697106981069910700107011070210703107041070510706107071070810709107101071110712107131071410715107161071710718107191072010721107221072310724107251072610727107281072910730107311073210733107341073510736107371073810739107401074110742107431074410745107461074710748107491075010751107521075310754107551075610757107581075910760107611076210763107641076510766107671076810769107701077110772107731077410775107761077710778107791078010781107821078310784107851078610787107881078910790107911079210793107941079510796107971079810799108001080110802108031080410805108061080710808108091081010811108121081310814108151081610817108181081910820108211082210823108241082510826108271082810829108301083110832108331083410835108361083710838108391084010841108421084310844108451084610847108481084910850108511085210853108541085510856108571085810859108601086110862108631086410865108661086710868108691087010871108721087310874108751087610877108781087910880108811088210883108841088510886108871088810889108901089110892108931089410895108961089710898108991090010901109021090310904
  1. {
  2. "__inputs": [
  3. {
  4. "name": "DS_PROMETHEUS",
  5. "label": "Prometheus",
  6. "description": "",
  7. "type": "datasource",
  8. "pluginId": "prometheus",
  9. "pluginName": "Prometheus"
  10. }
  11. ],
  12. "__requires": [
  13. {
  14. "type": "grafana",
  15. "id": "grafana",
  16. "name": "Grafana",
  17. "version": "7.3.7"
  18. },
  19. {
  20. "type": "panel",
  21. "id": "graph",
  22. "name": "Graph",
  23. "version": ""
  24. },
  25. {
  26. "type": "panel",
  27. "id": "heatmap",
  28. "name": "Heatmap",
  29. "version": ""
  30. },
  31. {
  32. "type": "datasource",
  33. "id": "prometheus",
  34. "name": "Prometheus",
  35. "version": "1.0.0"
  36. }
  37. ],
  38. "annotations": {
  39. "list": [
  40. {
  41. "builtIn": 1,
  42. "datasource": "$datasource",
  43. "enable": false,
  44. "hide": true,
  45. "iconColor": "rgba(0, 211, 255, 1)",
  46. "limit": 100,
  47. "name": "Annotations & Alerts",
  48. "showIn": 0,
  49. "type": "dashboard"
  50. }
  51. ]
  52. },
  53. "editable": true,
  54. "gnetId": null,
  55. "graphTooltip": 0,
  56. "id": null,
  57. "iteration": 1628606819564,
  58. "links": [
  59. {
  60. "asDropdown": false,
  61. "icon": "external link",
  62. "includeVars": true,
  63. "keepTime": true,
  64. "tags": [
  65. "matrix"
  66. ],
  67. "title": "Dashboards",
  68. "type": "dashboards"
  69. },
  70. {
  71. "asDropdown": false,
  72. "icon": "external link",
  73. "includeVars": false,
  74. "keepTime": false,
  75. "tags": [],
  76. "targetBlank": true,
  77. "title": "Synapse Documentation",
  78. "tooltip": "Open Documentation",
  79. "type": "link",
  80. "url": "https://matrix-org.github.io/synapse/latest/"
  81. }
  82. ],
  83. "panels": [
  84. {
  85. "collapsed": false,
  86. "datasource": "${DS_PROMETHEUS}",
  87. "gridPos": {
  88. "h": 1,
  89. "w": 24,
  90. "x": 0,
  91. "y": 0
  92. },
  93. "id": 73,
  94. "panels": [],
  95. "title": "Overview",
  96. "type": "row"
  97. },
  98. {
  99. "cards": {
  100. "cardPadding": -1,
  101. "cardRound": 0
  102. },
  103. "color": {
  104. "cardColor": "#b4ff00",
  105. "colorScale": "sqrt",
  106. "colorScheme": "interpolateInferno",
  107. "exponent": 0.5,
  108. "mode": "spectrum"
  109. },
  110. "dataFormat": "tsbuckets",
  111. "datasource": "$datasource",
  112. "fieldConfig": {
  113. "defaults": {
  114. "custom": {}
  115. },
  116. "overrides": []
  117. },
  118. "gridPos": {
  119. "h": 9,
  120. "w": 12,
  121. "x": 0,
  122. "y": 1
  123. },
  124. "heatmap": {},
  125. "hideZeroBuckets": false,
  126. "highlightCards": true,
  127. "id": 189,
  128. "legend": {
  129. "show": false
  130. },
  131. "links": [],
  132. "reverseYBuckets": false,
  133. "targets": [
  134. {
  135. "expr": "sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le)",
  136. "format": "heatmap",
  137. "interval": "",
  138. "intervalFactor": 1,
  139. "legendFormat": "{{le}}",
  140. "refId": "A"
  141. }
  142. ],
  143. "title": "Event Send Time (excluding errors, all workers)",
  144. "tooltip": {
  145. "show": true,
  146. "showHistogram": true
  147. },
  148. "type": "heatmap",
  149. "xAxis": {
  150. "show": true
  151. },
  152. "xBucketNumber": null,
  153. "xBucketSize": null,
  154. "yAxis": {
  155. "decimals": null,
  156. "format": "s",
  157. "logBase": 2,
  158. "max": null,
  159. "min": null,
  160. "show": true,
  161. "splitFactor": null
  162. },
  163. "yBucketBound": "auto",
  164. "yBucketNumber": null,
  165. "yBucketSize": null
  166. },
  167. {
  168. "aliasColors": {},
  169. "bars": false,
  170. "dashLength": 10,
  171. "dashes": false,
  172. "datasource": "$datasource",
  173. "description": "",
  174. "fieldConfig": {
  175. "defaults": {
  176. "custom": {},
  177. "links": []
  178. },
  179. "overrides": []
  180. },
  181. "fill": 0,
  182. "fillGradient": 0,
  183. "gridPos": {
  184. "h": 9,
  185. "w": 12,
  186. "x": 12,
  187. "y": 1
  188. },
  189. "hiddenSeries": false,
  190. "id": 152,
  191. "legend": {
  192. "avg": false,
  193. "current": false,
  194. "max": false,
  195. "min": false,
  196. "rightSide": false,
  197. "show": true,
  198. "total": false,
  199. "values": false
  200. },
  201. "lines": true,
  202. "linewidth": 0,
  203. "links": [],
  204. "nullPointMode": "connected",
  205. "options": {
  206. "alertThreshold": true
  207. },
  208. "paceLength": 10,
  209. "percentage": false,
  210. "pluginVersion": "7.3.7",
  211. "pointradius": 5,
  212. "points": false,
  213. "renderer": "flot",
  214. "seriesOverrides": [
  215. {
  216. "alias": "Avg",
  217. "fill": 0,
  218. "linewidth": 3
  219. },
  220. {
  221. "alias": "99%",
  222. "color": "#C4162A",
  223. "fillBelowTo": "90%"
  224. },
  225. {
  226. "alias": "90%",
  227. "color": "#FF7383",
  228. "fillBelowTo": "75%"
  229. },
  230. {
  231. "alias": "75%",
  232. "color": "#FFEE52",
  233. "fillBelowTo": "50%"
  234. },
  235. {
  236. "alias": "50%",
  237. "color": "#73BF69",
  238. "fillBelowTo": "25%"
  239. },
  240. {
  241. "alias": "25%",
  242. "color": "#1F60C4",
  243. "fillBelowTo": "5%"
  244. },
  245. {
  246. "alias": "5%",
  247. "lines": false
  248. },
  249. {
  250. "alias": "Average",
  251. "color": "rgb(255, 255, 255)",
  252. "lines": true,
  253. "linewidth": 3
  254. },
  255. {
  256. "alias": "Events",
  257. "color": "#B877D9",
  258. "hideTooltip": true,
  259. "points": true,
  260. "yaxis": 2,
  261. "zindex": -3
  262. }
  263. ],
  264. "spaceLength": 10,
  265. "stack": false,
  266. "steppedLine": false,
  267. "targets": [
  268. {
  269. "expr": "histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
  270. "format": "time_series",
  271. "intervalFactor": 1,
  272. "legendFormat": "99%",
  273. "refId": "D"
  274. },
  275. {
  276. "expr": "histogram_quantile(0.9, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
  277. "format": "time_series",
  278. "interval": "",
  279. "intervalFactor": 1,
  280. "legendFormat": "90%",
  281. "refId": "A"
  282. },
  283. {
  284. "expr": "histogram_quantile(0.75, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
  285. "format": "time_series",
  286. "intervalFactor": 1,
  287. "legendFormat": "75%",
  288. "refId": "C"
  289. },
  290. {
  291. "expr": "histogram_quantile(0.5, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
  292. "format": "time_series",
  293. "intervalFactor": 1,
  294. "legendFormat": "50%",
  295. "refId": "B"
  296. },
  297. {
  298. "expr": "histogram_quantile(0.25, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
  299. "legendFormat": "25%",
  300. "refId": "F"
  301. },
  302. {
  303. "expr": "histogram_quantile(0.05, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
  304. "legendFormat": "5%",
  305. "refId": "G"
  306. },
  307. {
  308. "expr": "sum(rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) / sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size]))",
  309. "legendFormat": "Average",
  310. "refId": "H"
  311. },
  312. {
  313. "expr": "sum(rate(synapse_storage_events_persisted_events{instance=\"$instance\"}[$bucket_size]))",
  314. "hide": false,
  315. "instant": false,
  316. "legendFormat": "Events",
  317. "refId": "E"
  318. }
  319. ],
  320. "thresholds": [
  321. {
  322. "colorMode": "warning",
  323. "fill": false,
  324. "line": true,
  325. "op": "gt",
  326. "value": 1,
  327. "yaxis": "left"
  328. },
  329. {
  330. "colorMode": "critical",
  331. "fill": false,
  332. "line": true,
  333. "op": "gt",
  334. "value": 2,
  335. "yaxis": "left"
  336. }
  337. ],
  338. "timeFrom": null,
  339. "timeRegions": [],
  340. "timeShift": null,
  341. "title": "Event Send Time Quantiles (excluding errors, all workers)",
  342. "tooltip": {
  343. "shared": true,
  344. "sort": 2,
  345. "value_type": "individual"
  346. },
  347. "type": "graph",
  348. "xaxis": {
  349. "buckets": null,
  350. "mode": "time",
  351. "name": null,
  352. "show": true,
  353. "values": []
  354. },
  355. "yaxes": [
  356. {
  357. "decimals": null,
  358. "format": "s",
  359. "label": "",
  360. "logBase": 1,
  361. "max": null,
  362. "min": "0",
  363. "show": true
  364. },
  365. {
  366. "format": "hertz",
  367. "label": "",
  368. "logBase": 1,
  369. "max": null,
  370. "min": "0",
  371. "show": true
  372. }
  373. ],
  374. "yaxis": {
  375. "align": false,
  376. "alignLevel": null
  377. }
  378. },
  379. {
  380. "aliasColors": {},
  381. "bars": false,
  382. "dashLength": 10,
  383. "dashes": false,
  384. "datasource": "$datasource",
  385. "fieldConfig": {
  386. "defaults": {
  387. "custom": {},
  388. "links": []
  389. },
  390. "overrides": []
  391. },
  392. "fill": 1,
  393. "fillGradient": 0,
  394. "gridPos": {
  395. "h": 9,
  396. "w": 12,
  397. "x": 0,
  398. "y": 10
  399. },
  400. "hiddenSeries": false,
  401. "id": 75,
  402. "legend": {
  403. "avg": false,
  404. "current": false,
  405. "max": false,
  406. "min": false,
  407. "show": true,
  408. "total": false,
  409. "values": false
  410. },
  411. "lines": true,
  412. "linewidth": 3,
  413. "links": [],
  414. "nullPointMode": "null",
  415. "options": {
  416. "alertThreshold": true
  417. },
  418. "paceLength": 10,
  419. "percentage": false,
  420. "pluginVersion": "7.3.7",
  421. "pointradius": 5,
  422. "points": false,
  423. "renderer": "flot",
  424. "seriesOverrides": [],
  425. "spaceLength": 10,
  426. "stack": false,
  427. "steppedLine": false,
  428. "targets": [
  429. {
  430. "expr": "rate(process_cpu_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
  431. "format": "time_series",
  432. "interval": "",
  433. "intervalFactor": 1,
  434. "legendFormat": "{{job}}-{{index}} ",
  435. "refId": "A"
  436. }
  437. ],
  438. "thresholds": [
  439. {
  440. "colorMode": "critical",
  441. "fill": true,
  442. "line": true,
  443. "op": "gt",
  444. "value": 1,
  445. "yaxis": "left"
  446. }
  447. ],
  448. "timeFrom": null,
  449. "timeRegions": [],
  450. "timeShift": null,
  451. "title": "CPU usage",
  452. "tooltip": {
  453. "shared": false,
  454. "sort": 0,
  455. "value_type": "individual"
  456. },
  457. "type": "graph",
  458. "xaxis": {
  459. "buckets": null,
  460. "mode": "time",
  461. "name": null,
  462. "show": true,
  463. "values": []
  464. },
  465. "yaxes": [
  466. {
  467. "decimals": null,
  468. "format": "percentunit",
  469. "label": null,
  470. "logBase": 1,
  471. "max": "1.5",
  472. "min": "0",
  473. "show": true
  474. },
  475. {
  476. "format": "short",
  477. "label": null,
  478. "logBase": 1,
  479. "max": null,
  480. "min": null,
  481. "show": true
  482. }
  483. ],
  484. "yaxis": {
  485. "align": false,
  486. "alignLevel": null
  487. }
  488. },
  489. {
  490. "aliasColors": {},
  491. "bars": false,
  492. "dashLength": 10,
  493. "dashes": false,
  494. "datasource": "$datasource",
  495. "editable": true,
  496. "error": false,
  497. "fieldConfig": {
  498. "defaults": {
  499. "custom": {},
  500. "links": []
  501. },
  502. "overrides": []
  503. },
  504. "fill": 1,
  505. "fillGradient": 0,
  506. "grid": {},
  507. "gridPos": {
  508. "h": 9,
  509. "w": 12,
  510. "x": 12,
  511. "y": 10
  512. },
  513. "hiddenSeries": false,
  514. "id": 198,
  515. "legend": {
  516. "avg": false,
  517. "current": false,
  518. "max": false,
  519. "min": false,
  520. "show": true,
  521. "total": false,
  522. "values": false
  523. },
  524. "lines": true,
  525. "linewidth": 3,
  526. "links": [],
  527. "nullPointMode": "null",
  528. "options": {
  529. "alertThreshold": true
  530. },
  531. "paceLength": 10,
  532. "percentage": false,
  533. "pluginVersion": "7.3.7",
  534. "pointradius": 5,
  535. "points": false,
  536. "renderer": "flot",
  537. "seriesOverrides": [],
  538. "spaceLength": 10,
  539. "stack": false,
  540. "steppedLine": false,
  541. "targets": [
  542. {
  543. "expr": "process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
  544. "format": "time_series",
  545. "interval": "",
  546. "intervalFactor": 2,
  547. "legendFormat": "{{job}} {{index}}",
  548. "refId": "A",
  549. "step": 20,
  550. "target": ""
  551. },
  552. {
  553. "expr": "sum(process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})",
  554. "hide": true,
  555. "interval": "",
  556. "legendFormat": "total",
  557. "refId": "B"
  558. }
  559. ],
  560. "thresholds": [],
  561. "timeFrom": null,
  562. "timeRegions": [],
  563. "timeShift": null,
  564. "title": "Memory",
  565. "tooltip": {
  566. "shared": false,
  567. "sort": 0,
  568. "value_type": "cumulative"
  569. },
  570. "transformations": [],
  571. "type": "graph",
  572. "xaxis": {
  573. "buckets": null,
  574. "mode": "time",
  575. "name": null,
  576. "show": true,
  577. "values": []
  578. },
  579. "yaxes": [
  580. {
  581. "format": "bytes",
  582. "logBase": 1,
  583. "max": null,
  584. "min": "0",
  585. "show": true
  586. },
  587. {
  588. "format": "short",
  589. "logBase": 1,
  590. "max": null,
  591. "min": null,
  592. "show": true
  593. }
  594. ],
  595. "yaxis": {
  596. "align": false,
  597. "alignLevel": null
  598. }
  599. },
  600. {
  601. "aliasColors": {},
  602. "bars": false,
  603. "dashLength": 10,
  604. "dashes": false,
  605. "datasource": "$datasource",
  606. "fieldConfig": {
  607. "defaults": {
  608. "custom": {},
  609. "links": []
  610. },
  611. "overrides": []
  612. },
  613. "fill": 1,
  614. "fillGradient": 0,
  615. "gridPos": {
  616. "h": 7,
  617. "w": 12,
  618. "x": 12,
  619. "y": 19
  620. },
  621. "hiddenSeries": false,
  622. "id": 37,
  623. "legend": {
  624. "avg": false,
  625. "current": false,
  626. "max": false,
  627. "min": false,
  628. "show": true,
  629. "total": false,
  630. "values": false
  631. },
  632. "lines": true,
  633. "linewidth": 1,
  634. "links": [],
  635. "nullPointMode": "null",
  636. "options": {
  637. "alertThreshold": true
  638. },
  639. "paceLength": 10,
  640. "percentage": false,
  641. "pluginVersion": "7.3.7",
  642. "pointradius": 5,
  643. "points": false,
  644. "renderer": "flot",
  645. "seriesOverrides": [
  646. {
  647. "alias": "/max$/",
  648. "color": "#890F02",
  649. "fill": 0,
  650. "legend": false
  651. }
  652. ],
  653. "spaceLength": 10,
  654. "stack": false,
  655. "steppedLine": false,
  656. "targets": [
  657. {
  658. "expr": "process_open_fds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
  659. "format": "time_series",
  660. "hide": false,
  661. "interval": "",
  662. "intervalFactor": 2,
  663. "legendFormat": "{{job}}-{{index}}",
  664. "refId": "A",
  665. "step": 20
  666. },
  667. {
  668. "expr": "process_max_fds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
  669. "format": "time_series",
  670. "hide": true,
  671. "interval": "",
  672. "intervalFactor": 2,
  673. "legendFormat": "{{job}}-{{index}} max",
  674. "refId": "B",
  675. "step": 20
  676. }
  677. ],
  678. "thresholds": [],
  679. "timeFrom": null,
  680. "timeRegions": [],
  681. "timeShift": null,
  682. "title": "Open FDs",
  683. "tooltip": {
  684. "shared": false,
  685. "sort": 0,
  686. "value_type": "individual"
  687. },
  688. "type": "graph",
  689. "xaxis": {
  690. "buckets": null,
  691. "mode": "time",
  692. "name": null,
  693. "show": true,
  694. "values": []
  695. },
  696. "yaxes": [
  697. {
  698. "decimals": null,
  699. "format": "none",
  700. "label": "",
  701. "logBase": 1,
  702. "max": null,
  703. "min": null,
  704. "show": true
  705. },
  706. {
  707. "decimals": null,
  708. "format": "short",
  709. "label": null,
  710. "logBase": 1,
  711. "max": null,
  712. "min": null,
  713. "show": true
  714. }
  715. ],
  716. "yaxis": {
  717. "align": false,
  718. "alignLevel": null
  719. }
  720. },
  721. {
  722. "collapsed": true,
  723. "datasource": "${DS_PROMETHEUS}",
  724. "gridPos": {
  725. "h": 1,
  726. "w": 24,
  727. "x": 0,
  728. "y": 26
  729. },
  730. "id": 54,
  731. "panels": [
  732. {
  733. "aliasColors": {},
  734. "bars": false,
  735. "dashLength": 10,
  736. "dashes": false,
  737. "datasource": "$datasource",
  738. "editable": true,
  739. "error": false,
  740. "fieldConfig": {
  741. "defaults": {
  742. "custom": {},
  743. "links": []
  744. },
  745. "overrides": []
  746. },
  747. "fill": 1,
  748. "fillGradient": 0,
  749. "grid": {},
  750. "gridPos": {
  751. "h": 7,
  752. "w": 12,
  753. "x": 0,
  754. "y": 25
  755. },
  756. "hiddenSeries": false,
  757. "id": 5,
  758. "legend": {
  759. "alignAsTable": false,
  760. "avg": false,
  761. "current": false,
  762. "hideEmpty": false,
  763. "hideZero": false,
  764. "max": false,
  765. "min": false,
  766. "rightSide": false,
  767. "show": true,
  768. "total": false,
  769. "values": false
  770. },
  771. "lines": true,
  772. "linewidth": 3,
  773. "links": [],
  774. "nullPointMode": "null",
  775. "options": {
  776. "alertThreshold": true
  777. },
  778. "paceLength": 10,
  779. "percentage": false,
  780. "pluginVersion": "7.3.7",
  781. "pointradius": 5,
  782. "points": false,
  783. "renderer": "flot",
  784. "seriesOverrides": [
  785. {
  786. "alias": "/user/"
  787. },
  788. {
  789. "alias": "/system/"
  790. }
  791. ],
  792. "spaceLength": 10,
  793. "stack": false,
  794. "steppedLine": false,
  795. "targets": [
  796. {
  797. "expr": "rate(process_cpu_system_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
  798. "format": "time_series",
  799. "intervalFactor": 1,
  800. "legendFormat": "{{job}}-{{index}} system ",
  801. "metric": "",
  802. "refId": "B",
  803. "step": 20
  804. },
  805. {
  806. "expr": "rate(process_cpu_user_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
  807. "format": "time_series",
  808. "hide": false,
  809. "interval": "",
  810. "intervalFactor": 1,
  811. "legendFormat": "{{job}}-{{index}} user",
  812. "refId": "A",
  813. "step": 20
  814. }
  815. ],
  816. "thresholds": [
  817. {
  818. "colorMode": "custom",
  819. "fillColor": "rgba(255, 255, 255, 1)",
  820. "line": true,
  821. "lineColor": "rgba(216, 200, 27, 0.27)",
  822. "op": "gt",
  823. "value": 0.5,
  824. "yaxis": "left"
  825. },
  826. {
  827. "colorMode": "custom",
  828. "fillColor": "rgba(255, 255, 255, 1)",
  829. "line": true,
  830. "lineColor": "rgb(87, 6, 16)",
  831. "op": "gt",
  832. "value": 0.8,
  833. "yaxis": "left"
  834. },
  835. {
  836. "colorMode": "critical",
  837. "fill": true,
  838. "line": true,
  839. "op": "gt",
  840. "value": 1,
  841. "yaxis": "left"
  842. }
  843. ],
  844. "timeFrom": null,
  845. "timeRegions": [],
  846. "timeShift": null,
  847. "title": "CPU",
  848. "tooltip": {
  849. "shared": false,
  850. "sort": 0,
  851. "value_type": "individual"
  852. },
  853. "type": "graph",
  854. "xaxis": {
  855. "buckets": null,
  856. "mode": "time",
  857. "name": null,
  858. "show": true,
  859. "values": []
  860. },
  861. "yaxes": [
  862. {
  863. "decimals": null,
  864. "format": "percentunit",
  865. "label": "",
  866. "logBase": 1,
  867. "max": "1.2",
  868. "min": 0,
  869. "show": true
  870. },
  871. {
  872. "format": "short",
  873. "logBase": 1,
  874. "max": null,
  875. "min": null,
  876. "show": true
  877. }
  878. ],
  879. "yaxis": {
  880. "align": false,
  881. "alignLevel": null
  882. }
  883. },
  884. {
  885. "aliasColors": {},
  886. "bars": false,
  887. "dashLength": 10,
  888. "dashes": false,
  889. "datasource": "$datasource",
  890. "description": "Shows the time in which the given percentage of reactor ticks completed, over the sampled timespan",
  891. "fieldConfig": {
  892. "defaults": {
  893. "custom": {},
  894. "links": []
  895. },
  896. "overrides": []
  897. },
  898. "fill": 1,
  899. "fillGradient": 0,
  900. "gridPos": {
  901. "h": 7,
  902. "w": 12,
  903. "x": 12,
  904. "y": 25
  905. },
  906. "hiddenSeries": false,
  907. "id": 105,
  908. "interval": "",
  909. "legend": {
  910. "avg": false,
  911. "current": false,
  912. "max": false,
  913. "min": false,
  914. "show": true,
  915. "total": false,
  916. "values": false
  917. },
  918. "lines": true,
  919. "linewidth": 1,
  920. "links": [],
  921. "nullPointMode": "null",
  922. "options": {
  923. "alertThreshold": true
  924. },
  925. "paceLength": 10,
  926. "percentage": false,
  927. "pluginVersion": "7.3.7",
  928. "pointradius": 5,
  929. "points": false,
  930. "renderer": "flot",
  931. "seriesOverrides": [],
  932. "spaceLength": 10,
  933. "stack": false,
  934. "steppedLine": false,
  935. "targets": [
  936. {
  937. "expr": "histogram_quantile(0.99, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
  938. "format": "time_series",
  939. "interval": "",
  940. "intervalFactor": 2,
  941. "legendFormat": "{{job}}-{{index}} 99%",
  942. "refId": "A",
  943. "step": 20
  944. },
  945. {
  946. "expr": "histogram_quantile(0.95, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
  947. "format": "time_series",
  948. "intervalFactor": 1,
  949. "legendFormat": "{{job}}-{{index}} 95%",
  950. "refId": "B"
  951. },
  952. {
  953. "expr": "histogram_quantile(0.90, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
  954. "format": "time_series",
  955. "intervalFactor": 1,
  956. "legendFormat": "{{job}}-{{index}} 90%",
  957. "refId": "C"
  958. },
  959. {
  960. "expr": "rate(python_twisted_reactor_tick_time_sum{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]) / rate(python_twisted_reactor_tick_time_count{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size])",
  961. "format": "time_series",
  962. "intervalFactor": 1,
  963. "legendFormat": "{{job}}-{{index}} mean",
  964. "refId": "D"
  965. }
  966. ],
  967. "thresholds": [],
  968. "timeFrom": null,
  969. "timeRegions": [],
  970. "timeShift": null,
  971. "title": "Reactor tick quantiles",
  972. "tooltip": {
  973. "shared": false,
  974. "sort": 0,
  975. "value_type": "individual"
  976. },
  977. "type": "graph",
  978. "xaxis": {
  979. "buckets": null,
  980. "mode": "time",
  981. "name": null,
  982. "show": true,
  983. "values": []
  984. },
  985. "yaxes": [
  986. {
  987. "format": "s",
  988. "label": null,
  989. "logBase": 1,
  990. "max": null,
  991. "min": null,
  992. "show": true
  993. },
  994. {
  995. "format": "short",
  996. "label": null,
  997. "logBase": 1,
  998. "max": null,
  999. "min": null,
  1000. "show": false
  1001. }
  1002. ],
  1003. "yaxis": {
  1004. "align": false,
  1005. "alignLevel": null
  1006. }
  1007. },
  1008. {
  1009. "aliasColors": {},
  1010. "bars": false,
  1011. "dashLength": 10,
  1012. "dashes": false,
  1013. "datasource": "$datasource",
  1014. "editable": true,
  1015. "error": false,
  1016. "fieldConfig": {
  1017. "defaults": {
  1018. "custom": {},
  1019. "links": []
  1020. },
  1021. "overrides": []
  1022. },
  1023. "fill": 0,
  1024. "fillGradient": 0,
  1025. "grid": {},
  1026. "gridPos": {
  1027. "h": 7,
  1028. "w": 12,
  1029. "x": 0,
  1030. "y": 32
  1031. },
  1032. "hiddenSeries": false,
  1033. "id": 34,
  1034. "legend": {
  1035. "avg": false,
  1036. "current": false,
  1037. "max": false,
  1038. "min": false,
  1039. "show": true,
  1040. "total": false,
  1041. "values": false
  1042. },
  1043. "lines": true,
  1044. "linewidth": 1,
  1045. "links": [],
  1046. "nullPointMode": "null",
  1047. "options": {
  1048. "alertThreshold": true
  1049. },
  1050. "paceLength": 10,
  1051. "percentage": false,
  1052. "pluginVersion": "7.3.7",
  1053. "pointradius": 5,
  1054. "points": false,
  1055. "renderer": "flot",
  1056. "seriesOverrides": [],
  1057. "spaceLength": 10,
  1058. "stack": false,
  1059. "steppedLine": false,
  1060. "targets": [
  1061. {
  1062. "expr": "process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
  1063. "format": "time_series",
  1064. "interval": "",
  1065. "intervalFactor": 2,
  1066. "legendFormat": "{{job}} {{index}}",
  1067. "refId": "A",
  1068. "step": 20,
  1069. "target": ""
  1070. },
  1071. {
  1072. "expr": "sum(process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})",
  1073. "interval": "",
  1074. "legendFormat": "total",
  1075. "refId": "B"
  1076. }
  1077. ],
  1078. "thresholds": [],
  1079. "timeFrom": null,
  1080. "timeRegions": [],
  1081. "timeShift": null,
  1082. "title": "Memory",
  1083. "tooltip": {
  1084. "shared": false,
  1085. "sort": 0,
  1086. "value_type": "cumulative"
  1087. },
  1088. "transformations": [],
  1089. "type": "graph",
  1090. "xaxis": {
  1091. "buckets": null,
  1092. "mode": "time",
  1093. "name": null,
  1094. "show": true,
  1095. "values": []
  1096. },
  1097. "yaxes": [
  1098. {
  1099. "format": "bytes",
  1100. "logBase": 1,
  1101. "max": null,
  1102. "min": "0",
  1103. "show": true
  1104. },
  1105. {
  1106. "format": "short",
  1107. "logBase": 1,
  1108. "max": null,
  1109. "min": null,
  1110. "show": true
  1111. }
  1112. ],
  1113. "yaxis": {
  1114. "align": false,
  1115. "alignLevel": null
  1116. }
  1117. },
  1118. {
  1119. "aliasColors": {},
  1120. "bars": false,
  1121. "dashLength": 10,
  1122. "dashes": false,
  1123. "datasource": "$datasource",
  1124. "fieldConfig": {
  1125. "defaults": {
  1126. "custom": {},
  1127. "links": []
  1128. },
  1129. "overrides": []
  1130. },
  1131. "fill": 1,
  1132. "fillGradient": 0,
  1133. "gridPos": {
  1134. "h": 7,
  1135. "w": 12,
  1136. "x": 12,
  1137. "y": 32
  1138. },
  1139. "hiddenSeries": false,
  1140. "id": 49,
  1141. "legend": {
  1142. "avg": false,
  1143. "current": false,
  1144. "max": false,
  1145. "min": false,
  1146. "show": true,
  1147. "total": false,
  1148. "values": false
  1149. },
  1150. "lines": true,
  1151. "linewidth": 1,
  1152. "links": [],
  1153. "nullPointMode": "null",
  1154. "options": {
  1155. "alertThreshold": true
  1156. },
  1157. "paceLength": 10,
  1158. "percentage": false,
  1159. "pluginVersion": "7.3.7",
  1160. "pointradius": 5,
  1161. "points": false,
  1162. "renderer": "flot",
  1163. "seriesOverrides": [
  1164. {
  1165. "alias": "/^up/",
  1166. "legend": false,
  1167. "yaxis": 2
  1168. }
  1169. ],
  1170. "spaceLength": 10,
  1171. "stack": false,
  1172. "steppedLine": false,
  1173. "targets": [
  1174. {
  1175. "expr": "scrape_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
  1176. "format": "time_series",
  1177. "interval": "",
  1178. "intervalFactor": 2,
  1179. "legendFormat": "{{job}}-{{index}}",
  1180. "refId": "A",
  1181. "step": 20
  1182. }
  1183. ],
  1184. "thresholds": [],
  1185. "timeFrom": null,
  1186. "timeRegions": [],
  1187. "timeShift": null,
  1188. "title": "Prometheus scrape time",
  1189. "tooltip": {
  1190. "shared": false,
  1191. "sort": 0,
  1192. "value_type": "individual"
  1193. },
  1194. "type": "graph",
  1195. "xaxis": {
  1196. "buckets": null,
  1197. "mode": "time",
  1198. "name": null,
  1199. "show": true,
  1200. "values": []
  1201. },
  1202. "yaxes": [
  1203. {
  1204. "format": "s",
  1205. "label": null,
  1206. "logBase": 1,
  1207. "max": null,
  1208. "min": "0",
  1209. "show": true
  1210. },
  1211. {
  1212. "decimals": 0,
  1213. "format": "none",
  1214. "label": "",
  1215. "logBase": 1,
  1216. "max": "0",
  1217. "min": "-1",
  1218. "show": false
  1219. }
  1220. ],
  1221. "yaxis": {
  1222. "align": false,
  1223. "alignLevel": null
  1224. }
  1225. },
  1226. {
  1227. "aliasColors": {},
  1228. "bars": false,
  1229. "dashLength": 10,
  1230. "dashes": false,
  1231. "datasource": "$datasource",
  1232. "fieldConfig": {
  1233. "defaults": {
  1234. "custom": {},
  1235. "links": []
  1236. },
  1237. "overrides": []
  1238. },
  1239. "fill": 0,
  1240. "fillGradient": 0,
  1241. "gridPos": {
  1242. "h": 7,
  1243. "w": 12,
  1244. "x": 0,
  1245. "y": 39
  1246. },
  1247. "hiddenSeries": false,
  1248. "id": 53,
  1249. "legend": {
  1250. "avg": false,
  1251. "current": false,
  1252. "max": false,
  1253. "min": false,
  1254. "show": true,
  1255. "total": false,
  1256. "values": false
  1257. },
  1258. "lines": true,
  1259. "linewidth": 1,
  1260. "links": [],
  1261. "nullPointMode": "null",
  1262. "options": {
  1263. "alertThreshold": true
  1264. },
  1265. "paceLength": 10,
  1266. "percentage": false,
  1267. "pluginVersion": "7.3.7",
  1268. "pointradius": 5,
  1269. "points": false,
  1270. "renderer": "flot",
  1271. "seriesOverrides": [],
  1272. "spaceLength": 10,
  1273. "stack": false,
  1274. "steppedLine": false,
  1275. "targets": [
  1276. {
  1277. "expr": "min_over_time(up{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
  1278. "format": "time_series",
  1279. "intervalFactor": 2,
  1280. "legendFormat": "{{job}}-{{index}}",
  1281. "refId": "A"
  1282. }
  1283. ],
  1284. "thresholds": [],
  1285. "timeFrom": null,
  1286. "timeRegions": [],
  1287. "timeShift": null,
  1288. "title": "Up",
  1289. "tooltip": {
  1290. "shared": false,
  1291. "sort": 0,
  1292. "value_type": "individual"
  1293. },
  1294. "type": "graph",
  1295. "xaxis": {
  1296. "buckets": null,
  1297. "mode": "time",
  1298. "name": null,
  1299. "show": true,
  1300. "values": []
  1301. },
  1302. "yaxes": [
  1303. {
  1304. "format": "short",
  1305. "label": null,
  1306. "logBase": 1,
  1307. "max": null,
  1308. "min": null,
  1309. "show": true
  1310. },
  1311. {
  1312. "format": "short",
  1313. "label": null,
  1314. "logBase": 1,
  1315. "max": null,
  1316. "min": null,
  1317. "show": true
  1318. }
  1319. ],
  1320. "yaxis": {
  1321. "align": false,
  1322. "alignLevel": null
  1323. }
  1324. },
  1325. {
  1326. "aliasColors": {},
  1327. "bars": false,
  1328. "dashLength": 10,
  1329. "dashes": false,
  1330. "datasource": "$datasource",
  1331. "fieldConfig": {
  1332. "defaults": {
  1333. "custom": {},
  1334. "links": []
  1335. },
  1336. "overrides": []
  1337. },
  1338. "fill": 1,
  1339. "fillGradient": 0,
  1340. "gridPos": {
  1341. "h": 7,
  1342. "w": 12,
  1343. "x": 12,
  1344. "y": 39
  1345. },
  1346. "hiddenSeries": false,
  1347. "id": 120,
  1348. "legend": {
  1349. "avg": false,
  1350. "current": false,
  1351. "max": false,
  1352. "min": false,
  1353. "show": true,
  1354. "total": false,
  1355. "values": false
  1356. },
  1357. "lines": true,
  1358. "linewidth": 1,
  1359. "links": [],
  1360. "nullPointMode": "null as zero",
  1361. "options": {
  1362. "alertThreshold": true
  1363. },
  1364. "percentage": false,
  1365. "pluginVersion": "7.3.7",
  1366. "pointradius": 2,
  1367. "points": false,
  1368. "renderer": "flot",
  1369. "seriesOverrides": [],
  1370. "spaceLength": 10,
  1371. "stack": true,
  1372. "steppedLine": false,
  1373. "targets": [
  1374. {
  1375. "expr": "rate(synapse_http_server_response_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_response_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
  1376. "format": "time_series",
  1377. "hide": false,
  1378. "instant": false,
  1379. "intervalFactor": 1,
  1380. "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
  1381. "refId": "A"
  1382. },
  1383. {
  1384. "expr": "rate(synapse_background_process_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
  1385. "format": "time_series",
  1386. "hide": false,
  1387. "instant": false,
  1388. "interval": "",
  1389. "intervalFactor": 1,
  1390. "legendFormat": "{{job}}-{{index}} {{name}}",
  1391. "refId": "B"
  1392. }
  1393. ],
  1394. "thresholds": [
  1395. {
  1396. "colorMode": "critical",
  1397. "fill": true,
  1398. "line": true,
  1399. "op": "gt",
  1400. "value": 1,
  1401. "yaxis": "left"
  1402. }
  1403. ],
  1404. "timeFrom": null,
  1405. "timeRegions": [],
  1406. "timeShift": null,
  1407. "title": "Stacked CPU usage",
  1408. "tooltip": {
  1409. "shared": false,
  1410. "sort": 0,
  1411. "value_type": "individual"
  1412. },
  1413. "type": "graph",
  1414. "xaxis": {
  1415. "buckets": null,
  1416. "mode": "time",
  1417. "name": null,
  1418. "show": true,
  1419. "values": []
  1420. },
  1421. "yaxes": [
  1422. {
  1423. "format": "percentunit",
  1424. "label": null,
  1425. "logBase": 1,
  1426. "max": null,
  1427. "min": null,
  1428. "show": true
  1429. },
  1430. {
  1431. "format": "short",
  1432. "label": null,
  1433. "logBase": 1,
  1434. "max": null,
  1435. "min": null,
  1436. "show": true
  1437. }
  1438. ],
  1439. "yaxis": {
  1440. "align": false,
  1441. "alignLevel": null
  1442. }
  1443. },
  1444. {
  1445. "aliasColors": {},
  1446. "bars": false,
  1447. "dashLength": 10,
  1448. "dashes": false,
  1449. "datasource": "$datasource",
  1450. "fieldConfig": {
  1451. "defaults": {
  1452. "custom": {},
  1453. "links": []
  1454. },
  1455. "overrides": []
  1456. },
  1457. "fill": 1,
  1458. "fillGradient": 0,
  1459. "gridPos": {
  1460. "h": 7,
  1461. "w": 12,
  1462. "x": 0,
  1463. "y": 46
  1464. },
  1465. "hiddenSeries": false,
  1466. "id": 136,
  1467. "interval": "",
  1468. "legend": {
  1469. "avg": false,
  1470. "current": false,
  1471. "max": false,
  1472. "min": false,
  1473. "show": true,
  1474. "total": false,
  1475. "values": false
  1476. },
  1477. "lines": true,
  1478. "linewidth": 1,
  1479. "nullPointMode": "null",
  1480. "options": {
  1481. "alertThreshold": true
  1482. },
  1483. "percentage": false,
  1484. "pluginVersion": "7.3.7",
  1485. "pointradius": 2,
  1486. "points": false,
  1487. "renderer": "flot",
  1488. "seriesOverrides": [],
  1489. "spaceLength": 10,
  1490. "stack": false,
  1491. "steppedLine": false,
  1492. "targets": [
  1493. {
  1494. "expr": "rate(synapse_http_client_requests{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
  1495. "legendFormat": "{{job}}-{{index}} {{method}}",
  1496. "refId": "A"
  1497. },
  1498. {
  1499. "expr": "rate(synapse_http_matrixfederationclient_requests{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
  1500. "legendFormat": "{{job}}-{{index}} {{method}} (federation)",
  1501. "refId": "B"
  1502. }
  1503. ],
  1504. "thresholds": [],
  1505. "timeFrom": null,
  1506. "timeRegions": [],
  1507. "timeShift": null,
  1508. "title": "Outgoing HTTP request rate",
  1509. "tooltip": {
  1510. "shared": false,
  1511. "sort": 0,
  1512. "value_type": "individual"
  1513. },
  1514. "type": "graph",
  1515. "xaxis": {
  1516. "buckets": null,
  1517. "mode": "time",
  1518. "name": null,
  1519. "show": true,
  1520. "values": []
  1521. },
  1522. "yaxes": [
  1523. {
  1524. "format": "reqps",
  1525. "label": null,
  1526. "logBase": 1,
  1527. "max": null,
  1528. "min": null,
  1529. "show": true
  1530. },
  1531. {
  1532. "format": "short",
  1533. "label": null,
  1534. "logBase": 1,
  1535. "max": null,
  1536. "min": null,
  1537. "show": true
  1538. }
  1539. ],
  1540. "yaxis": {
  1541. "align": false,
  1542. "alignLevel": null
  1543. }
  1544. }
  1545. ],
  1546. "repeat": null,
  1547. "title": "Process info",
  1548. "type": "row"
  1549. },
  1550. {
  1551. "collapsed": true,
  1552. "datasource": "${DS_PROMETHEUS}",
  1553. "gridPos": {
  1554. "h": 1,
  1555. "w": 24,
  1556. "x": 0,
  1557. "y": 27
  1558. },
  1559. "id": 56,
  1560. "panels": [
  1561. {
  1562. "cards": {
  1563. "cardPadding": -1,
  1564. "cardRound": 0
  1565. },
  1566. "color": {
  1567. "cardColor": "#b4ff00",
  1568. "colorScale": "sqrt",
  1569. "colorScheme": "interpolateInferno",
  1570. "exponent": 0.5,
  1571. "mode": "spectrum"
  1572. },
  1573. "dataFormat": "tsbuckets",
  1574. "datasource": "$datasource",
  1575. "fieldConfig": {
  1576. "defaults": {
  1577. "custom": {}
  1578. },
  1579. "overrides": []
  1580. },
  1581. "gridPos": {
  1582. "h": 9,
  1583. "w": 12,
  1584. "x": 0,
  1585. "y": 21
  1586. },
  1587. "heatmap": {},
  1588. "hideZeroBuckets": false,
  1589. "highlightCards": true,
  1590. "id": 85,
  1591. "legend": {
  1592. "show": false
  1593. },
  1594. "links": [],
  1595. "reverseYBuckets": false,
  1596. "targets": [
  1597. {
  1598. "expr": "sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\"}[$bucket_size])) by (le)",
  1599. "format": "heatmap",
  1600. "intervalFactor": 1,
  1601. "legendFormat": "{{le}}",
  1602. "refId": "A"
  1603. }
  1604. ],
  1605. "title": "Event Send Time (Including errors, across all workers)",
  1606. "tooltip": {
  1607. "show": true,
  1608. "showHistogram": true
  1609. },
  1610. "type": "heatmap",
  1611. "xAxis": {
  1612. "show": true
  1613. },
  1614. "xBucketNumber": null,
  1615. "xBucketSize": null,
  1616. "yAxis": {
  1617. "decimals": null,
  1618. "format": "s",
  1619. "logBase": 2,
  1620. "max": null,
  1621. "min": null,
  1622. "show": true,
  1623. "splitFactor": null
  1624. },
  1625. "yBucketBound": "auto",
  1626. "yBucketNumber": null,
  1627. "yBucketSize": null
  1628. },
  1629. {
  1630. "aliasColors": {},
  1631. "bars": false,
  1632. "dashLength": 10,
  1633. "dashes": false,
  1634. "datasource": "$datasource",
  1635. "description": "",
  1636. "editable": true,
  1637. "error": false,
  1638. "fieldConfig": {
  1639. "defaults": {
  1640. "custom": {},
  1641. "links": []
  1642. },
  1643. "overrides": []
  1644. },
  1645. "fill": 1,
  1646. "fillGradient": 0,
  1647. "grid": {},
  1648. "gridPos": {
  1649. "h": 9,
  1650. "w": 12,
  1651. "x": 12,
  1652. "y": 21
  1653. },
  1654. "hiddenSeries": false,
  1655. "id": 33,
  1656. "legend": {
  1657. "avg": false,
  1658. "current": false,
  1659. "max": false,
  1660. "min": false,
  1661. "show": false,
  1662. "total": false,
  1663. "values": false
  1664. },
  1665. "lines": true,
  1666. "linewidth": 2,
  1667. "links": [],
  1668. "nullPointMode": "null",
  1669. "options": {
  1670. "alertThreshold": true
  1671. },
  1672. "paceLength": 10,
  1673. "percentage": false,
  1674. "pluginVersion": "7.3.7",
  1675. "pointradius": 5,
  1676. "points": false,
  1677. "renderer": "flot",
  1678. "seriesOverrides": [],
  1679. "spaceLength": 10,
  1680. "stack": false,
  1681. "steppedLine": false,
  1682. "targets": [
  1683. {
  1684. "expr": "sum(rate(synapse_storage_events_persisted_events{instance=\"$instance\"}[$bucket_size])) without (job,index)",
  1685. "format": "time_series",
  1686. "interval": "",
  1687. "intervalFactor": 2,
  1688. "legendFormat": "",
  1689. "refId": "A",
  1690. "step": 20,
  1691. "target": ""
  1692. }
  1693. ],
  1694. "thresholds": [],
  1695. "timeFrom": null,
  1696. "timeRegions": [],
  1697. "timeShift": null,
  1698. "title": "Events Persisted (all workers)",
  1699. "tooltip": {
  1700. "shared": true,
  1701. "sort": 0,
  1702. "value_type": "cumulative"
  1703. },
  1704. "type": "graph",
  1705. "xaxis": {
  1706. "buckets": null,
  1707. "mode": "time",
  1708. "name": null,
  1709. "show": true,
  1710. "values": []
  1711. },
  1712. "yaxes": [
  1713. {
  1714. "format": "hertz",
  1715. "logBase": 1,
  1716. "max": null,
  1717. "min": null,
  1718. "show": true
  1719. },
  1720. {
  1721. "format": "short",
  1722. "logBase": 1,
  1723. "max": null,
  1724. "min": null,
  1725. "show": true
  1726. }
  1727. ],
  1728. "yaxis": {
  1729. "align": false,
  1730. "alignLevel": null
  1731. }
  1732. },
  1733. {
  1734. "aliasColors": {},
  1735. "bars": false,
  1736. "dashLength": 10,
  1737. "dashes": false,
  1738. "datasource": "$datasource",
  1739. "decimals": 1,
  1740. "fieldConfig": {
  1741. "defaults": {
  1742. "custom": {}
  1743. },
  1744. "overrides": []
  1745. },
  1746. "fill": 1,
  1747. "fillGradient": 0,
  1748. "gridPos": {
  1749. "h": 7,
  1750. "w": 12,
  1751. "x": 0,
  1752. "y": 30
  1753. },
  1754. "hiddenSeries": false,
  1755. "id": 40,
  1756. "legend": {
  1757. "avg": false,
  1758. "current": false,
  1759. "max": false,
  1760. "min": false,
  1761. "show": true,
  1762. "total": false,
  1763. "values": false
  1764. },
  1765. "lines": true,
  1766. "linewidth": 1,
  1767. "links": [],
  1768. "nullPointMode": "null",
  1769. "options": {
  1770. "alertThreshold": true
  1771. },
  1772. "percentage": false,
  1773. "pluginVersion": "7.3.7",
  1774. "pointradius": 5,
  1775. "points": false,
  1776. "renderer": "flot",
  1777. "seriesOverrides": [],
  1778. "spaceLength": 10,
  1779. "stack": false,
  1780. "steppedLine": false,
  1781. "targets": [
  1782. {
  1783. "expr": "rate(synapse_storage_events_persisted_by_source_type{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
  1784. "format": "time_series",
  1785. "intervalFactor": 2,
  1786. "legendFormat": "{{type}}",
  1787. "refId": "D"
  1788. }
  1789. ],
  1790. "thresholds": [],
  1791. "timeFrom": null,
  1792. "timeRegions": [],
  1793. "timeShift": null,
  1794. "title": "Events/s Local vs Remote",
  1795. "tooltip": {
  1796. "shared": true,
  1797. "sort": 2,
  1798. "value_type": "individual"
  1799. },
  1800. "type": "graph",
  1801. "xaxis": {
  1802. "buckets": null,
  1803. "mode": "time",
  1804. "name": null,
  1805. "show": true,
  1806. "values": []
  1807. },
  1808. "yaxes": [
  1809. {
  1810. "format": "hertz",
  1811. "label": "",
  1812. "logBase": 1,
  1813. "max": null,
  1814. "min": "0",
  1815. "show": true
  1816. },
  1817. {
  1818. "format": "short",
  1819. "label": null,
  1820. "logBase": 1,
  1821. "max": null,
  1822. "min": null,
  1823. "show": true
  1824. }
  1825. ],
  1826. "yaxis": {
  1827. "align": false,
  1828. "alignLevel": null
  1829. }
  1830. },
  1831. {
  1832. "aliasColors": {},
  1833. "bars": false,
  1834. "dashLength": 10,
  1835. "dashes": false,
  1836. "datasource": "$datasource",
  1837. "decimals": 1,
  1838. "fieldConfig": {
  1839. "defaults": {
  1840. "custom": {}
  1841. },
  1842. "overrides": []
  1843. },
  1844. "fill": 1,
  1845. "fillGradient": 0,
  1846. "gridPos": {
  1847. "h": 7,
  1848. "w": 12,
  1849. "x": 12,
  1850. "y": 30
  1851. },
  1852. "hiddenSeries": false,
  1853. "id": 46,
  1854. "legend": {
  1855. "avg": false,
  1856. "current": false,
  1857. "max": false,
  1858. "min": false,
  1859. "show": true,
  1860. "total": false,
  1861. "values": false
  1862. },
  1863. "lines": true,
  1864. "linewidth": 1,
  1865. "links": [],
  1866. "nullPointMode": "null",
  1867. "options": {
  1868. "alertThreshold": true
  1869. },
  1870. "percentage": false,
  1871. "pluginVersion": "7.3.7",
  1872. "pointradius": 5,
  1873. "points": false,
  1874. "renderer": "flot",
  1875. "seriesOverrides": [],
  1876. "spaceLength": 10,
  1877. "stack": false,
  1878. "steppedLine": false,
  1879. "targets": [
  1880. {
  1881. "expr": "rate(synapse_storage_events_persisted_by_event_type{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
  1882. "format": "time_series",
  1883. "instant": false,
  1884. "intervalFactor": 2,
  1885. "legendFormat": "{{type}}",
  1886. "refId": "A",
  1887. "step": 20
  1888. }
  1889. ],
  1890. "thresholds": [],
  1891. "timeFrom": null,
  1892. "timeRegions": [],
  1893. "timeShift": null,
  1894. "title": "Events/s by Type",
  1895. "tooltip": {
  1896. "shared": false,
  1897. "sort": 2,
  1898. "value_type": "individual"
  1899. },
  1900. "type": "graph",
  1901. "xaxis": {
  1902. "buckets": null,
  1903. "mode": "time",
  1904. "name": null,
  1905. "show": true,
  1906. "values": []
  1907. },
  1908. "yaxes": [
  1909. {
  1910. "format": "hertz",
  1911. "label": null,
  1912. "logBase": 1,
  1913. "max": null,
  1914. "min": "0",
  1915. "show": true
  1916. },
  1917. {
  1918. "format": "short",
  1919. "label": null,
  1920. "logBase": 1,
  1921. "max": null,
  1922. "min": null,
  1923. "show": true
  1924. }
  1925. ],
  1926. "yaxis": {
  1927. "align": false,
  1928. "alignLevel": null
  1929. }
  1930. },
  1931. {
  1932. "aliasColors": {
  1933. "irc-freenode (local)": "#EAB839"
  1934. },
  1935. "bars": false,
  1936. "dashLength": 10,
  1937. "dashes": false,
  1938. "datasource": "$datasource",
  1939. "decimals": 1,
  1940. "fieldConfig": {
  1941. "defaults": {
  1942. "custom": {}
  1943. },
  1944. "overrides": []
  1945. },
  1946. "fill": 1,
  1947. "fillGradient": 0,
  1948. "gridPos": {
  1949. "h": 7,
  1950. "w": 12,
  1951. "x": 0,
  1952. "y": 37
  1953. },
  1954. "hiddenSeries": false,
  1955. "id": 44,
  1956. "legend": {
  1957. "alignAsTable": true,
  1958. "avg": false,
  1959. "current": false,
  1960. "hideEmpty": true,
  1961. "hideZero": true,
  1962. "max": false,
  1963. "min": false,
  1964. "show": true,
  1965. "total": false,
  1966. "values": false
  1967. },
  1968. "lines": true,
  1969. "linewidth": 1,
  1970. "links": [],
  1971. "nullPointMode": "null",
  1972. "options": {
  1973. "alertThreshold": true
  1974. },
  1975. "percentage": false,
  1976. "pluginVersion": "7.3.7",
  1977. "pointradius": 5,
  1978. "points": false,
  1979. "renderer": "flot",
  1980. "seriesOverrides": [],
  1981. "spaceLength": 10,
  1982. "stack": false,
  1983. "steppedLine": false,
  1984. "targets": [
  1985. {
  1986. "expr": "rate(synapse_storage_events_persisted_by_origin{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
  1987. "format": "time_series",
  1988. "intervalFactor": 2,
  1989. "legendFormat": "{{origin_entity}} ({{origin_type}})",
  1990. "refId": "A",
  1991. "step": 20
  1992. }
  1993. ],
  1994. "thresholds": [],
  1995. "timeFrom": null,
  1996. "timeRegions": [],
  1997. "timeShift": null,
  1998. "title": "Events/s by Origin",
  1999. "tooltip": {
  2000. "shared": false,
  2001. "sort": 2,
  2002. "value_type": "individual"
  2003. },
  2004. "type": "graph",
  2005. "xaxis": {
  2006. "buckets": null,
  2007. "mode": "time",
  2008. "name": null,
  2009. "show": true,
  2010. "values": []
  2011. },
  2012. "yaxes": [
  2013. {
  2014. "format": "hertz",
  2015. "label": null,
  2016. "logBase": 1,
  2017. "max": null,
  2018. "min": "0",
  2019. "show": true
  2020. },
  2021. {
  2022. "format": "short",
  2023. "label": null,
  2024. "logBase": 1,
  2025. "max": null,
  2026. "min": null,
  2027. "show": true
  2028. }
  2029. ],
  2030. "yaxis": {
  2031. "align": false,
  2032. "alignLevel": null
  2033. }
  2034. },
  2035. {
  2036. "aliasColors": {},
  2037. "bars": false,
  2038. "dashLength": 10,
  2039. "dashes": false,
  2040. "datasource": "$datasource",
  2041. "decimals": 1,
  2042. "fieldConfig": {
  2043. "defaults": {
  2044. "custom": {}
  2045. },
  2046. "overrides": []
  2047. },
  2048. "fill": 1,
  2049. "fillGradient": 0,
  2050. "gridPos": {
  2051. "h": 7,
  2052. "w": 12,
  2053. "x": 12,
  2054. "y": 37
  2055. },
  2056. "hiddenSeries": false,
  2057. "id": 45,
  2058. "legend": {
  2059. "alignAsTable": true,
  2060. "avg": false,
  2061. "current": false,
  2062. "hideEmpty": true,
  2063. "hideZero": true,
  2064. "max": false,
  2065. "min": false,
  2066. "show": true,
  2067. "total": false,
  2068. "values": false
  2069. },
  2070. "lines": true,
  2071. "linewidth": 1,
  2072. "links": [],
  2073. "nullPointMode": "null",
  2074. "options": {
  2075. "alertThreshold": true
  2076. },
  2077. "percentage": false,
  2078. "pluginVersion": "7.3.7",
  2079. "pointradius": 5,
  2080. "points": false,
  2081. "renderer": "flot",
  2082. "seriesOverrides": [],
  2083. "spaceLength": 10,
  2084. "stack": false,
  2085. "steppedLine": false,
  2086. "targets": [
  2087. {
  2088. "expr": "sum(rate(synapse_storage_events_persisted_events_sep{job=~\"$job\",index=~\"$index\", type=\"m.room.member\",instance=\"$instance\", origin_type=\"local\"}[$bucket_size])) by (origin_type, origin_entity)",
  2089. "format": "time_series",
  2090. "intervalFactor": 2,
  2091. "legendFormat": "{{origin_entity}} ({{origin_type}})",
  2092. "refId": "A",
  2093. "step": 20
  2094. }
  2095. ],
  2096. "thresholds": [],
  2097. "timeFrom": null,
  2098. "timeRegions": [],
  2099. "timeShift": null,
  2100. "title": "Memberships/s by Origin",
  2101. "tooltip": {
  2102. "shared": true,
  2103. "sort": 2,
  2104. "value_type": "individual"
  2105. },
  2106. "type": "graph",
  2107. "xaxis": {
  2108. "buckets": null,
  2109. "mode": "time",
  2110. "name": null,
  2111. "show": true,
  2112. "values": []
  2113. },
  2114. "yaxes": [
  2115. {
  2116. "format": "hertz",
  2117. "label": null,
  2118. "logBase": 1,
  2119. "max": null,
  2120. "min": "0",
  2121. "show": true
  2122. },
  2123. {
  2124. "format": "short",
  2125. "label": null,
  2126. "logBase": 1,
  2127. "max": null,
  2128. "min": null,
  2129. "show": true
  2130. }
  2131. ],
  2132. "yaxis": {
  2133. "align": false,
  2134. "alignLevel": null
  2135. }
  2136. },
  2137. {
  2138. "aliasColors": {},
  2139. "bars": false,
  2140. "dashLength": 10,
  2141. "dashes": false,
  2142. "datasource": "$datasource",
  2143. "fieldConfig": {
  2144. "defaults": {
  2145. "custom": {},
  2146. "links": []
  2147. },
  2148. "overrides": []
  2149. },
  2150. "fill": 0,
  2151. "fillGradient": 0,
  2152. "gridPos": {
  2153. "h": 9,
  2154. "w": 12,
  2155. "x": 0,
  2156. "y": 44
  2157. },
  2158. "hiddenSeries": false,
  2159. "id": 118,
  2160. "legend": {
  2161. "avg": false,
  2162. "current": false,
  2163. "max": false,
  2164. "min": false,
  2165. "show": true,
  2166. "total": false,
  2167. "values": false
  2168. },
  2169. "lines": true,
  2170. "linewidth": 1,
  2171. "links": [],
  2172. "nullPointMode": "null",
  2173. "options": {
  2174. "alertThreshold": true
  2175. },
  2176. "paceLength": 10,
  2177. "percentage": false,
  2178. "pluginVersion": "7.3.7",
  2179. "pointradius": 5,
  2180. "points": false,
  2181. "renderer": "flot",
  2182. "repeatDirection": "h",
  2183. "seriesOverrides": [
  2184. {
  2185. "alias": "mean",
  2186. "linewidth": 2
  2187. }
  2188. ],
  2189. "spaceLength": 10,
  2190. "stack": false,
  2191. "steppedLine": false,
  2192. "targets": [
  2193. {
  2194. "expr": "histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
  2195. "format": "time_series",
  2196. "interval": "",
  2197. "intervalFactor": 1,
  2198. "legendFormat": "{{job}}-{{index}} 99%",
  2199. "refId": "A"
  2200. },
  2201. {
  2202. "expr": "histogram_quantile(0.95, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
  2203. "format": "time_series",
  2204. "interval": "",
  2205. "intervalFactor": 1,
  2206. "legendFormat": "{{job}}-{{index}} 95%",
  2207. "refId": "B"
  2208. },
  2209. {
  2210. "expr": "histogram_quantile(0.90, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
  2211. "format": "time_series",
  2212. "intervalFactor": 1,
  2213. "legendFormat": "{{job}}-{{index}} 90%",
  2214. "refId": "C"
  2215. },
  2216. {
  2217. "expr": "histogram_quantile(0.50, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
  2218. "format": "time_series",
  2219. "intervalFactor": 1,
  2220. "legendFormat": "{{job}}-{{index}} 50%",
  2221. "refId": "D"
  2222. },
  2223. {
  2224. "expr": "sum(rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method) / sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method)",
  2225. "format": "time_series",
  2226. "intervalFactor": 1,
  2227. "legendFormat": "{{job}}-{{index}} mean",
  2228. "refId": "E"
  2229. }
  2230. ],
  2231. "thresholds": [],
  2232. "timeFrom": null,
  2233. "timeRegions": [],
  2234. "timeShift": null,
  2235. "title": "Event send time quantiles by worker",
  2236. "tooltip": {
  2237. "shared": true,
  2238. "sort": 0,
  2239. "value_type": "individual"
  2240. },
  2241. "type": "graph",
  2242. "xaxis": {
  2243. "buckets": null,
  2244. "mode": "time",
  2245. "name": null,
  2246. "show": true,
  2247. "values": []
  2248. },
  2249. "yaxes": [
  2250. {
  2251. "format": "s",
  2252. "label": null,
  2253. "logBase": 1,
  2254. "max": null,
  2255. "min": null,
  2256. "show": true
  2257. },
  2258. {
  2259. "format": "short",
  2260. "label": null,
  2261. "logBase": 1,
  2262. "max": null,
  2263. "min": null,
  2264. "show": true
  2265. }
  2266. ],
  2267. "yaxis": {
  2268. "align": false,
  2269. "alignLevel": null
  2270. }
  2271. }
  2272. ],
  2273. "repeat": null,
  2274. "title": "Event persistence",
  2275. "type": "row"
  2276. },
  2277. {
  2278. "collapsed": true,
  2279. "datasource": "${DS_PROMETHEUS}",
  2280. "gridPos": {
  2281. "h": 1,
  2282. "w": 24,
  2283. "x": 0,
  2284. "y": 28
  2285. },
  2286. "id": 57,
  2287. "panels": [
  2288. {
  2289. "aliasColors": {},
  2290. "bars": false,
  2291. "dashLength": 10,
  2292. "dashes": false,
  2293. "datasource": "$datasource",
  2294. "decimals": null,
  2295. "editable": true,
  2296. "error": false,
  2297. "fieldConfig": {
  2298. "defaults": {
  2299. "custom": {},
  2300. "links": []
  2301. },
  2302. "overrides": []
  2303. },
  2304. "fill": 2,
  2305. "fillGradient": 0,
  2306. "grid": {},
  2307. "gridPos": {
  2308. "h": 8,
  2309. "w": 12,
  2310. "x": 0,
  2311. "y": 31
  2312. },
  2313. "hiddenSeries": false,
  2314. "id": 4,
  2315. "legend": {
  2316. "alignAsTable": true,
  2317. "avg": false,
  2318. "current": false,
  2319. "hideEmpty": false,
  2320. "hideZero": true,
  2321. "max": false,
  2322. "min": false,
  2323. "rightSide": false,
  2324. "show": true,
  2325. "total": false,
  2326. "values": false
  2327. },
  2328. "lines": true,
  2329. "linewidth": 1,
  2330. "links": [],
  2331. "nullPointMode": "null",
  2332. "options": {
  2333. "alertThreshold": true
  2334. },
  2335. "percentage": false,
  2336. "pluginVersion": "7.3.7",
  2337. "pointradius": 5,
  2338. "points": false,
  2339. "renderer": "flot",
  2340. "seriesOverrides": [],
  2341. "spaceLength": 10,
  2342. "stack": false,
  2343. "steppedLine": false,
  2344. "targets": [
  2345. {
  2346. "expr": "rate(synapse_http_server_requests_received{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
  2347. "format": "time_series",
  2348. "interval": "",
  2349. "intervalFactor": 2,
  2350. "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
  2351. "refId": "A",
  2352. "step": 20
  2353. }
  2354. ],
  2355. "thresholds": [
  2356. {
  2357. "colorMode": "custom",
  2358. "fill": true,
  2359. "fillColor": "rgba(216, 200, 27, 0.27)",
  2360. "op": "gt",
  2361. "value": 100,
  2362. "yaxis": "left"
  2363. },
  2364. {
  2365. "colorMode": "custom",
  2366. "fill": true,
  2367. "fillColor": "rgba(234, 112, 112, 0.22)",
  2368. "op": "gt",
  2369. "value": 250,
  2370. "yaxis": "left"
  2371. }
  2372. ],
  2373. "timeFrom": null,
  2374. "timeRegions": [],
  2375. "timeShift": null,
  2376. "title": "Request Count by arrival time",
  2377. "tooltip": {
  2378. "shared": false,
  2379. "sort": 2,
  2380. "value_type": "individual"
  2381. },
  2382. "type": "graph",
  2383. "xaxis": {
  2384. "buckets": null,
  2385. "mode": "time",
  2386. "name": null,
  2387. "show": true,
  2388. "values": []
  2389. },
  2390. "yaxes": [
  2391. {
  2392. "format": "hertz",
  2393. "logBase": 1,
  2394. "max": null,
  2395. "min": null,
  2396. "show": true
  2397. },
  2398. {
  2399. "format": "short",
  2400. "logBase": 1,
  2401. "max": null,
  2402. "min": null,
  2403. "show": true
  2404. }
  2405. ],
  2406. "yaxis": {
  2407. "align": false,
  2408. "alignLevel": null
  2409. }
  2410. },
  2411. {
  2412. "aliasColors": {},
  2413. "bars": false,
  2414. "dashLength": 10,
  2415. "dashes": false,
  2416. "datasource": "$datasource",
  2417. "editable": true,
  2418. "error": false,
  2419. "fieldConfig": {
  2420. "defaults": {
  2421. "custom": {},
  2422. "links": []
  2423. },
  2424. "overrides": []
  2425. },
  2426. "fill": 1,
  2427. "fillGradient": 0,
  2428. "grid": {},
  2429. "gridPos": {
  2430. "h": 8,
  2431. "w": 12,
  2432. "x": 12,
  2433. "y": 31
  2434. },
  2435. "hiddenSeries": false,
  2436. "id": 32,
  2437. "legend": {
  2438. "avg": false,
  2439. "current": false,
  2440. "max": false,
  2441. "min": false,
  2442. "show": true,
  2443. "total": false,
  2444. "values": false
  2445. },
  2446. "lines": true,
  2447. "linewidth": 2,
  2448. "links": [],
  2449. "nullPointMode": "null",
  2450. "options": {
  2451. "alertThreshold": true
  2452. },
  2453. "percentage": false,
  2454. "pluginVersion": "7.3.7",
  2455. "pointradius": 5,
  2456. "points": false,
  2457. "renderer": "flot",
  2458. "seriesOverrides": [],
  2459. "spaceLength": 10,
  2460. "stack": false,
  2461. "steppedLine": false,
  2462. "targets": [
  2463. {
  2464. "expr": "rate(synapse_http_server_requests_received{instance=\"$instance\",job=~\"$job\",index=~\"$index\",method!=\"OPTIONS\"}[$bucket_size]) and topk(10,synapse_http_server_requests_received{instance=\"$instance\",job=~\"$job\",method!=\"OPTIONS\"})",
  2465. "format": "time_series",
  2466. "intervalFactor": 2,
  2467. "legendFormat": "{{method}} {{servlet}} {{job}}-{{index}}",
  2468. "refId": "A",
  2469. "step": 20,
  2470. "target": ""
  2471. }
  2472. ],
  2473. "thresholds": [],
  2474. "timeFrom": null,
  2475. "timeRegions": [],
  2476. "timeShift": null,
  2477. "title": "Top 10 Request Counts",
  2478. "tooltip": {
  2479. "shared": false,
  2480. "sort": 2,
  2481. "value_type": "cumulative"
  2482. },
  2483. "type": "graph",
  2484. "xaxis": {
  2485. "buckets": null,
  2486. "mode": "time",
  2487. "name": null,
  2488. "show": true,
  2489. "values": []
  2490. },
  2491. "yaxes": [
  2492. {
  2493. "format": "hertz",
  2494. "logBase": 1,
  2495. "max": null,
  2496. "min": null,
  2497. "show": true
  2498. },
  2499. {
  2500. "format": "short",
  2501. "logBase": 1,
  2502. "max": null,
  2503. "min": null,
  2504. "show": true
  2505. }
  2506. ],
  2507. "yaxis": {
  2508. "align": false,
  2509. "alignLevel": null
  2510. }
  2511. },
  2512. {
  2513. "aliasColors": {},
  2514. "bars": false,
  2515. "dashLength": 10,
  2516. "dashes": false,
  2517. "datasource": "$datasource",
  2518. "decimals": null,
  2519. "editable": true,
  2520. "error": false,
  2521. "fieldConfig": {
  2522. "defaults": {
  2523. "custom": {},
  2524. "links": []
  2525. },
  2526. "overrides": []
  2527. },
  2528. "fill": 2,
  2529. "fillGradient": 0,
  2530. "grid": {},
  2531. "gridPos": {
  2532. "h": 8,
  2533. "w": 12,
  2534. "x": 0,
  2535. "y": 39
  2536. },
  2537. "hiddenSeries": false,
  2538. "id": 139,
  2539. "legend": {
  2540. "alignAsTable": true,
  2541. "avg": false,
  2542. "current": false,
  2543. "hideEmpty": false,
  2544. "hideZero": true,
  2545. "max": false,
  2546. "min": false,
  2547. "rightSide": false,
  2548. "show": true,
  2549. "total": false,
  2550. "values": false
  2551. },
  2552. "lines": true,
  2553. "linewidth": 1,
  2554. "links": [],
  2555. "nullPointMode": "null",
  2556. "options": {
  2557. "alertThreshold": true
  2558. },
  2559. "percentage": false,
  2560. "pluginVersion": "7.3.7",
  2561. "pointradius": 5,
  2562. "points": false,
  2563. "renderer": "flot",
  2564. "seriesOverrides": [],
  2565. "spaceLength": 10,
  2566. "stack": false,
  2567. "steppedLine": false,
  2568. "targets": [
  2569. {
  2570. "expr": "rate(synapse_http_server_in_flight_requests_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_in_flight_requests_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
  2571. "format": "time_series",
  2572. "interval": "",
  2573. "intervalFactor": 1,
  2574. "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
  2575. "refId": "A",
  2576. "step": 20
  2577. }
  2578. ],
  2579. "thresholds": [
  2580. {
  2581. "colorMode": "custom",
  2582. "fill": true,
  2583. "fillColor": "rgba(216, 200, 27, 0.27)",
  2584. "op": "gt",
  2585. "value": 100,
  2586. "yaxis": "left"
  2587. },
  2588. {
  2589. "colorMode": "custom",
  2590. "fill": true,
  2591. "fillColor": "rgba(234, 112, 112, 0.22)",
  2592. "op": "gt",
  2593. "value": 250,
  2594. "yaxis": "left"
  2595. }
  2596. ],
  2597. "timeFrom": null,
  2598. "timeRegions": [],
  2599. "timeShift": null,
  2600. "title": "Total CPU Usage by Endpoint",
  2601. "tooltip": {
  2602. "shared": false,
  2603. "sort": 2,
  2604. "value_type": "individual"
  2605. },
  2606. "type": "graph",
  2607. "xaxis": {
  2608. "buckets": null,
  2609. "mode": "time",
  2610. "name": null,
  2611. "show": true,
  2612. "values": []
  2613. },
  2614. "yaxes": [
  2615. {
  2616. "format": "percentunit",
  2617. "logBase": 1,
  2618. "max": null,
  2619. "min": null,
  2620. "show": true
  2621. },
  2622. {
  2623. "format": "short",
  2624. "logBase": 1,
  2625. "max": null,
  2626. "min": null,
  2627. "show": true
  2628. }
  2629. ],
  2630. "yaxis": {
  2631. "align": false,
  2632. "alignLevel": null
  2633. }
  2634. },
  2635. {
  2636. "aliasColors": {},
  2637. "bars": false,
  2638. "dashLength": 10,
  2639. "dashes": false,
  2640. "datasource": "$datasource",
  2641. "decimals": null,
  2642. "editable": true,
  2643. "error": false,
  2644. "fieldConfig": {
  2645. "defaults": {
  2646. "custom": {},
  2647. "links": []
  2648. },
  2649. "overrides": []
  2650. },
  2651. "fill": 0,
  2652. "fillGradient": 0,
  2653. "grid": {},
  2654. "gridPos": {
  2655. "h": 8,
  2656. "w": 12,
  2657. "x": 12,
  2658. "y": 39
  2659. },
  2660. "hiddenSeries": false,
  2661. "id": 52,
  2662. "legend": {
  2663. "alignAsTable": true,
  2664. "avg": false,
  2665. "current": false,
  2666. "hideEmpty": false,
  2667. "hideZero": true,
  2668. "max": false,
  2669. "min": false,
  2670. "rightSide": false,
  2671. "show": true,
  2672. "total": false,
  2673. "values": false
  2674. },
  2675. "lines": true,
  2676. "linewidth": 1,
  2677. "links": [],
  2678. "nullPointMode": "null",
  2679. "options": {
  2680. "alertThreshold": true
  2681. },
  2682. "percentage": false,
  2683. "pluginVersion": "7.3.7",
  2684. "pointradius": 5,
  2685. "points": false,
  2686. "renderer": "flot",
  2687. "seriesOverrides": [],
  2688. "spaceLength": 10,
  2689. "stack": false,
  2690. "steppedLine": false,
  2691. "targets": [
  2692. {
  2693. "expr": "(rate(synapse_http_server_in_flight_requests_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_in_flight_requests_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) / rate(synapse_http_server_requests_received{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
  2694. "format": "time_series",
  2695. "interval": "",
  2696. "intervalFactor": 1,
  2697. "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
  2698. "refId": "A",
  2699. "step": 20
  2700. }
  2701. ],
  2702. "thresholds": [
  2703. {
  2704. "colorMode": "custom",
  2705. "fill": true,
  2706. "fillColor": "rgba(216, 200, 27, 0.27)",
  2707. "op": "gt",
  2708. "value": 100,
  2709. "yaxis": "left"
  2710. },
  2711. {
  2712. "colorMode": "custom",
  2713. "fill": true,
  2714. "fillColor": "rgba(234, 112, 112, 0.22)",
  2715. "op": "gt",
  2716. "value": 250,
  2717. "yaxis": "left"
  2718. }
  2719. ],
  2720. "timeFrom": null,
  2721. "timeRegions": [],
  2722. "timeShift": null,
  2723. "title": "Average CPU Usage by Endpoint",
  2724. "tooltip": {
  2725. "shared": false,
  2726. "sort": 2,
  2727. "value_type": "individual"
  2728. },
  2729. "type": "graph",
  2730. "xaxis": {
  2731. "buckets": null,
  2732. "mode": "time",
  2733. "name": null,
  2734. "show": true,
  2735. "values": []
  2736. },
  2737. "yaxes": [
  2738. {
  2739. "format": "s",
  2740. "logBase": 1,
  2741. "max": null,
  2742. "min": null,
  2743. "show": true
  2744. },
  2745. {
  2746. "format": "short",
  2747. "logBase": 1,
  2748. "max": null,
  2749. "min": null,
  2750. "show": true
  2751. }
  2752. ],
  2753. "yaxis": {
  2754. "align": false,
  2755. "alignLevel": null
  2756. }
  2757. },
  2758. {
  2759. "aliasColors": {},
  2760. "bars": false,
  2761. "dashLength": 10,
  2762. "dashes": false,
  2763. "datasource": "$datasource",
  2764. "editable": true,
  2765. "error": false,
  2766. "fieldConfig": {
  2767. "defaults": {
  2768. "custom": {},
  2769. "links": []
  2770. },
  2771. "overrides": []
  2772. },
  2773. "fill": 1,
  2774. "fillGradient": 0,
  2775. "grid": {},
  2776. "gridPos": {
  2777. "h": 8,
  2778. "w": 12,
  2779. "x": 0,
  2780. "y": 47
  2781. },
  2782. "hiddenSeries": false,
  2783. "id": 7,
  2784. "legend": {
  2785. "alignAsTable": true,
  2786. "avg": false,
  2787. "current": false,
  2788. "hideEmpty": true,
  2789. "hideZero": true,
  2790. "max": false,
  2791. "min": false,
  2792. "show": true,
  2793. "total": false,
  2794. "values": false
  2795. },
  2796. "lines": true,
  2797. "linewidth": 1,
  2798. "links": [],
  2799. "nullPointMode": "null",
  2800. "options": {
  2801. "alertThreshold": true
  2802. },
  2803. "percentage": false,
  2804. "pluginVersion": "7.3.7",
  2805. "pointradius": 5,
  2806. "points": false,
  2807. "renderer": "flot",
  2808. "seriesOverrides": [],
  2809. "spaceLength": 10,
  2810. "stack": false,
  2811. "steppedLine": false,
  2812. "targets": [
  2813. {
  2814. "expr": "rate(synapse_http_server_in_flight_requests_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
  2815. "format": "time_series",
  2816. "interval": "",
  2817. "intervalFactor": 1,
  2818. "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
  2819. "refId": "A",
  2820. "step": 20
  2821. }
  2822. ],
  2823. "thresholds": [],
  2824. "timeFrom": null,
  2825. "timeRegions": [],
  2826. "timeShift": null,
  2827. "title": "DB Usage by endpoint",
  2828. "tooltip": {
  2829. "shared": false,
  2830. "sort": 2,
  2831. "value_type": "cumulative"
  2832. },
  2833. "type": "graph",
  2834. "xaxis": {
  2835. "buckets": null,
  2836. "mode": "time",
  2837. "name": null,
  2838. "show": true,
  2839. "values": []
  2840. },
  2841. "yaxes": [
  2842. {
  2843. "format": "percentunit",
  2844. "logBase": 1,
  2845. "max": null,
  2846. "min": null,
  2847. "show": true
  2848. },
  2849. {
  2850. "format": "short",
  2851. "logBase": 1,
  2852. "max": null,
  2853. "min": null,
  2854. "show": true
  2855. }
  2856. ],
  2857. "yaxis": {
  2858. "align": false,
  2859. "alignLevel": null
  2860. }
  2861. },
  2862. {
  2863. "aliasColors": {},
  2864. "bars": false,
  2865. "dashLength": 10,
  2866. "dashes": false,
  2867. "datasource": "$datasource",
  2868. "decimals": null,
  2869. "editable": true,
  2870. "error": false,
  2871. "fieldConfig": {
  2872. "defaults": {
  2873. "custom": {},
  2874. "links": []
  2875. },
  2876. "overrides": []
  2877. },
  2878. "fill": 2,
  2879. "fillGradient": 0,
  2880. "grid": {},
  2881. "gridPos": {
  2882. "h": 8,
  2883. "w": 12,
  2884. "x": 12,
  2885. "y": 47
  2886. },
  2887. "hiddenSeries": false,
  2888. "id": 47,
  2889. "legend": {
  2890. "alignAsTable": true,
  2891. "avg": true,
  2892. "current": false,
  2893. "hideEmpty": false,
  2894. "hideZero": true,
  2895. "max": true,
  2896. "min": false,
  2897. "rightSide": false,
  2898. "show": true,
  2899. "total": false,
  2900. "values": true
  2901. },
  2902. "lines": true,
  2903. "linewidth": 1,
  2904. "links": [],
  2905. "nullPointMode": "null",
  2906. "options": {
  2907. "alertThreshold": true
  2908. },
  2909. "percentage": false,
  2910. "pluginVersion": "7.3.7",
  2911. "pointradius": 5,
  2912. "points": false,
  2913. "renderer": "flot",
  2914. "seriesOverrides": [],
  2915. "spaceLength": 10,
  2916. "stack": false,
  2917. "steppedLine": false,
  2918. "targets": [
  2919. {
  2920. "expr": "(sum(rate(synapse_http_server_response_time_seconds_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\",tag!=\"incremental_sync\"}[$bucket_size])) without (code))/(sum(rate(synapse_http_server_response_time_seconds_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\",tag!=\"incremental_sync\"}[$bucket_size])) without (code))",
  2921. "format": "time_series",
  2922. "hide": false,
  2923. "interval": "",
  2924. "intervalFactor": 1,
  2925. "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}}",
  2926. "refId": "A",
  2927. "step": 20
  2928. }
  2929. ],
  2930. "thresholds": [],
  2931. "timeFrom": null,
  2932. "timeRegions": [],
  2933. "timeShift": null,
  2934. "title": "Non-sync avg response time",
  2935. "tooltip": {
  2936. "shared": false,
  2937. "sort": 2,
  2938. "value_type": "individual"
  2939. },
  2940. "type": "graph",
  2941. "xaxis": {
  2942. "buckets": null,
  2943. "mode": "time",
  2944. "name": null,
  2945. "show": true,
  2946. "values": []
  2947. },
  2948. "yaxes": [
  2949. {
  2950. "format": "s",
  2951. "logBase": 1,
  2952. "max": null,
  2953. "min": null,
  2954. "show": true
  2955. },
  2956. {
  2957. "format": "short",
  2958. "logBase": 1,
  2959. "max": null,
  2960. "min": null,
  2961. "show": false
  2962. }
  2963. ],
  2964. "yaxis": {
  2965. "align": false,
  2966. "alignLevel": null
  2967. }
  2968. },
  2969. {
  2970. "aliasColors": {},
  2971. "bars": false,
  2972. "dashLength": 10,
  2973. "dashes": false,
  2974. "datasource": "$datasource",
  2975. "fieldConfig": {
  2976. "defaults": {
  2977. "custom": {},
  2978. "links": []
  2979. },
  2980. "overrides": []
  2981. },
  2982. "fill": 1,
  2983. "fillGradient": 0,
  2984. "gridPos": {
  2985. "h": 9,
  2986. "w": 12,
  2987. "x": 0,
  2988. "y": 55
  2989. },
  2990. "hiddenSeries": false,
  2991. "id": 103,
  2992. "legend": {
  2993. "avg": false,
  2994. "current": false,
  2995. "max": false,
  2996. "min": false,
  2997. "show": true,
  2998. "total": false,
  2999. "values": false
  3000. },
  3001. "lines": true,
  3002. "linewidth": 1,
  3003. "links": [],
  3004. "nullPointMode": "null",
  3005. "options": {
  3006. "alertThreshold": true
  3007. },
  3008. "percentage": false,
  3009. "pluginVersion": "7.3.7",
  3010. "pointradius": 5,
  3011. "points": false,
  3012. "renderer": "flot",
  3013. "seriesOverrides": [
  3014. {
  3015. "alias": "Total",
  3016. "color": "rgb(255, 255, 255)",
  3017. "fill": 0,
  3018. "linewidth": 3
  3019. }
  3020. ],
  3021. "spaceLength": 10,
  3022. "stack": false,
  3023. "steppedLine": false,
  3024. "targets": [
  3025. {
  3026. "expr": "topk(10,synapse_http_server_in_flight_requests_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})",
  3027. "format": "time_series",
  3028. "interval": "",
  3029. "intervalFactor": 1,
  3030. "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}}",
  3031. "refId": "A"
  3032. },
  3033. {
  3034. "expr": "sum(avg_over_time(synapse_http_server_in_flight_requests_count{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
  3035. "interval": "",
  3036. "legendFormat": "Total",
  3037. "refId": "B"
  3038. }
  3039. ],
  3040. "thresholds": [],
  3041. "timeFrom": null,
  3042. "timeRegions": [],
  3043. "timeShift": null,
  3044. "title": "Requests in flight",
  3045. "tooltip": {
  3046. "shared": false,
  3047. "sort": 2,
  3048. "value_type": "individual"
  3049. },
  3050. "type": "graph",
  3051. "xaxis": {
  3052. "buckets": null,
  3053. "mode": "time",
  3054. "name": null,
  3055. "show": true,
  3056. "values": []
  3057. },
  3058. "yaxes": [
  3059. {
  3060. "format": "short",
  3061. "label": null,
  3062. "logBase": 1,
  3063. "max": null,
  3064. "min": null,
  3065. "show": true
  3066. },
  3067. {
  3068. "format": "short",
  3069. "label": null,
  3070. "logBase": 1,
  3071. "max": null,
  3072. "min": null,
  3073. "show": true
  3074. }
  3075. ],
  3076. "yaxis": {
  3077. "align": false,
  3078. "alignLevel": null
  3079. }
  3080. }
  3081. ],
  3082. "repeat": null,
  3083. "title": "Requests",
  3084. "type": "row"
  3085. },
  3086. {
  3087. "collapsed": true,
  3088. "datasource": "${DS_PROMETHEUS}",
  3089. "gridPos": {
  3090. "h": 1,
  3091. "w": 24,
  3092. "x": 0,
  3093. "y": 29
  3094. },
  3095. "id": 97,
  3096. "panels": [
  3097. {
  3098. "aliasColors": {},
  3099. "bars": false,
  3100. "dashLength": 10,
  3101. "dashes": false,
  3102. "datasource": "$datasource",
  3103. "fieldConfig": {
  3104. "defaults": {
  3105. "custom": {},
  3106. "links": []
  3107. },
  3108. "overrides": []
  3109. },
  3110. "fill": 1,
  3111. "fillGradient": 0,
  3112. "gridPos": {
  3113. "h": 9,
  3114. "w": 12,
  3115. "x": 0,
  3116. "y": 32
  3117. },
  3118. "hiddenSeries": false,
  3119. "id": 99,
  3120. "legend": {
  3121. "avg": false,
  3122. "current": false,
  3123. "max": false,
  3124. "min": false,
  3125. "show": true,
  3126. "total": false,
  3127. "values": false
  3128. },
  3129. "lines": true,
  3130. "linewidth": 1,
  3131. "links": [],
  3132. "nullPointMode": "null",
  3133. "paceLength": 10,
  3134. "percentage": false,
  3135. "pluginVersion": "7.1.3",
  3136. "pointradius": 5,
  3137. "points": false,
  3138. "renderer": "flot",
  3139. "seriesOverrides": [],
  3140. "spaceLength": 10,
  3141. "stack": false,
  3142. "steppedLine": false,
  3143. "targets": [
  3144. {
  3145. "expr": "rate(synapse_background_process_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
  3146. "format": "time_series",
  3147. "interval": "",
  3148. "intervalFactor": 1,
  3149. "legendFormat": "{{job}}-{{index}} {{name}}",
  3150. "refId": "A"
  3151. }
  3152. ],
  3153. "thresholds": [],
  3154. "timeFrom": null,
  3155. "timeRegions": [],
  3156. "timeShift": null,
  3157. "title": "CPU usage by background jobs",
  3158. "tooltip": {
  3159. "shared": false,
  3160. "sort": 0,
  3161. "value_type": "individual"
  3162. },
  3163. "type": "graph",
  3164. "xaxis": {
  3165. "buckets": null,
  3166. "mode": "time",
  3167. "name": null,
  3168. "show": true,
  3169. "values": []
  3170. },
  3171. "yaxes": [
  3172. {
  3173. "format": "percentunit",
  3174. "label": null,
  3175. "logBase": 1,
  3176. "max": null,
  3177. "min": null,
  3178. "show": true
  3179. },
  3180. {
  3181. "format": "short",
  3182. "label": null,
  3183. "logBase": 1,
  3184. "max": null,
  3185. "min": null,
  3186. "show": true
  3187. }
  3188. ],
  3189. "yaxis": {
  3190. "align": false,
  3191. "alignLevel": null
  3192. }
  3193. },
  3194. {
  3195. "aliasColors": {},
  3196. "bars": false,
  3197. "dashLength": 10,
  3198. "dashes": false,
  3199. "datasource": "$datasource",
  3200. "fieldConfig": {
  3201. "defaults": {
  3202. "custom": {},
  3203. "links": []
  3204. },
  3205. "overrides": []
  3206. },
  3207. "fill": 1,
  3208. "fillGradient": 0,
  3209. "gridPos": {
  3210. "h": 9,
  3211. "w": 12,
  3212. "x": 12,
  3213. "y": 32
  3214. },
  3215. "hiddenSeries": false,
  3216. "id": 101,
  3217. "legend": {
  3218. "avg": false,
  3219. "current": false,
  3220. "max": false,
  3221. "min": false,
  3222. "show": true,
  3223. "total": false,
  3224. "values": false
  3225. },
  3226. "lines": true,
  3227. "linewidth": 1,
  3228. "links": [],
  3229. "nullPointMode": "null",
  3230. "paceLength": 10,
  3231. "percentage": false,
  3232. "pluginVersion": "7.1.3",
  3233. "pointradius": 5,
  3234. "points": false,
  3235. "renderer": "flot",
  3236. "seriesOverrides": [],
  3237. "spaceLength": 10,
  3238. "stack": false,
  3239. "steppedLine": false,
  3240. "targets": [
  3241. {
  3242. "expr": "rate(synapse_background_process_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) + rate(synapse_background_process_db_sched_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
  3243. "format": "time_series",
  3244. "hide": false,
  3245. "intervalFactor": 1,
  3246. "legendFormat": "{{job}}-{{index}} {{name}}",
  3247. "refId": "A"
  3248. }
  3249. ],
  3250. "thresholds": [],
  3251. "timeFrom": null,
  3252. "timeRegions": [],
  3253. "timeShift": null,
  3254. "title": "DB usage by background jobs (including scheduling time)",
  3255. "tooltip": {
  3256. "shared": false,
  3257. "sort": 0,
  3258. "value_type": "individual"
  3259. },
  3260. "type": "graph",
  3261. "xaxis": {
  3262. "buckets": null,
  3263. "mode": "time",
  3264. "name": null,
  3265. "show": true,
  3266. "values": []
  3267. },
  3268. "yaxes": [
  3269. {
  3270. "format": "percentunit",
  3271. "label": null,
  3272. "logBase": 1,
  3273. "max": null,
  3274. "min": null,
  3275. "show": true
  3276. },
  3277. {
  3278. "format": "short",
  3279. "label": null,
  3280. "logBase": 1,
  3281. "max": null,
  3282. "min": null,
  3283. "show": true
  3284. }
  3285. ],
  3286. "yaxis": {
  3287. "align": false,
  3288. "alignLevel": null
  3289. }
  3290. },
  3291. {
  3292. "aliasColors": {},
  3293. "bars": false,
  3294. "dashLength": 10,
  3295. "dashes": false,
  3296. "datasource": "$datasource",
  3297. "fieldConfig": {
  3298. "defaults": {
  3299. "custom": {},
  3300. "links": []
  3301. },
  3302. "overrides": []
  3303. },
  3304. "fill": 1,
  3305. "fillGradient": 0,
  3306. "gridPos": {
  3307. "h": 8,
  3308. "w": 12,
  3309. "x": 0,
  3310. "y": 41
  3311. },
  3312. "hiddenSeries": false,
  3313. "id": 138,
  3314. "legend": {
  3315. "avg": false,
  3316. "current": false,
  3317. "max": false,
  3318. "min": false,
  3319. "show": true,
  3320. "total": false,
  3321. "values": false
  3322. },
  3323. "lines": true,
  3324. "linewidth": 1,
  3325. "nullPointMode": "null",
  3326. "percentage": false,
  3327. "pluginVersion": "7.1.3",
  3328. "pointradius": 2,
  3329. "points": false,
  3330. "renderer": "flot",
  3331. "seriesOverrides": [],
  3332. "spaceLength": 10,
  3333. "stack": false,
  3334. "steppedLine": false,
  3335. "targets": [
  3336. {
  3337. "expr": "synapse_background_process_in_flight_count{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
  3338. "legendFormat": "{{job}}-{{index}} {{name}}",
  3339. "refId": "A"
  3340. }
  3341. ],
  3342. "thresholds": [],
  3343. "timeFrom": null,
  3344. "timeRegions": [],
  3345. "timeShift": null,
  3346. "title": "Background jobs in flight",
  3347. "tooltip": {
  3348. "shared": false,
  3349. "sort": 0,
  3350. "value_type": "individual"
  3351. },
  3352. "type": "graph",
  3353. "xaxis": {
  3354. "buckets": null,
  3355. "mode": "time",
  3356. "name": null,
  3357. "show": true,
  3358. "values": []
  3359. },
  3360. "yaxes": [
  3361. {
  3362. "format": "short",
  3363. "label": null,
  3364. "logBase": 1,
  3365. "max": null,
  3366. "min": null,
  3367. "show": true
  3368. },
  3369. {
  3370. "format": "short",
  3371. "label": null,
  3372. "logBase": 1,
  3373. "max": null,
  3374. "min": null,
  3375. "show": true
  3376. }
  3377. ],
  3378. "yaxis": {
  3379. "align": false,
  3380. "alignLevel": null
  3381. }
  3382. }
  3383. ],
  3384. "title": "Background jobs",
  3385. "type": "row"
  3386. },
  3387. {
  3388. "collapsed": true,
  3389. "datasource": "${DS_PROMETHEUS}",
  3390. "gridPos": {
  3391. "h": 1,
  3392. "w": 24,
  3393. "x": 0,
  3394. "y": 30
  3395. },
  3396. "id": 81,
  3397. "panels": [
  3398. {
  3399. "aliasColors": {},
  3400. "bars": false,
  3401. "dashLength": 10,
  3402. "dashes": false,
  3403. "datasource": "$datasource",
  3404. "fieldConfig": {
  3405. "defaults": {
  3406. "custom": {},
  3407. "links": []
  3408. },
  3409. "overrides": []
  3410. },
  3411. "fill": 1,
  3412. "fillGradient": 0,
  3413. "gridPos": {
  3414. "h": 9,
  3415. "w": 12,
  3416. "x": 0,
  3417. "y": 6
  3418. },
  3419. "hiddenSeries": false,
  3420. "id": 79,
  3421. "legend": {
  3422. "avg": false,
  3423. "current": false,
  3424. "max": false,
  3425. "min": false,
  3426. "show": true,
  3427. "total": false,
  3428. "values": false
  3429. },
  3430. "lines": true,
  3431. "linewidth": 1,
  3432. "links": [],
  3433. "nullPointMode": "null",
  3434. "options": {
  3435. "alertThreshold": true
  3436. },
  3437. "paceLength": 10,
  3438. "percentage": false,
  3439. "pluginVersion": "7.3.7",
  3440. "pointradius": 5,
  3441. "points": false,
  3442. "renderer": "flot",
  3443. "seriesOverrides": [],
  3444. "spaceLength": 10,
  3445. "stack": false,
  3446. "steppedLine": false,
  3447. "targets": [
  3448. {
  3449. "expr": "sum(rate(synapse_federation_client_sent_transactions{instance=\"$instance\"}[$bucket_size]))",
  3450. "format": "time_series",
  3451. "intervalFactor": 1,
  3452. "legendFormat": "successful txn rate",
  3453. "refId": "A"
  3454. },
  3455. {
  3456. "expr": "sum(rate(synapse_util_metrics_block_count{block_name=\"_send_new_transaction\",instance=\"$instance\"}[$bucket_size]) - ignoring (block_name) rate(synapse_federation_client_sent_transactions{instance=\"$instance\"}[$bucket_size]))",
  3457. "legendFormat": "failed txn rate",
  3458. "refId": "B"
  3459. }
  3460. ],
  3461. "thresholds": [],
  3462. "timeFrom": null,
  3463. "timeRegions": [],
  3464. "timeShift": null,
  3465. "title": "Outgoing federation transaction rate",
  3466. "tooltip": {
  3467. "shared": true,
  3468. "sort": 0,
  3469. "value_type": "individual"
  3470. },
  3471. "type": "graph",
  3472. "xaxis": {
  3473. "buckets": null,
  3474. "mode": "time",
  3475. "name": null,
  3476. "show": true,
  3477. "values": []
  3478. },
  3479. "yaxes": [
  3480. {
  3481. "format": "hertz",
  3482. "label": null,
  3483. "logBase": 1,
  3484. "max": null,
  3485. "min": null,
  3486. "show": true
  3487. },
  3488. {
  3489. "format": "short",
  3490. "label": null,
  3491. "logBase": 1,
  3492. "max": null,
  3493. "min": null,
  3494. "show": true
  3495. }
  3496. ],
  3497. "yaxis": {
  3498. "align": false,
  3499. "alignLevel": null
  3500. }
  3501. },
  3502. {
  3503. "aliasColors": {},
  3504. "bars": false,
  3505. "dashLength": 10,
  3506. "dashes": false,
  3507. "datasource": "$datasource",
  3508. "fieldConfig": {
  3509. "defaults": {
  3510. "custom": {},
  3511. "links": []
  3512. },
  3513. "overrides": []
  3514. },
  3515. "fill": 1,
  3516. "fillGradient": 0,
  3517. "gridPos": {
  3518. "h": 9,
  3519. "w": 12,
  3520. "x": 12,
  3521. "y": 6
  3522. },
  3523. "hiddenSeries": false,
  3524. "id": 83,
  3525. "legend": {
  3526. "avg": false,
  3527. "current": false,
  3528. "max": false,
  3529. "min": false,
  3530. "show": true,
  3531. "total": false,
  3532. "values": false
  3533. },
  3534. "lines": true,
  3535. "linewidth": 1,
  3536. "links": [],
  3537. "nullPointMode": "null",
  3538. "options": {
  3539. "alertThreshold": true
  3540. },
  3541. "paceLength": 10,
  3542. "percentage": false,
  3543. "pluginVersion": "7.3.7",
  3544. "pointradius": 5,
  3545. "points": false,
  3546. "renderer": "flot",
  3547. "seriesOverrides": [],
  3548. "spaceLength": 10,
  3549. "stack": false,
  3550. "steppedLine": false,
  3551. "targets": [
  3552. {
  3553. "expr": "sum(rate(synapse_federation_server_received_pdus{instance=~\"$instance\"}[$bucket_size]))",
  3554. "format": "time_series",
  3555. "intervalFactor": 1,
  3556. "legendFormat": "pdus",
  3557. "refId": "A"
  3558. },
  3559. {
  3560. "expr": "sum(rate(synapse_federation_server_received_edus{instance=~\"$instance\"}[$bucket_size]))",
  3561. "format": "time_series",
  3562. "intervalFactor": 1,
  3563. "legendFormat": "edus",
  3564. "refId": "B"
  3565. }
  3566. ],
  3567. "thresholds": [],
  3568. "timeFrom": null,
  3569. "timeRegions": [],
  3570. "timeShift": null,
  3571. "title": "Incoming PDU/EDU rate",
  3572. "tooltip": {
  3573. "shared": true,
  3574. "sort": 0,
  3575. "value_type": "individual"
  3576. },
  3577. "type": "graph",
  3578. "xaxis": {
  3579. "buckets": null,
  3580. "mode": "time",
  3581. "name": null,
  3582. "show": true,
  3583. "values": []
  3584. },
  3585. "yaxes": [
  3586. {
  3587. "format": "hertz",
  3588. "label": null,
  3589. "logBase": 1,
  3590. "max": null,
  3591. "min": null,
  3592. "show": true
  3593. },
  3594. {
  3595. "format": "short",
  3596. "label": null,
  3597. "logBase": 1,
  3598. "max": null,
  3599. "min": null,
  3600. "show": true
  3601. }
  3602. ],
  3603. "yaxis": {
  3604. "align": false,
  3605. "alignLevel": null
  3606. }
  3607. },
  3608. {
  3609. "aliasColors": {},
  3610. "bars": false,
  3611. "dashLength": 10,
  3612. "dashes": false,
  3613. "datasource": "$datasource",
  3614. "fieldConfig": {
  3615. "defaults": {
  3616. "custom": {},
  3617. "links": []
  3618. },
  3619. "overrides": []
  3620. },
  3621. "fill": 1,
  3622. "fillGradient": 0,
  3623. "gridPos": {
  3624. "h": 9,
  3625. "w": 12,
  3626. "x": 0,
  3627. "y": 15
  3628. },
  3629. "hiddenSeries": false,
  3630. "id": 109,
  3631. "legend": {
  3632. "avg": false,
  3633. "current": false,
  3634. "max": false,
  3635. "min": false,
  3636. "show": true,
  3637. "total": false,
  3638. "values": false
  3639. },
  3640. "lines": true,
  3641. "linewidth": 1,
  3642. "links": [],
  3643. "nullPointMode": "null",
  3644. "options": {
  3645. "alertThreshold": true
  3646. },
  3647. "paceLength": 10,
  3648. "percentage": false,
  3649. "pluginVersion": "7.3.7",
  3650. "pointradius": 5,
  3651. "points": false,
  3652. "renderer": "flot",
  3653. "seriesOverrides": [],
  3654. "spaceLength": 10,
  3655. "stack": false,
  3656. "steppedLine": false,
  3657. "targets": [
  3658. {
  3659. "expr": "sum(rate(synapse_federation_client_sent_pdu_destinations:total{instance=\"$instance\"}[$bucket_size]))",
  3660. "format": "time_series",
  3661. "interval": "",
  3662. "intervalFactor": 1,
  3663. "legendFormat": "pdus",
  3664. "refId": "A"
  3665. },
  3666. {
  3667. "expr": "sum(rate(synapse_federation_client_sent_edus{instance=\"$instance\"}[$bucket_size]))",
  3668. "format": "time_series",
  3669. "intervalFactor": 1,
  3670. "legendFormat": "edus",
  3671. "refId": "B"
  3672. }
  3673. ],
  3674. "thresholds": [],
  3675. "timeFrom": null,
  3676. "timeRegions": [],
  3677. "timeShift": null,
  3678. "title": "Outgoing PDU/EDU rate",
  3679. "tooltip": {
  3680. "shared": true,
  3681. "sort": 0,
  3682. "value_type": "individual"
  3683. },
  3684. "type": "graph",
  3685. "xaxis": {
  3686. "buckets": null,
  3687. "mode": "time",
  3688. "name": null,
  3689. "show": true,
  3690. "values": []
  3691. },
  3692. "yaxes": [
  3693. {
  3694. "format": "hertz",
  3695. "label": null,
  3696. "logBase": 1,
  3697. "max": null,
  3698. "min": null,
  3699. "show": true
  3700. },
  3701. {
  3702. "format": "short",
  3703. "label": null,
  3704. "logBase": 1,
  3705. "max": null,
  3706. "min": null,
  3707. "show": true
  3708. }
  3709. ],
  3710. "yaxis": {
  3711. "align": false,
  3712. "alignLevel": null
  3713. }
  3714. },
  3715. {
  3716. "aliasColors": {},
  3717. "bars": false,
  3718. "dashLength": 10,
  3719. "dashes": false,
  3720. "datasource": "$datasource",
  3721. "fieldConfig": {
  3722. "defaults": {
  3723. "custom": {},
  3724. "links": []
  3725. },
  3726. "overrides": []
  3727. },
  3728. "fill": 1,
  3729. "fillGradient": 0,
  3730. "gridPos": {
  3731. "h": 9,
  3732. "w": 12,
  3733. "x": 12,
  3734. "y": 15
  3735. },
  3736. "hiddenSeries": false,
  3737. "id": 111,
  3738. "legend": {
  3739. "avg": false,
  3740. "current": false,
  3741. "max": false,
  3742. "min": false,
  3743. "show": true,
  3744. "total": false,
  3745. "values": false
  3746. },
  3747. "lines": true,
  3748. "linewidth": 1,
  3749. "links": [],
  3750. "nullPointMode": "null",
  3751. "options": {
  3752. "alertThreshold": true
  3753. },
  3754. "paceLength": 10,
  3755. "percentage": false,
  3756. "pluginVersion": "7.3.7",
  3757. "pointradius": 5,
  3758. "points": false,
  3759. "renderer": "flot",
  3760. "seriesOverrides": [],
  3761. "spaceLength": 10,
  3762. "stack": false,
  3763. "steppedLine": false,
  3764. "targets": [
  3765. {
  3766. "expr": "rate(synapse_federation_client_sent_edus_by_type{instance=\"$instance\"}[$bucket_size])",
  3767. "format": "time_series",
  3768. "interval": "",
  3769. "intervalFactor": 1,
  3770. "legendFormat": "{{type}}",
  3771. "refId": "A"
  3772. }
  3773. ],
  3774. "thresholds": [],
  3775. "timeFrom": null,
  3776. "timeRegions": [],
  3777. "timeShift": null,
  3778. "title": "Outgoing EDUs by type",
  3779. "tooltip": {
  3780. "shared": true,
  3781. "sort": 0,
  3782. "value_type": "individual"
  3783. },
  3784. "type": "graph",
  3785. "xaxis": {
  3786. "buckets": null,
  3787. "mode": "time",
  3788. "name": null,
  3789. "show": true,
  3790. "values": []
  3791. },
  3792. "yaxes": [
  3793. {
  3794. "format": "hertz",
  3795. "label": null,
  3796. "logBase": 1,
  3797. "max": null,
  3798. "min": null,
  3799. "show": true
  3800. },
  3801. {
  3802. "format": "short",
  3803. "label": null,
  3804. "logBase": 1,
  3805. "max": null,
  3806. "min": null,
  3807. "show": true
  3808. }
  3809. ],
  3810. "yaxis": {
  3811. "align": false,
  3812. "alignLevel": null
  3813. }
  3814. },
  3815. {
  3816. "aliasColors": {},
  3817. "bars": false,
  3818. "dashLength": 10,
  3819. "dashes": false,
  3820. "datasource": "${DS_PROMETHEUS}",
  3821. "description": "The number of events in the in-memory queues ",
  3822. "fieldConfig": {
  3823. "defaults": {
  3824. "custom": {},
  3825. "links": []
  3826. },
  3827. "overrides": []
  3828. },
  3829. "fill": 1,
  3830. "fillGradient": 0,
  3831. "gridPos": {
  3832. "h": 8,
  3833. "w": 12,
  3834. "x": 0,
  3835. "y": 24
  3836. },
  3837. "hiddenSeries": false,
  3838. "id": 142,
  3839. "legend": {
  3840. "avg": false,
  3841. "current": false,
  3842. "max": false,
  3843. "min": false,
  3844. "show": true,
  3845. "total": false,
  3846. "values": false
  3847. },
  3848. "lines": true,
  3849. "linewidth": 1,
  3850. "nullPointMode": "null",
  3851. "options": {
  3852. "alertThreshold": true
  3853. },
  3854. "percentage": false,
  3855. "pluginVersion": "7.3.7",
  3856. "pointradius": 2,
  3857. "points": false,
  3858. "renderer": "flot",
  3859. "seriesOverrides": [],
  3860. "spaceLength": 10,
  3861. "stack": false,
  3862. "steppedLine": false,
  3863. "targets": [
  3864. {
  3865. "expr": "synapse_federation_transaction_queue_pending_pdus{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
  3866. "interval": "",
  3867. "legendFormat": "pending PDUs {{job}}-{{index}}",
  3868. "refId": "A"
  3869. },
  3870. {
  3871. "expr": "synapse_federation_transaction_queue_pending_edus{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
  3872. "interval": "",
  3873. "legendFormat": "pending EDUs {{job}}-{{index}}",
  3874. "refId": "B"
  3875. }
  3876. ],
  3877. "thresholds": [],
  3878. "timeFrom": null,
  3879. "timeRegions": [],
  3880. "timeShift": null,
  3881. "title": "In-memory federation transmission queues",
  3882. "tooltip": {
  3883. "shared": true,
  3884. "sort": 0,
  3885. "value_type": "individual"
  3886. },
  3887. "type": "graph",
  3888. "xaxis": {
  3889. "buckets": null,
  3890. "mode": "time",
  3891. "name": null,
  3892. "show": true,
  3893. "values": []
  3894. },
  3895. "yaxes": [
  3896. {
  3897. "format": "short",
  3898. "label": "events",
  3899. "logBase": 1,
  3900. "max": null,
  3901. "min": "0",
  3902. "show": true
  3903. },
  3904. {
  3905. "format": "short",
  3906. "label": "",
  3907. "logBase": 1,
  3908. "max": null,
  3909. "min": null,
  3910. "show": true
  3911. }
  3912. ],
  3913. "yaxis": {
  3914. "align": false,
  3915. "alignLevel": null
  3916. }
  3917. },
  3918. {
  3919. "aliasColors": {},
  3920. "bars": false,
  3921. "dashLength": 10,
  3922. "dashes": false,
  3923. "datasource": "$datasource",
  3924. "description": "Number of events queued up on the master process for processing by the federation sender",
  3925. "fieldConfig": {
  3926. "defaults": {
  3927. "custom": {},
  3928. "links": []
  3929. },
  3930. "overrides": []
  3931. },
  3932. "fill": 1,
  3933. "fillGradient": 0,
  3934. "gridPos": {
  3935. "h": 9,
  3936. "w": 12,
  3937. "x": 12,
  3938. "y": 24
  3939. },
  3940. "hiddenSeries": false,
  3941. "id": 140,
  3942. "legend": {
  3943. "avg": false,
  3944. "current": false,
  3945. "max": false,
  3946. "min": false,
  3947. "show": true,
  3948. "total": false,
  3949. "values": false
  3950. },
  3951. "lines": true,
  3952. "linewidth": 1,
  3953. "links": [],
  3954. "nullPointMode": "null",
  3955. "options": {
  3956. "alertThreshold": true
  3957. },
  3958. "paceLength": 10,
  3959. "percentage": false,
  3960. "pluginVersion": "7.3.7",
  3961. "pointradius": 5,
  3962. "points": false,
  3963. "renderer": "flot",
  3964. "seriesOverrides": [],
  3965. "spaceLength": 10,
  3966. "stack": false,
  3967. "steppedLine": false,
  3968. "targets": [
  3969. {
  3970. "expr": "synapse_federation_send_queue_presence_changed_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
  3971. "format": "time_series",
  3972. "interval": "",
  3973. "intervalFactor": 1,
  3974. "legendFormat": "presence changed",
  3975. "refId": "A"
  3976. },
  3977. {
  3978. "expr": "synapse_federation_send_queue_presence_map_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
  3979. "format": "time_series",
  3980. "hide": false,
  3981. "interval": "",
  3982. "intervalFactor": 1,
  3983. "legendFormat": "presence map",
  3984. "refId": "B"
  3985. },
  3986. {
  3987. "expr": "synapse_federation_send_queue_presence_destinations_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
  3988. "format": "time_series",
  3989. "hide": false,
  3990. "interval": "",
  3991. "intervalFactor": 1,
  3992. "legendFormat": "presence destinations",
  3993. "refId": "E"
  3994. },
  3995. {
  3996. "expr": "synapse_federation_send_queue_keyed_edu_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
  3997. "format": "time_series",
  3998. "hide": false,
  3999. "interval": "",
  4000. "intervalFactor": 1,
  4001. "legendFormat": "keyed edus",
  4002. "refId": "C"
  4003. },
  4004. {
  4005. "expr": "synapse_federation_send_queue_edus_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
  4006. "format": "time_series",
  4007. "hide": false,
  4008. "interval": "",
  4009. "intervalFactor": 1,
  4010. "legendFormat": "other edus",
  4011. "refId": "D"
  4012. },
  4013. {
  4014. "expr": "synapse_federation_send_queue_pos_time_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
  4015. "format": "time_series",
  4016. "hide": false,
  4017. "interval": "",
  4018. "intervalFactor": 1,
  4019. "legendFormat": "stream positions",
  4020. "refId": "F"
  4021. }
  4022. ],
  4023. "thresholds": [],
  4024. "timeFrom": null,
  4025. "timeRegions": [],
  4026. "timeShift": null,
  4027. "title": "Outgoing EDU queues on master",
  4028. "tooltip": {
  4029. "shared": true,
  4030. "sort": 0,
  4031. "value_type": "individual"
  4032. },
  4033. "type": "graph",
  4034. "xaxis": {
  4035. "buckets": null,
  4036. "mode": "time",
  4037. "name": null,
  4038. "show": true,
  4039. "values": []
  4040. },
  4041. "yaxes": [
  4042. {
  4043. "format": "none",
  4044. "label": null,
  4045. "logBase": 1,
  4046. "max": null,
  4047. "min": "0",
  4048. "show": true
  4049. },
  4050. {
  4051. "format": "short",
  4052. "label": null,
  4053. "logBase": 1,
  4054. "max": null,
  4055. "min": null,
  4056. "show": true
  4057. }
  4058. ],
  4059. "yaxis": {
  4060. "align": false,
  4061. "alignLevel": null
  4062. }
  4063. },
  4064. {
  4065. "cards": {
  4066. "cardPadding": -1,
  4067. "cardRound": null
  4068. },
  4069. "color": {
  4070. "cardColor": "#b4ff00",
  4071. "colorScale": "sqrt",
  4072. "colorScheme": "interpolateInferno",
  4073. "exponent": 0.5,
  4074. "min": 0,
  4075. "mode": "spectrum"
  4076. },
  4077. "dataFormat": "tsbuckets",
  4078. "datasource": "$datasource",
  4079. "fieldConfig": {
  4080. "defaults": {
  4081. "custom": {}
  4082. },
  4083. "overrides": []
  4084. },
  4085. "gridPos": {
  4086. "h": 9,
  4087. "w": 12,
  4088. "x": 0,
  4089. "y": 32
  4090. },
  4091. "heatmap": {},
  4092. "hideZeroBuckets": false,
  4093. "highlightCards": true,
  4094. "id": 166,
  4095. "legend": {
  4096. "show": false
  4097. },
  4098. "links": [],
  4099. "reverseYBuckets": false,
  4100. "targets": [
  4101. {
  4102. "expr": "sum(rate(synapse_event_processing_lag_by_event_bucket{instance=\"$instance\",name=\"federation_sender\"}[$bucket_size])) by (le)",
  4103. "format": "heatmap",
  4104. "instant": false,
  4105. "interval": "",
  4106. "intervalFactor": 1,
  4107. "legendFormat": "{{ le }}",
  4108. "refId": "A"
  4109. }
  4110. ],
  4111. "title": "Federation send PDU lag",
  4112. "tooltip": {
  4113. "show": true,
  4114. "showHistogram": true
  4115. },
  4116. "tooltipDecimals": 2,
  4117. "type": "heatmap",
  4118. "xAxis": {
  4119. "show": true
  4120. },
  4121. "xBucketNumber": null,
  4122. "xBucketSize": null,
  4123. "yAxis": {
  4124. "decimals": 0,
  4125. "format": "s",
  4126. "logBase": 1,
  4127. "max": null,
  4128. "min": null,
  4129. "show": true,
  4130. "splitFactor": null
  4131. },
  4132. "yBucketBound": "auto",
  4133. "yBucketNumber": null,
  4134. "yBucketSize": null
  4135. },
  4136. {
  4137. "aliasColors": {},
  4138. "bars": false,
  4139. "dashLength": 10,
  4140. "dashes": false,
  4141. "datasource": "$datasource",
  4142. "fieldConfig": {
  4143. "defaults": {
  4144. "custom": {},
  4145. "links": []
  4146. },
  4147. "overrides": []
  4148. },
  4149. "fill": 0,
  4150. "fillGradient": 0,
  4151. "gridPos": {
  4152. "h": 9,
  4153. "w": 12,
  4154. "x": 12,
  4155. "y": 33
  4156. },
  4157. "hiddenSeries": false,
  4158. "id": 162,
  4159. "legend": {
  4160. "avg": false,
  4161. "current": false,
  4162. "max": false,
  4163. "min": false,
  4164. "rightSide": false,
  4165. "show": true,
  4166. "total": false,
  4167. "values": false
  4168. },
  4169. "lines": true,
  4170. "linewidth": 0,
  4171. "links": [],
  4172. "nullPointMode": "connected",
  4173. "options": {
  4174. "alertThreshold": true
  4175. },
  4176. "paceLength": 10,
  4177. "percentage": false,
  4178. "pluginVersion": "7.3.7",
  4179. "pointradius": 5,
  4180. "points": false,
  4181. "renderer": "flot",
  4182. "seriesOverrides": [
  4183. {
  4184. "alias": "Avg",
  4185. "fill": 0,
  4186. "linewidth": 3
  4187. },
  4188. {
  4189. "alias": "99%",
  4190. "color": "#C4162A",
  4191. "fillBelowTo": "90%"
  4192. },
  4193. {
  4194. "alias": "90%",
  4195. "color": "#FF7383",
  4196. "fillBelowTo": "75%"
  4197. },
  4198. {
  4199. "alias": "75%",
  4200. "color": "#FFEE52",
  4201. "fillBelowTo": "50%"
  4202. },
  4203. {
  4204. "alias": "50%",
  4205. "color": "#73BF69",
  4206. "fillBelowTo": "25%"
  4207. },
  4208. {
  4209. "alias": "25%",
  4210. "color": "#1F60C4",
  4211. "fillBelowTo": "5%"
  4212. },
  4213. {
  4214. "alias": "5%",
  4215. "lines": false
  4216. },
  4217. {
  4218. "alias": "Average",
  4219. "color": "rgb(255, 255, 255)",
  4220. "lines": true,
  4221. "linewidth": 3
  4222. }
  4223. ],
  4224. "spaceLength": 10,
  4225. "stack": false,
  4226. "steppedLine": false,
  4227. "targets": [
  4228. {
  4229. "expr": "histogram_quantile(0.99, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
  4230. "format": "time_series",
  4231. "interval": "",
  4232. "intervalFactor": 1,
  4233. "legendFormat": "99%",
  4234. "refId": "D"
  4235. },
  4236. {
  4237. "expr": "histogram_quantile(0.9, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
  4238. "format": "time_series",
  4239. "interval": "",
  4240. "intervalFactor": 1,
  4241. "legendFormat": "90%",
  4242. "refId": "A"
  4243. },
  4244. {
  4245. "expr": "histogram_quantile(0.75, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
  4246. "format": "time_series",
  4247. "interval": "",
  4248. "intervalFactor": 1,
  4249. "legendFormat": "75%",
  4250. "refId": "C"
  4251. },
  4252. {
  4253. "expr": "histogram_quantile(0.5, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
  4254. "format": "time_series",
  4255. "interval": "",
  4256. "intervalFactor": 1,
  4257. "legendFormat": "50%",
  4258. "refId": "B"
  4259. },
  4260. {
  4261. "expr": "histogram_quantile(0.25, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
  4262. "interval": "",
  4263. "legendFormat": "25%",
  4264. "refId": "F"
  4265. },
  4266. {
  4267. "expr": "histogram_quantile(0.05, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
  4268. "interval": "",
  4269. "legendFormat": "5%",
  4270. "refId": "G"
  4271. },
  4272. {
  4273. "expr": "sum(rate(synapse_event_processing_lag_by_event_sum{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) / sum(rate(synapse_event_processing_lag_by_event_count{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
  4274. "interval": "",
  4275. "legendFormat": "Average",
  4276. "refId": "H"
  4277. }
  4278. ],
  4279. "thresholds": [
  4280. {
  4281. "colorMode": "warning",
  4282. "fill": false,
  4283. "line": true,
  4284. "op": "gt",
  4285. "value": 0.25,
  4286. "yaxis": "left"
  4287. },
  4288. {
  4289. "colorMode": "critical",
  4290. "fill": false,
  4291. "line": true,
  4292. "op": "gt",
  4293. "value": 1,
  4294. "yaxis": "left"
  4295. }
  4296. ],
  4297. "timeFrom": null,
  4298. "timeRegions": [],
  4299. "timeShift": null,
  4300. "title": "Federation send PDU lag quantiles",
  4301. "tooltip": {
  4302. "shared": true,
  4303. "sort": 2,
  4304. "value_type": "individual"
  4305. },
  4306. "type": "graph",
  4307. "xaxis": {
  4308. "buckets": null,
  4309. "mode": "time",
  4310. "name": null,
  4311. "show": true,
  4312. "values": []
  4313. },
  4314. "yaxes": [
  4315. {
  4316. "decimals": null,
  4317. "format": "s",
  4318. "label": "",
  4319. "logBase": 1,
  4320. "max": null,
  4321. "min": "0",
  4322. "show": true
  4323. },
  4324. {
  4325. "format": "hertz",
  4326. "label": "",
  4327. "logBase": 1,
  4328. "max": null,
  4329. "min": "0",
  4330. "show": true
  4331. }
  4332. ],
  4333. "yaxis": {
  4334. "align": false,
  4335. "alignLevel": null
  4336. }
  4337. },
  4338. {
  4339. "cards": {
  4340. "cardPadding": -1,
  4341. "cardRound": null
  4342. },
  4343. "color": {
  4344. "cardColor": "#b4ff00",
  4345. "colorScale": "sqrt",
  4346. "colorScheme": "interpolateInferno",
  4347. "exponent": 0.5,
  4348. "min": 0,
  4349. "mode": "spectrum"
  4350. },
  4351. "dataFormat": "tsbuckets",
  4352. "datasource": "$datasource",
  4353. "fieldConfig": {
  4354. "defaults": {
  4355. "custom": {}
  4356. },
  4357. "overrides": []
  4358. },
  4359. "gridPos": {
  4360. "h": 9,
  4361. "w": 12,
  4362. "x": 0,
  4363. "y": 41
  4364. },
  4365. "heatmap": {},
  4366. "hideZeroBuckets": false,
  4367. "highlightCards": true,
  4368. "id": 164,
  4369. "legend": {
  4370. "show": false
  4371. },
  4372. "links": [],
  4373. "reverseYBuckets": false,
  4374. "targets": [
  4375. {
  4376. "expr": "sum(rate(synapse_federation_server_pdu_process_time_bucket{instance=\"$instance\"}[$bucket_size])) by (le)",
  4377. "format": "heatmap",
  4378. "instant": false,
  4379. "interval": "",
  4380. "intervalFactor": 1,
  4381. "legendFormat": "{{ le }}",
  4382. "refId": "A"
  4383. }
  4384. ],
  4385. "title": "Handle inbound PDU time",
  4386. "tooltip": {
  4387. "show": true,
  4388. "showHistogram": true
  4389. },
  4390. "tooltipDecimals": 2,
  4391. "type": "heatmap",
  4392. "xAxis": {
  4393. "show": true
  4394. },
  4395. "xBucketNumber": null,
  4396. "xBucketSize": null,
  4397. "yAxis": {
  4398. "decimals": 0,
  4399. "format": "s",
  4400. "logBase": 1,
  4401. "max": null,
  4402. "min": null,
  4403. "show": true,
  4404. "splitFactor": null
  4405. },
  4406. "yBucketBound": "auto",
  4407. "yBucketNumber": null,
  4408. "yBucketSize": null
  4409. },
  4410. {
  4411. "aliasColors": {},
  4412. "bars": false,
  4413. "dashLength": 10,
  4414. "dashes": false,
  4415. "datasource": "$datasource",
  4416. "editable": true,
  4417. "error": false,
  4418. "fieldConfig": {
  4419. "defaults": {
  4420. "custom": {},
  4421. "links": []
  4422. },
  4423. "overrides": []
  4424. },
  4425. "fill": 1,
  4426. "fillGradient": 0,
  4427. "grid": {},
  4428. "gridPos": {
  4429. "h": 9,
  4430. "w": 12,
  4431. "x": 12,
  4432. "y": 42
  4433. },
  4434. "hiddenSeries": false,
  4435. "id": 203,
  4436. "legend": {
  4437. "avg": false,
  4438. "current": false,
  4439. "max": false,
  4440. "min": false,
  4441. "show": true,
  4442. "total": false,
  4443. "values": false
  4444. },
  4445. "lines": true,
  4446. "linewidth": 2,
  4447. "links": [],
  4448. "nullPointMode": "null",
  4449. "options": {
  4450. "alertThreshold": true
  4451. },
  4452. "paceLength": 10,
  4453. "percentage": false,
  4454. "pluginVersion": "7.3.7",
  4455. "pointradius": 5,
  4456. "points": false,
  4457. "renderer": "flot",
  4458. "seriesOverrides": [],
  4459. "spaceLength": 10,
  4460. "stack": false,
  4461. "steppedLine": false,
  4462. "targets": [
  4463. {
  4464. "expr": "synapse_federation_server_oldest_inbound_pdu_in_staging{job=\"$job\",index=~\"$index\",instance=\"$instance\"}",
  4465. "format": "time_series",
  4466. "interval": "",
  4467. "intervalFactor": 1,
  4468. "legendFormat": "rss {{index}}",
  4469. "refId": "A",
  4470. "step": 4
  4471. }
  4472. ],
  4473. "thresholds": [],
  4474. "timeFrom": null,
  4475. "timeRegions": [],
  4476. "timeShift": null,
  4477. "title": "Age of oldest event in staging area",
  4478. "tooltip": {
  4479. "msResolution": false,
  4480. "shared": true,
  4481. "sort": 0,
  4482. "value_type": "cumulative"
  4483. },
  4484. "type": "graph",
  4485. "xaxis": {
  4486. "buckets": null,
  4487. "mode": "time",
  4488. "name": null,
  4489. "show": true,
  4490. "values": []
  4491. },
  4492. "yaxes": [
  4493. {
  4494. "format": "ms",
  4495. "label": null,
  4496. "logBase": 1,
  4497. "max": null,
  4498. "min": 0,
  4499. "show": true
  4500. },
  4501. {
  4502. "format": "short",
  4503. "label": null,
  4504. "logBase": 1,
  4505. "max": null,
  4506. "min": null,
  4507. "show": true
  4508. }
  4509. ],
  4510. "yaxis": {
  4511. "align": false,
  4512. "alignLevel": null
  4513. }
  4514. },
  4515. {
  4516. "aliasColors": {},
  4517. "bars": false,
  4518. "dashLength": 10,
  4519. "dashes": false,
  4520. "datasource": "$datasource",
  4521. "editable": true,
  4522. "error": false,
  4523. "fieldConfig": {
  4524. "defaults": {
  4525. "custom": {},
  4526. "links": []
  4527. },
  4528. "overrides": []
  4529. },
  4530. "fill": 1,
  4531. "fillGradient": 0,
  4532. "grid": {},
  4533. "gridPos": {
  4534. "h": 9,
  4535. "w": 12,
  4536. "x": 0,
  4537. "y": 50
  4538. },
  4539. "hiddenSeries": false,
  4540. "id": 202,
  4541. "legend": {
  4542. "avg": false,
  4543. "current": false,
  4544. "max": false,
  4545. "min": false,
  4546. "show": true,
  4547. "total": false,
  4548. "values": false
  4549. },
  4550. "lines": true,
  4551. "linewidth": 2,
  4552. "links": [],
  4553. "nullPointMode": "null",
  4554. "options": {
  4555. "alertThreshold": true
  4556. },
  4557. "paceLength": 10,
  4558. "percentage": false,
  4559. "pluginVersion": "7.3.7",
  4560. "pointradius": 5,
  4561. "points": false,
  4562. "renderer": "flot",
  4563. "seriesOverrides": [],
  4564. "spaceLength": 10,
  4565. "stack": false,
  4566. "steppedLine": false,
  4567. "targets": [
  4568. {
  4569. "expr": "synapse_federation_server_number_inbound_pdu_in_staging{job=\"$job\",index=~\"$index\",instance=\"$instance\"}",
  4570. "format": "time_series",
  4571. "interval": "",
  4572. "intervalFactor": 1,
  4573. "legendFormat": "rss {{index}}",
  4574. "refId": "A",
  4575. "step": 4
  4576. }
  4577. ],
  4578. "thresholds": [],
  4579. "timeFrom": null,
  4580. "timeRegions": [],
  4581. "timeShift": null,
  4582. "title": "Number of events in federation staging area",
  4583. "tooltip": {
  4584. "msResolution": false,
  4585. "shared": true,
  4586. "sort": 0,
  4587. "value_type": "cumulative"
  4588. },
  4589. "type": "graph",
  4590. "xaxis": {
  4591. "buckets": null,
  4592. "mode": "time",
  4593. "name": null,
  4594. "show": true,
  4595. "values": []
  4596. },
  4597. "yaxes": [
  4598. {
  4599. "format": "none",
  4600. "label": null,
  4601. "logBase": 1,
  4602. "max": null,
  4603. "min": 0,
  4604. "show": true
  4605. },
  4606. {
  4607. "format": "short",
  4608. "label": null,
  4609. "logBase": 1,
  4610. "max": null,
  4611. "min": null,
  4612. "show": true
  4613. }
  4614. ],
  4615. "yaxis": {
  4616. "align": false,
  4617. "alignLevel": null
  4618. }
  4619. },
  4620. {
  4621. "aliasColors": {},
  4622. "bars": false,
  4623. "dashLength": 10,
  4624. "dashes": false,
  4625. "datasource": "${DS_PROMETHEUS}",
  4626. "fieldConfig": {
  4627. "defaults": {
  4628. "custom": {}
  4629. },
  4630. "overrides": []
  4631. },
  4632. "fill": 1,
  4633. "fillGradient": 0,
  4634. "gridPos": {
  4635. "h": 8,
  4636. "w": 12,
  4637. "x": 12,
  4638. "y": 51
  4639. },
  4640. "hiddenSeries": false,
  4641. "id": 205,
  4642. "legend": {
  4643. "avg": false,
  4644. "current": false,
  4645. "max": false,
  4646. "min": false,
  4647. "show": true,
  4648. "total": false,
  4649. "values": false
  4650. },
  4651. "lines": true,
  4652. "linewidth": 1,
  4653. "nullPointMode": "null",
  4654. "options": {
  4655. "alertThreshold": true
  4656. },
  4657. "percentage": false,
  4658. "pluginVersion": "7.3.7",
  4659. "pointradius": 2,
  4660. "points": false,
  4661. "renderer": "flot",
  4662. "seriesOverrides": [],
  4663. "spaceLength": 10,
  4664. "stack": false,
  4665. "steppedLine": false,
  4666. "targets": [
  4667. {
  4668. "expr": "sum(rate(synapse_federation_soft_failed_events_total{instance=\"$instance\"}[$bucket_size]))",
  4669. "interval": "",
  4670. "legendFormat": "soft-failed events",
  4671. "refId": "A"
  4672. }
  4673. ],
  4674. "thresholds": [],
  4675. "timeFrom": null,
  4676. "timeRegions": [],
  4677. "timeShift": null,
  4678. "title": "Soft-failed event rate",
  4679. "tooltip": {
  4680. "shared": true,
  4681. "sort": 0,
  4682. "value_type": "individual"
  4683. },
  4684. "type": "graph",
  4685. "xaxis": {
  4686. "buckets": null,
  4687. "mode": "time",
  4688. "name": null,
  4689. "show": true,
  4690. "values": []
  4691. },
  4692. "yaxes": [
  4693. {
  4694. "format": "hertz",
  4695. "label": null,
  4696. "logBase": 1,
  4697. "max": null,
  4698. "min": null,
  4699. "show": true
  4700. },
  4701. {
  4702. "format": "short",
  4703. "label": null,
  4704. "logBase": 1,
  4705. "max": null,
  4706. "min": null,
  4707. "show": false
  4708. }
  4709. ],
  4710. "yaxis": {
  4711. "align": false,
  4712. "alignLevel": null
  4713. }
  4714. }
  4715. ],
  4716. "title": "Federation",
  4717. "type": "row"
  4718. },
  4719. {
  4720. "collapsed": true,
  4721. "datasource": "${DS_PROMETHEUS}",
  4722. "gridPos": {
  4723. "h": 1,
  4724. "w": 24,
  4725. "x": 0,
  4726. "y": 31
  4727. },
  4728. "id": 60,
  4729. "panels": [
  4730. {
  4731. "aliasColors": {},
  4732. "bars": false,
  4733. "dashLength": 10,
  4734. "dashes": false,
  4735. "datasource": "$datasource",
  4736. "fieldConfig": {
  4737. "defaults": {
  4738. "custom": {},
  4739. "links": []
  4740. },
  4741. "overrides": []
  4742. },
  4743. "fill": 1,
  4744. "fillGradient": 0,
  4745. "gridPos": {
  4746. "h": 8,
  4747. "w": 12,
  4748. "x": 0,
  4749. "y": 34
  4750. },
  4751. "hiddenSeries": false,
  4752. "id": 51,
  4753. "legend": {
  4754. "avg": false,
  4755. "current": false,
  4756. "max": false,
  4757. "min": false,
  4758. "show": true,
  4759. "total": false,
  4760. "values": false
  4761. },
  4762. "lines": true,
  4763. "linewidth": 1,
  4764. "links": [],
  4765. "nullPointMode": "null",
  4766. "paceLength": 10,
  4767. "percentage": false,
  4768. "pluginVersion": "7.1.3",
  4769. "pointradius": 5,
  4770. "points": false,
  4771. "renderer": "flot",
  4772. "seriesOverrides": [],
  4773. "spaceLength": 10,
  4774. "stack": false,
  4775. "steppedLine": false,
  4776. "targets": [
  4777. {
  4778. "expr": "rate(synapse_http_httppusher_http_pushes_processed{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed + synapse_http_httppusher_http_pushes_processed) > 0",
  4779. "format": "time_series",
  4780. "interval": "",
  4781. "intervalFactor": 2,
  4782. "legendFormat": "processed {{job}}",
  4783. "refId": "A",
  4784. "step": 20
  4785. },
  4786. {
  4787. "expr": "rate(synapse_http_httppusher_http_pushes_failed{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed + synapse_http_httppusher_http_pushes_processed) > 0",
  4788. "format": "time_series",
  4789. "intervalFactor": 2,
  4790. "legendFormat": "failed {{job}}",
  4791. "refId": "B",
  4792. "step": 20
  4793. }
  4794. ],
  4795. "thresholds": [],
  4796. "timeFrom": null,
  4797. "timeRegions": [],
  4798. "timeShift": null,
  4799. "title": "HTTP Push rate",
  4800. "tooltip": {
  4801. "shared": true,
  4802. "sort": 0,
  4803. "value_type": "individual"
  4804. },
  4805. "type": "graph",
  4806. "xaxis": {
  4807. "buckets": null,
  4808. "mode": "time",
  4809. "name": null,
  4810. "show": true,
  4811. "values": []
  4812. },
  4813. "yaxes": [
  4814. {
  4815. "format": "hertz",
  4816. "label": null,
  4817. "logBase": 1,
  4818. "max": null,
  4819. "min": null,
  4820. "show": true
  4821. },
  4822. {
  4823. "format": "short",
  4824. "label": null,
  4825. "logBase": 1,
  4826. "max": null,
  4827. "min": null,
  4828. "show": true
  4829. }
  4830. ],
  4831. "yaxis": {
  4832. "align": false,
  4833. "alignLevel": null
  4834. }
  4835. },
  4836. {
  4837. "aliasColors": {},
  4838. "bars": false,
  4839. "dashLength": 10,
  4840. "dashes": false,
  4841. "datasource": "$datasource",
  4842. "description": "",
  4843. "fieldConfig": {
  4844. "defaults": {
  4845. "custom": {},
  4846. "links": []
  4847. },
  4848. "overrides": []
  4849. },
  4850. "fill": 1,
  4851. "fillGradient": 0,
  4852. "gridPos": {
  4853. "h": 8,
  4854. "w": 12,
  4855. "x": 12,
  4856. "y": 34
  4857. },
  4858. "hiddenSeries": false,
  4859. "id": 134,
  4860. "legend": {
  4861. "avg": false,
  4862. "current": false,
  4863. "hideZero": false,
  4864. "max": false,
  4865. "min": false,
  4866. "show": true,
  4867. "total": false,
  4868. "values": false
  4869. },
  4870. "lines": true,
  4871. "linewidth": 1,
  4872. "nullPointMode": "null",
  4873. "percentage": false,
  4874. "pluginVersion": "7.1.3",
  4875. "pointradius": 2,
  4876. "points": false,
  4877. "renderer": "flot",
  4878. "seriesOverrides": [],
  4879. "spaceLength": 10,
  4880. "stack": false,
  4881. "steppedLine": false,
  4882. "targets": [
  4883. {
  4884. "expr": "topk(10,synapse_pushers{job=~\"$job\",index=~\"$index\", instance=\"$instance\"})",
  4885. "legendFormat": "{{kind}} {{app_id}}",
  4886. "refId": "A"
  4887. }
  4888. ],
  4889. "thresholds": [],
  4890. "timeFrom": null,
  4891. "timeRegions": [],
  4892. "timeShift": null,
  4893. "title": "Active pusher instances by app",
  4894. "tooltip": {
  4895. "shared": false,
  4896. "sort": 2,
  4897. "value_type": "individual"
  4898. },
  4899. "type": "graph",
  4900. "xaxis": {
  4901. "buckets": null,
  4902. "mode": "time",
  4903. "name": null,
  4904. "show": true,
  4905. "values": []
  4906. },
  4907. "yaxes": [
  4908. {
  4909. "format": "short",
  4910. "label": null,
  4911. "logBase": 1,
  4912. "max": null,
  4913. "min": null,
  4914. "show": true
  4915. },
  4916. {
  4917. "format": "short",
  4918. "label": null,
  4919. "logBase": 1,
  4920. "max": null,
  4921. "min": null,
  4922. "show": true
  4923. }
  4924. ],
  4925. "yaxis": {
  4926. "align": false,
  4927. "alignLevel": null
  4928. }
  4929. }
  4930. ],
  4931. "repeat": null,
  4932. "title": "Pushes",
  4933. "type": "row"
  4934. },
  4935. {
  4936. "collapsed": true,
  4937. "datasource": "${DS_PROMETHEUS}",
  4938. "gridPos": {
  4939. "h": 1,
  4940. "w": 24,
  4941. "x": 0,
  4942. "y": 32
  4943. },
  4944. "id": 58,
  4945. "panels": [
  4946. {
  4947. "aliasColors": {},
  4948. "bars": false,
  4949. "dashLength": 10,
  4950. "dashes": false,
  4951. "datasource": "$datasource",
  4952. "fieldConfig": {
  4953. "defaults": {
  4954. "custom": {},
  4955. "links": []
  4956. },
  4957. "overrides": []
  4958. },
  4959. "fill": 1,
  4960. "fillGradient": 0,
  4961. "gridPos": {
  4962. "h": 7,
  4963. "w": 12,
  4964. "x": 0,
  4965. "y": 33
  4966. },
  4967. "hiddenSeries": false,
  4968. "id": 48,
  4969. "legend": {
  4970. "avg": false,
  4971. "current": false,
  4972. "max": false,
  4973. "min": false,
  4974. "show": true,
  4975. "total": false,
  4976. "values": false
  4977. },
  4978. "lines": true,
  4979. "linewidth": 1,
  4980. "links": [],
  4981. "nullPointMode": "null",
  4982. "options": {
  4983. "alertThreshold": true
  4984. },
  4985. "paceLength": 10,
  4986. "percentage": false,
  4987. "pluginVersion": "7.3.7",
  4988. "pointradius": 5,
  4989. "points": false,
  4990. "renderer": "flot",
  4991. "seriesOverrides": [],
  4992. "spaceLength": 10,
  4993. "stack": false,
  4994. "steppedLine": false,
  4995. "targets": [
  4996. {
  4997. "expr": "rate(synapse_storage_schedule_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_schedule_time_count[$bucket_size])",
  4998. "format": "time_series",
  4999. "intervalFactor": 2,
  5000. "legendFormat": "{{job}}-{{index}}",
  5001. "refId": "A",
  5002. "step": 20
  5003. }
  5004. ],
  5005. "thresholds": [],
  5006. "timeFrom": null,
  5007. "timeRegions": [],
  5008. "timeShift": null,
  5009. "title": "Avg time waiting for db conn",
  5010. "tooltip": {
  5011. "shared": true,
  5012. "sort": 0,
  5013. "value_type": "individual"
  5014. },
  5015. "type": "graph",
  5016. "xaxis": {
  5017. "buckets": null,
  5018. "mode": "time",
  5019. "name": null,
  5020. "show": true,
  5021. "values": []
  5022. },
  5023. "yaxes": [
  5024. {
  5025. "decimals": null,
  5026. "format": "s",
  5027. "label": "",
  5028. "logBase": 1,
  5029. "max": null,
  5030. "min": "0",
  5031. "show": true
  5032. },
  5033. {
  5034. "format": "short",
  5035. "label": null,
  5036. "logBase": 1,
  5037. "max": null,
  5038. "min": null,
  5039. "show": false
  5040. }
  5041. ],
  5042. "yaxis": {
  5043. "align": false,
  5044. "alignLevel": null
  5045. }
  5046. },
  5047. {
  5048. "aliasColors": {},
  5049. "bars": false,
  5050. "dashLength": 10,
  5051. "dashes": false,
  5052. "datasource": "$datasource",
  5053. "description": "Shows the time in which the given percentage of database queries were scheduled, over the sampled timespan",
  5054. "fieldConfig": {
  5055. "defaults": {
  5056. "custom": {},
  5057. "links": []
  5058. },
  5059. "overrides": []
  5060. },
  5061. "fill": 1,
  5062. "fillGradient": 0,
  5063. "gridPos": {
  5064. "h": 7,
  5065. "w": 12,
  5066. "x": 12,
  5067. "y": 33
  5068. },
  5069. "hiddenSeries": false,
  5070. "id": 104,
  5071. "legend": {
  5072. "alignAsTable": true,
  5073. "avg": false,
  5074. "current": false,
  5075. "max": false,
  5076. "min": false,
  5077. "show": true,
  5078. "total": false,
  5079. "values": false
  5080. },
  5081. "lines": true,
  5082. "linewidth": 1,
  5083. "links": [],
  5084. "nullPointMode": "null",
  5085. "options": {
  5086. "alertThreshold": true
  5087. },
  5088. "paceLength": 10,
  5089. "percentage": false,
  5090. "pluginVersion": "7.3.7",
  5091. "pointradius": 5,
  5092. "points": false,
  5093. "renderer": "flot",
  5094. "seriesOverrides": [],
  5095. "spaceLength": 10,
  5096. "stack": false,
  5097. "steppedLine": false,
  5098. "targets": [
  5099. {
  5100. "expr": "histogram_quantile(0.99, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
  5101. "format": "time_series",
  5102. "hide": false,
  5103. "intervalFactor": 1,
  5104. "legendFormat": "{{job}} {{index}} 99%",
  5105. "refId": "A",
  5106. "step": 20
  5107. },
  5108. {
  5109. "expr": "histogram_quantile(0.95, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
  5110. "format": "time_series",
  5111. "intervalFactor": 1,
  5112. "legendFormat": "{{job}} {{index}} 95%",
  5113. "refId": "B"
  5114. },
  5115. {
  5116. "expr": "histogram_quantile(0.90, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
  5117. "format": "time_series",
  5118. "intervalFactor": 1,
  5119. "legendFormat": "{{job}} {{index}} 90%",
  5120. "refId": "C"
  5121. },
  5122. {
  5123. "expr": "rate(synapse_storage_schedule_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_schedule_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
  5124. "format": "time_series",
  5125. "interval": "",
  5126. "intervalFactor": 1,
  5127. "legendFormat": "{{job}} {{index}} mean",
  5128. "refId": "D"
  5129. }
  5130. ],
  5131. "thresholds": [],
  5132. "timeFrom": null,
  5133. "timeRegions": [],
  5134. "timeShift": null,
  5135. "title": "Db scheduling time quantiles",
  5136. "tooltip": {
  5137. "shared": false,
  5138. "sort": 0,
  5139. "value_type": "individual"
  5140. },
  5141. "type": "graph",
  5142. "xaxis": {
  5143. "buckets": null,
  5144. "mode": "time",
  5145. "name": null,
  5146. "show": true,
  5147. "values": []
  5148. },
  5149. "yaxes": [
  5150. {
  5151. "decimals": null,
  5152. "format": "s",
  5153. "label": "",
  5154. "logBase": 1,
  5155. "max": null,
  5156. "min": "0",
  5157. "show": true
  5158. },
  5159. {
  5160. "format": "short",
  5161. "label": null,
  5162. "logBase": 1,
  5163. "max": null,
  5164. "min": null,
  5165. "show": false
  5166. }
  5167. ],
  5168. "yaxis": {
  5169. "align": false,
  5170. "alignLevel": null
  5171. }
  5172. },
  5173. {
  5174. "aliasColors": {},
  5175. "bars": false,
  5176. "dashLength": 10,
  5177. "dashes": false,
  5178. "datasource": "$datasource",
  5179. "editable": true,
  5180. "error": false,
  5181. "fieldConfig": {
  5182. "defaults": {
  5183. "custom": {},
  5184. "links": []
  5185. },
  5186. "overrides": []
  5187. },
  5188. "fill": 0,
  5189. "fillGradient": 0,
  5190. "grid": {},
  5191. "gridPos": {
  5192. "h": 7,
  5193. "w": 12,
  5194. "x": 0,
  5195. "y": 40
  5196. },
  5197. "hiddenSeries": false,
  5198. "id": 10,
  5199. "legend": {
  5200. "avg": false,
  5201. "current": false,
  5202. "hideEmpty": true,
  5203. "hideZero": true,
  5204. "max": false,
  5205. "min": false,
  5206. "show": true,
  5207. "total": false,
  5208. "values": false
  5209. },
  5210. "lines": true,
  5211. "linewidth": 2,
  5212. "links": [],
  5213. "nullPointMode": "null",
  5214. "options": {
  5215. "alertThreshold": true
  5216. },
  5217. "paceLength": 10,
  5218. "percentage": false,
  5219. "pluginVersion": "7.3.7",
  5220. "pointradius": 5,
  5221. "points": false,
  5222. "renderer": "flot",
  5223. "seriesOverrides": [],
  5224. "spaceLength": 10,
  5225. "stack": false,
  5226. "steppedLine": false,
  5227. "targets": [
  5228. {
  5229. "expr": "topk(10, rate(synapse_storage_transaction_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
  5230. "format": "time_series",
  5231. "interval": "",
  5232. "intervalFactor": 2,
  5233. "legendFormat": "{{job}}-{{index}} {{desc}}",
  5234. "refId": "A",
  5235. "step": 20
  5236. }
  5237. ],
  5238. "thresholds": [],
  5239. "timeFrom": null,
  5240. "timeRegions": [],
  5241. "timeShift": null,
  5242. "title": "Top DB transactions by txn rate",
  5243. "tooltip": {
  5244. "shared": false,
  5245. "sort": 0,
  5246. "value_type": "cumulative"
  5247. },
  5248. "type": "graph",
  5249. "xaxis": {
  5250. "buckets": null,
  5251. "mode": "time",
  5252. "name": null,
  5253. "show": true,
  5254. "values": []
  5255. },
  5256. "yaxes": [
  5257. {
  5258. "format": "hertz",
  5259. "logBase": 1,
  5260. "max": null,
  5261. "min": 0,
  5262. "show": true
  5263. },
  5264. {
  5265. "format": "short",
  5266. "logBase": 1,
  5267. "max": null,
  5268. "min": null,
  5269. "show": true
  5270. }
  5271. ],
  5272. "yaxis": {
  5273. "align": false,
  5274. "alignLevel": null
  5275. }
  5276. },
  5277. {
  5278. "aliasColors": {},
  5279. "bars": false,
  5280. "dashLength": 10,
  5281. "dashes": false,
  5282. "datasource": "$datasource",
  5283. "editable": true,
  5284. "error": false,
  5285. "fieldConfig": {
  5286. "defaults": {
  5287. "custom": {},
  5288. "links": []
  5289. },
  5290. "overrides": []
  5291. },
  5292. "fill": 1,
  5293. "fillGradient": 0,
  5294. "grid": {},
  5295. "gridPos": {
  5296. "h": 7,
  5297. "w": 12,
  5298. "x": 12,
  5299. "y": 40
  5300. },
  5301. "hiddenSeries": false,
  5302. "id": 11,
  5303. "legend": {
  5304. "avg": false,
  5305. "current": false,
  5306. "hideEmpty": true,
  5307. "hideZero": true,
  5308. "max": false,
  5309. "min": false,
  5310. "show": true,
  5311. "total": false,
  5312. "values": false
  5313. },
  5314. "lines": true,
  5315. "linewidth": 1,
  5316. "links": [],
  5317. "nullPointMode": "null",
  5318. "options": {
  5319. "alertThreshold": true
  5320. },
  5321. "paceLength": 10,
  5322. "percentage": false,
  5323. "pluginVersion": "7.3.7",
  5324. "pointradius": 5,
  5325. "points": false,
  5326. "renderer": "flot",
  5327. "seriesOverrides": [],
  5328. "spaceLength": 10,
  5329. "stack": false,
  5330. "steppedLine": false,
  5331. "targets": [
  5332. {
  5333. "expr": "rate(synapse_storage_transaction_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
  5334. "format": "time_series",
  5335. "instant": false,
  5336. "interval": "",
  5337. "intervalFactor": 1,
  5338. "legendFormat": "{{job}}-{{index}} {{desc}}",
  5339. "refId": "A",
  5340. "step": 20
  5341. }
  5342. ],
  5343. "thresholds": [],
  5344. "timeFrom": null,
  5345. "timeRegions": [],
  5346. "timeShift": null,
  5347. "title": "DB transactions by total txn time",
  5348. "tooltip": {
  5349. "shared": false,
  5350. "sort": 0,
  5351. "value_type": "cumulative"
  5352. },
  5353. "type": "graph",
  5354. "xaxis": {
  5355. "buckets": null,
  5356. "mode": "time",
  5357. "name": null,
  5358. "show": true,
  5359. "values": []
  5360. },
  5361. "yaxes": [
  5362. {
  5363. "format": "percentunit",
  5364. "logBase": 1,
  5365. "max": null,
  5366. "min": null,
  5367. "show": true
  5368. },
  5369. {
  5370. "format": "short",
  5371. "logBase": 1,
  5372. "max": null,
  5373. "min": null,
  5374. "show": true
  5375. }
  5376. ],
  5377. "yaxis": {
  5378. "align": false,
  5379. "alignLevel": null
  5380. }
  5381. },
  5382. {
  5383. "aliasColors": {},
  5384. "bars": false,
  5385. "dashLength": 10,
  5386. "dashes": false,
  5387. "datasource": "$datasource",
  5388. "editable": true,
  5389. "error": false,
  5390. "fieldConfig": {
  5391. "defaults": {
  5392. "custom": {},
  5393. "links": []
  5394. },
  5395. "overrides": []
  5396. },
  5397. "fill": 0,
  5398. "fillGradient": 0,
  5399. "grid": {},
  5400. "gridPos": {
  5401. "h": 7,
  5402. "w": 12,
  5403. "x": 0,
  5404. "y": 47
  5405. },
  5406. "hiddenSeries": false,
  5407. "id": 180,
  5408. "legend": {
  5409. "avg": false,
  5410. "current": false,
  5411. "hideEmpty": true,
  5412. "hideZero": true,
  5413. "max": false,
  5414. "min": false,
  5415. "show": true,
  5416. "total": false,
  5417. "values": false
  5418. },
  5419. "lines": true,
  5420. "linewidth": 1,
  5421. "links": [],
  5422. "nullPointMode": "null",
  5423. "options": {
  5424. "alertThreshold": false
  5425. },
  5426. "paceLength": 10,
  5427. "percentage": false,
  5428. "pluginVersion": "7.3.7",
  5429. "pointradius": 5,
  5430. "points": false,
  5431. "renderer": "flot",
  5432. "seriesOverrides": [],
  5433. "spaceLength": 10,
  5434. "stack": false,
  5435. "steppedLine": false,
  5436. "targets": [
  5437. {
  5438. "expr": "rate(synapse_storage_transaction_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_transaction_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
  5439. "format": "time_series",
  5440. "instant": false,
  5441. "interval": "",
  5442. "intervalFactor": 1,
  5443. "legendFormat": "{{job}}-{{index}} {{desc}}",
  5444. "refId": "A",
  5445. "step": 20
  5446. }
  5447. ],
  5448. "thresholds": [],
  5449. "timeFrom": null,
  5450. "timeRegions": [],
  5451. "timeShift": null,
  5452. "title": "Average DB txn time",
  5453. "tooltip": {
  5454. "shared": false,
  5455. "sort": 0,
  5456. "value_type": "cumulative"
  5457. },
  5458. "type": "graph",
  5459. "xaxis": {
  5460. "buckets": null,
  5461. "mode": "time",
  5462. "name": null,
  5463. "show": true,
  5464. "values": []
  5465. },
  5466. "yaxes": [
  5467. {
  5468. "format": "s",
  5469. "logBase": 1,
  5470. "max": null,
  5471. "min": null,
  5472. "show": true
  5473. },
  5474. {
  5475. "format": "short",
  5476. "logBase": 1,
  5477. "max": null,
  5478. "min": null,
  5479. "show": true
  5480. }
  5481. ],
  5482. "yaxis": {
  5483. "align": false,
  5484. "alignLevel": null
  5485. }
  5486. },
  5487. {
  5488. "aliasColors": {},
  5489. "bars": false,
  5490. "dashLength": 10,
  5491. "dashes": false,
  5492. "datasource": "$datasource",
  5493. "fieldConfig": {
  5494. "defaults": {
  5495. "custom": {},
  5496. "links": []
  5497. },
  5498. "overrides": []
  5499. },
  5500. "fill": 6,
  5501. "fillGradient": 0,
  5502. "gridPos": {
  5503. "h": 9,
  5504. "w": 12,
  5505. "x": 12,
  5506. "y": 47
  5507. },
  5508. "hiddenSeries": false,
  5509. "id": 200,
  5510. "legend": {
  5511. "avg": false,
  5512. "current": false,
  5513. "max": false,
  5514. "min": false,
  5515. "show": true,
  5516. "total": false,
  5517. "values": false
  5518. },
  5519. "lines": true,
  5520. "linewidth": 1,
  5521. "links": [],
  5522. "nullPointMode": "null",
  5523. "options": {
  5524. "alertThreshold": true
  5525. },
  5526. "percentage": false,
  5527. "pluginVersion": "7.3.7",
  5528. "pointradius": 5,
  5529. "points": false,
  5530. "renderer": "flot",
  5531. "seriesOverrides": [],
  5532. "spaceLength": 10,
  5533. "stack": false,
  5534. "steppedLine": false,
  5535. "targets": [
  5536. {
  5537. "expr": "histogram_quantile(0.99, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
  5538. "format": "time_series",
  5539. "intervalFactor": 1,
  5540. "legendFormat": "99%",
  5541. "refId": "D"
  5542. },
  5543. {
  5544. "expr": "histogram_quantile(0.9, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
  5545. "format": "time_series",
  5546. "intervalFactor": 1,
  5547. "legendFormat": "90%",
  5548. "refId": "A"
  5549. },
  5550. {
  5551. "expr": "histogram_quantile(0.75, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
  5552. "format": "time_series",
  5553. "intervalFactor": 1,
  5554. "legendFormat": "75%",
  5555. "refId": "C"
  5556. },
  5557. {
  5558. "expr": "histogram_quantile(0.5, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
  5559. "format": "time_series",
  5560. "intervalFactor": 1,
  5561. "legendFormat": "50%",
  5562. "refId": "B"
  5563. }
  5564. ],
  5565. "thresholds": [],
  5566. "timeFrom": null,
  5567. "timeRegions": [],
  5568. "timeShift": null,
  5569. "title": "Time waiting for DB connection quantiles",
  5570. "tooltip": {
  5571. "shared": true,
  5572. "sort": 2,
  5573. "value_type": "individual"
  5574. },
  5575. "type": "graph",
  5576. "xaxis": {
  5577. "buckets": null,
  5578. "mode": "time",
  5579. "name": null,
  5580. "show": true,
  5581. "values": []
  5582. },
  5583. "yaxes": [
  5584. {
  5585. "decimals": null,
  5586. "format": "s",
  5587. "label": "",
  5588. "logBase": 1,
  5589. "max": null,
  5590. "min": "0",
  5591. "show": true
  5592. },
  5593. {
  5594. "format": "short",
  5595. "label": null,
  5596. "logBase": 1,
  5597. "max": null,
  5598. "min": null,
  5599. "show": false
  5600. }
  5601. ],
  5602. "yaxis": {
  5603. "align": false,
  5604. "alignLevel": null
  5605. }
  5606. }
  5607. ],
  5608. "repeat": null,
  5609. "title": "Database",
  5610. "type": "row"
  5611. },
  5612. {
  5613. "collapsed": true,
  5614. "datasource": "${DS_PROMETHEUS}",
  5615. "gridPos": {
  5616. "h": 1,
  5617. "w": 24,
  5618. "x": 0,
  5619. "y": 33
  5620. },
  5621. "id": 59,
  5622. "panels": [
  5623. {
  5624. "aliasColors": {},
  5625. "bars": false,
  5626. "dashLength": 10,
  5627. "dashes": false,
  5628. "datasource": "$datasource",
  5629. "editable": true,
  5630. "error": false,
  5631. "fieldConfig": {
  5632. "defaults": {
  5633. "custom": {},
  5634. "links": []
  5635. },
  5636. "overrides": []
  5637. },
  5638. "fill": 1,
  5639. "fillGradient": 0,
  5640. "grid": {},
  5641. "gridPos": {
  5642. "h": 13,
  5643. "w": 12,
  5644. "x": 0,
  5645. "y": 9
  5646. },
  5647. "hiddenSeries": false,
  5648. "id": 12,
  5649. "legend": {
  5650. "alignAsTable": true,
  5651. "avg": false,
  5652. "current": false,
  5653. "max": false,
  5654. "min": false,
  5655. "show": true,
  5656. "total": false,
  5657. "values": false
  5658. },
  5659. "lines": true,
  5660. "linewidth": 2,
  5661. "links": [],
  5662. "nullPointMode": "null",
  5663. "paceLength": 10,
  5664. "percentage": false,
  5665. "pluginVersion": "7.1.3",
  5666. "pointradius": 5,
  5667. "points": false,
  5668. "renderer": "flot",
  5669. "seriesOverrides": [],
  5670. "spaceLength": 10,
  5671. "stack": false,
  5672. "steppedLine": false,
  5673. "targets": [
  5674. {
  5675. "expr": "rate(synapse_util_metrics_block_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\",block_name!=\"wrapped_request_handler\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds[$bucket_size])",
  5676. "format": "time_series",
  5677. "interval": "",
  5678. "intervalFactor": 2,
  5679. "legendFormat": "{{job}}-{{index}} {{block_name}}",
  5680. "refId": "A",
  5681. "step": 20
  5682. }
  5683. ],
  5684. "thresholds": [],
  5685. "timeFrom": null,
  5686. "timeRegions": [],
  5687. "timeShift": null,
  5688. "title": "Total CPU Usage by Block",
  5689. "tooltip": {
  5690. "shared": true,
  5691. "sort": 2,
  5692. "value_type": "cumulative"
  5693. },
  5694. "type": "graph",
  5695. "xaxis": {
  5696. "buckets": null,
  5697. "mode": "time",
  5698. "name": null,
  5699. "show": true,
  5700. "values": []
  5701. },
  5702. "yaxes": [
  5703. {
  5704. "format": "percentunit",
  5705. "logBase": 1,
  5706. "max": null,
  5707. "min": null,
  5708. "show": true
  5709. },
  5710. {
  5711. "format": "short",
  5712. "logBase": 1,
  5713. "max": null,
  5714. "min": null,
  5715. "show": true
  5716. }
  5717. ],
  5718. "yaxis": {
  5719. "align": false,
  5720. "alignLevel": null
  5721. }
  5722. },
  5723. {
  5724. "aliasColors": {},
  5725. "bars": false,
  5726. "dashLength": 10,
  5727. "dashes": false,
  5728. "datasource": "$datasource",
  5729. "editable": true,
  5730. "error": false,
  5731. "fieldConfig": {
  5732. "defaults": {
  5733. "custom": {},
  5734. "links": []
  5735. },
  5736. "overrides": []
  5737. },
  5738. "fill": 1,
  5739. "fillGradient": 0,
  5740. "grid": {},
  5741. "gridPos": {
  5742. "h": 13,
  5743. "w": 12,
  5744. "x": 12,
  5745. "y": 9
  5746. },
  5747. "hiddenSeries": false,
  5748. "id": 26,
  5749. "legend": {
  5750. "alignAsTable": true,
  5751. "avg": false,
  5752. "current": false,
  5753. "max": false,
  5754. "min": false,
  5755. "show": true,
  5756. "total": false,
  5757. "values": false
  5758. },
  5759. "lines": true,
  5760. "linewidth": 2,
  5761. "links": [],
  5762. "nullPointMode": "null",
  5763. "paceLength": 10,
  5764. "percentage": false,
  5765. "pluginVersion": "7.1.3",
  5766. "pointradius": 5,
  5767. "points": false,
  5768. "renderer": "flot",
  5769. "seriesOverrides": [],
  5770. "spaceLength": 10,
  5771. "stack": false,
  5772. "steppedLine": false,
  5773. "targets": [
  5774. {
  5775. "expr": "(rate(synapse_util_metrics_block_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds[$bucket_size])) / rate(synapse_util_metrics_block_count[$bucket_size])",
  5776. "format": "time_series",
  5777. "interval": "",
  5778. "intervalFactor": 2,
  5779. "legendFormat": "{{job}}-{{index}} {{block_name}}",
  5780. "refId": "A",
  5781. "step": 20
  5782. }
  5783. ],
  5784. "thresholds": [],
  5785. "timeFrom": null,
  5786. "timeRegions": [],
  5787. "timeShift": null,
  5788. "title": "Average CPU Time per Block",
  5789. "tooltip": {
  5790. "shared": true,
  5791. "sort": 2,
  5792. "value_type": "cumulative"
  5793. },
  5794. "type": "graph",
  5795. "xaxis": {
  5796. "buckets": null,
  5797. "mode": "time",
  5798. "name": null,
  5799. "show": true,
  5800. "values": []
  5801. },
  5802. "yaxes": [
  5803. {
  5804. "format": "ms",
  5805. "logBase": 1,
  5806. "max": null,
  5807. "min": null,
  5808. "show": true
  5809. },
  5810. {
  5811. "format": "short",
  5812. "logBase": 1,
  5813. "max": null,
  5814. "min": null,
  5815. "show": true
  5816. }
  5817. ],
  5818. "yaxis": {
  5819. "align": false,
  5820. "alignLevel": null
  5821. }
  5822. },
  5823. {
  5824. "aliasColors": {},
  5825. "bars": false,
  5826. "dashLength": 10,
  5827. "dashes": false,
  5828. "datasource": "$datasource",
  5829. "editable": true,
  5830. "error": false,
  5831. "fieldConfig": {
  5832. "defaults": {
  5833. "custom": {},
  5834. "links": []
  5835. },
  5836. "overrides": []
  5837. },
  5838. "fill": 1,
  5839. "fillGradient": 0,
  5840. "grid": {},
  5841. "gridPos": {
  5842. "h": 13,
  5843. "w": 12,
  5844. "x": 0,
  5845. "y": 22
  5846. },
  5847. "hiddenSeries": false,
  5848. "id": 13,
  5849. "legend": {
  5850. "alignAsTable": true,
  5851. "avg": false,
  5852. "current": false,
  5853. "max": false,
  5854. "min": false,
  5855. "show": true,
  5856. "total": false,
  5857. "values": false
  5858. },
  5859. "lines": true,
  5860. "linewidth": 2,
  5861. "links": [],
  5862. "nullPointMode": "null",
  5863. "paceLength": 10,
  5864. "percentage": false,
  5865. "pluginVersion": "7.1.3",
  5866. "pointradius": 5,
  5867. "points": false,
  5868. "renderer": "flot",
  5869. "seriesOverrides": [],
  5870. "spaceLength": 10,
  5871. "stack": false,
  5872. "steppedLine": false,
  5873. "targets": [
  5874. {
  5875. "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
  5876. "format": "time_series",
  5877. "interval": "",
  5878. "intervalFactor": 2,
  5879. "legendFormat": "{{job}} {{block_name}}",
  5880. "refId": "A",
  5881. "step": 20
  5882. }
  5883. ],
  5884. "thresholds": [],
  5885. "timeFrom": null,
  5886. "timeRegions": [],
  5887. "timeShift": null,
  5888. "title": "Total DB Usage by Block",
  5889. "tooltip": {
  5890. "shared": true,
  5891. "sort": 2,
  5892. "value_type": "cumulative"
  5893. },
  5894. "type": "graph",
  5895. "xaxis": {
  5896. "buckets": null,
  5897. "mode": "time",
  5898. "name": null,
  5899. "show": true,
  5900. "values": []
  5901. },
  5902. "yaxes": [
  5903. {
  5904. "format": "percentunit",
  5905. "logBase": 1,
  5906. "max": null,
  5907. "min": 0,
  5908. "show": true
  5909. },
  5910. {
  5911. "format": "short",
  5912. "logBase": 1,
  5913. "max": null,
  5914. "min": null,
  5915. "show": true
  5916. }
  5917. ],
  5918. "yaxis": {
  5919. "align": false,
  5920. "alignLevel": null
  5921. }
  5922. },
  5923. {
  5924. "aliasColors": {},
  5925. "bars": false,
  5926. "dashLength": 10,
  5927. "dashes": false,
  5928. "datasource": "$datasource",
  5929. "description": "The time each database transaction takes to execute, on average, broken down by metrics block.",
  5930. "editable": true,
  5931. "error": false,
  5932. "fieldConfig": {
  5933. "defaults": {
  5934. "custom": {},
  5935. "links": []
  5936. },
  5937. "overrides": []
  5938. },
  5939. "fill": 1,
  5940. "fillGradient": 0,
  5941. "grid": {},
  5942. "gridPos": {
  5943. "h": 13,
  5944. "w": 12,
  5945. "x": 12,
  5946. "y": 22
  5947. },
  5948. "hiddenSeries": false,
  5949. "id": 27,
  5950. "legend": {
  5951. "alignAsTable": true,
  5952. "avg": false,
  5953. "current": false,
  5954. "max": false,
  5955. "min": false,
  5956. "show": true,
  5957. "total": false,
  5958. "values": false
  5959. },
  5960. "lines": true,
  5961. "linewidth": 2,
  5962. "links": [],
  5963. "nullPointMode": "null",
  5964. "paceLength": 10,
  5965. "percentage": false,
  5966. "pluginVersion": "7.1.3",
  5967. "pointradius": 5,
  5968. "points": false,
  5969. "renderer": "flot",
  5970. "seriesOverrides": [],
  5971. "spaceLength": 10,
  5972. "stack": false,
  5973. "steppedLine": false,
  5974. "targets": [
  5975. {
  5976. "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
  5977. "format": "time_series",
  5978. "interval": "",
  5979. "intervalFactor": 2,
  5980. "legendFormat": "{{job}}-{{index}} {{block_name}}",
  5981. "refId": "A",
  5982. "step": 20
  5983. }
  5984. ],
  5985. "thresholds": [],
  5986. "timeFrom": null,
  5987. "timeRegions": [],
  5988. "timeShift": null,
  5989. "title": "Average Database Transaction time, by Block",
  5990. "tooltip": {
  5991. "shared": true,
  5992. "sort": 2,
  5993. "value_type": "cumulative"
  5994. },
  5995. "type": "graph",
  5996. "xaxis": {
  5997. "buckets": null,
  5998. "mode": "time",
  5999. "name": null,
  6000. "show": true,
  6001. "values": []
  6002. },
  6003. "yaxes": [
  6004. {
  6005. "format": "ms",
  6006. "logBase": 1,
  6007. "max": null,
  6008. "min": null,
  6009. "show": true
  6010. },
  6011. {
  6012. "format": "short",
  6013. "logBase": 1,
  6014. "max": null,
  6015. "min": null,
  6016. "show": true
  6017. }
  6018. ],
  6019. "yaxis": {
  6020. "align": false,
  6021. "alignLevel": null
  6022. }
  6023. },
  6024. {
  6025. "aliasColors": {},
  6026. "bars": false,
  6027. "dashLength": 10,
  6028. "dashes": false,
  6029. "datasource": "$datasource",
  6030. "editable": true,
  6031. "error": false,
  6032. "fieldConfig": {
  6033. "defaults": {
  6034. "custom": {},
  6035. "links": []
  6036. },
  6037. "overrides": []
  6038. },
  6039. "fill": 1,
  6040. "fillGradient": 0,
  6041. "grid": {},
  6042. "gridPos": {
  6043. "h": 13,
  6044. "w": 12,
  6045. "x": 0,
  6046. "y": 35
  6047. },
  6048. "hiddenSeries": false,
  6049. "id": 28,
  6050. "legend": {
  6051. "avg": false,
  6052. "current": false,
  6053. "max": false,
  6054. "min": false,
  6055. "show": false,
  6056. "total": false,
  6057. "values": false
  6058. },
  6059. "lines": true,
  6060. "linewidth": 2,
  6061. "links": [],
  6062. "nullPointMode": "null",
  6063. "paceLength": 10,
  6064. "percentage": false,
  6065. "pluginVersion": "7.1.3",
  6066. "pointradius": 5,
  6067. "points": false,
  6068. "renderer": "flot",
  6069. "seriesOverrides": [],
  6070. "spaceLength": 10,
  6071. "stack": false,
  6072. "steppedLine": false,
  6073. "targets": [
  6074. {
  6075. "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
  6076. "format": "time_series",
  6077. "interval": "",
  6078. "intervalFactor": 2,
  6079. "legendFormat": "{{job}}-{{index}} {{block_name}}",
  6080. "refId": "A",
  6081. "step": 20
  6082. }
  6083. ],
  6084. "thresholds": [],
  6085. "timeFrom": null,
  6086. "timeRegions": [],
  6087. "timeShift": null,
  6088. "title": "Average Transactions per Block",
  6089. "tooltip": {
  6090. "shared": false,
  6091. "sort": 0,
  6092. "value_type": "cumulative"
  6093. },
  6094. "type": "graph",
  6095. "xaxis": {
  6096. "buckets": null,
  6097. "mode": "time",
  6098. "name": null,
  6099. "show": true,
  6100. "values": []
  6101. },
  6102. "yaxes": [
  6103. {
  6104. "format": "none",
  6105. "logBase": 1,
  6106. "max": null,
  6107. "min": null,
  6108. "show": true
  6109. },
  6110. {
  6111. "format": "short",
  6112. "logBase": 1,
  6113. "max": null,
  6114. "min": null,
  6115. "show": true
  6116. }
  6117. ],
  6118. "yaxis": {
  6119. "align": false,
  6120. "alignLevel": null
  6121. }
  6122. },
  6123. {
  6124. "aliasColors": {},
  6125. "bars": false,
  6126. "dashLength": 10,
  6127. "dashes": false,
  6128. "datasource": "$datasource",
  6129. "editable": true,
  6130. "error": false,
  6131. "fieldConfig": {
  6132. "defaults": {
  6133. "custom": {},
  6134. "links": []
  6135. },
  6136. "overrides": []
  6137. },
  6138. "fill": 1,
  6139. "fillGradient": 0,
  6140. "grid": {},
  6141. "gridPos": {
  6142. "h": 13,
  6143. "w": 12,
  6144. "x": 12,
  6145. "y": 35
  6146. },
  6147. "hiddenSeries": false,
  6148. "id": 25,
  6149. "legend": {
  6150. "avg": false,
  6151. "current": false,
  6152. "max": false,
  6153. "min": false,
  6154. "show": false,
  6155. "total": false,
  6156. "values": false
  6157. },
  6158. "lines": true,
  6159. "linewidth": 2,
  6160. "links": [],
  6161. "nullPointMode": "null",
  6162. "paceLength": 10,
  6163. "percentage": false,
  6164. "pluginVersion": "7.1.3",
  6165. "pointradius": 5,
  6166. "points": false,
  6167. "renderer": "flot",
  6168. "seriesOverrides": [],
  6169. "spaceLength": 10,
  6170. "stack": false,
  6171. "steppedLine": false,
  6172. "targets": [
  6173. {
  6174. "expr": "rate(synapse_util_metrics_block_time_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_count[$bucket_size])",
  6175. "format": "time_series",
  6176. "interval": "",
  6177. "intervalFactor": 2,
  6178. "legendFormat": "{{job}}-{{index}} {{block_name}}",
  6179. "refId": "A",
  6180. "step": 20
  6181. }
  6182. ],
  6183. "thresholds": [],
  6184. "timeFrom": null,
  6185. "timeRegions": [],
  6186. "timeShift": null,
  6187. "title": "Average Wallclock Time per Block",
  6188. "tooltip": {
  6189. "shared": false,
  6190. "sort": 0,
  6191. "value_type": "cumulative"
  6192. },
  6193. "type": "graph",
  6194. "xaxis": {
  6195. "buckets": null,
  6196. "mode": "time",
  6197. "name": null,
  6198. "show": true,
  6199. "values": []
  6200. },
  6201. "yaxes": [
  6202. {
  6203. "format": "ms",
  6204. "logBase": 1,
  6205. "max": null,
  6206. "min": null,
  6207. "show": true
  6208. },
  6209. {
  6210. "format": "short",
  6211. "logBase": 1,
  6212. "max": null,
  6213. "min": null,
  6214. "show": true
  6215. }
  6216. ],
  6217. "yaxis": {
  6218. "align": false,
  6219. "alignLevel": null
  6220. }
  6221. },
  6222. {
  6223. "aliasColors": {},
  6224. "bars": false,
  6225. "dashLength": 10,
  6226. "dashes": false,
  6227. "datasource": "$datasource",
  6228. "fieldConfig": {
  6229. "defaults": {
  6230. "custom": {}
  6231. },
  6232. "overrides": []
  6233. },
  6234. "fill": 1,
  6235. "fillGradient": 0,
  6236. "gridPos": {
  6237. "h": 15,
  6238. "w": 12,
  6239. "x": 0,
  6240. "y": 48
  6241. },
  6242. "hiddenSeries": false,
  6243. "id": 154,
  6244. "legend": {
  6245. "alignAsTable": true,
  6246. "avg": false,
  6247. "current": false,
  6248. "max": false,
  6249. "min": false,
  6250. "show": true,
  6251. "total": false,
  6252. "values": false
  6253. },
  6254. "lines": true,
  6255. "linewidth": 1,
  6256. "nullPointMode": "null",
  6257. "percentage": false,
  6258. "pluginVersion": "7.1.3",
  6259. "pointradius": 2,
  6260. "points": false,
  6261. "renderer": "flot",
  6262. "seriesOverrides": [],
  6263. "spaceLength": 10,
  6264. "stack": false,
  6265. "steppedLine": false,
  6266. "targets": [
  6267. {
  6268. "expr": "rate(synapse_util_metrics_block_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
  6269. "interval": "",
  6270. "legendFormat": "{{job}}-{{index}} {{block_name}}",
  6271. "refId": "A"
  6272. }
  6273. ],
  6274. "thresholds": [],
  6275. "timeFrom": null,
  6276. "timeRegions": [],
  6277. "timeShift": null,
  6278. "title": "Block count",
  6279. "tooltip": {
  6280. "shared": true,
  6281. "sort": 2,
  6282. "value_type": "individual"
  6283. },
  6284. "type": "graph",
  6285. "xaxis": {
  6286. "buckets": null,
  6287. "mode": "time",
  6288. "name": null,
  6289. "show": true,
  6290. "values": []
  6291. },
  6292. "yaxes": [
  6293. {
  6294. "format": "hertz",
  6295. "label": null,
  6296. "logBase": 1,
  6297. "max": null,
  6298. "min": null,
  6299. "show": true
  6300. },
  6301. {
  6302. "format": "short",
  6303. "label": null,
  6304. "logBase": 1,
  6305. "max": null,
  6306. "min": null,
  6307. "show": true
  6308. }
  6309. ],
  6310. "yaxis": {
  6311. "align": false,
  6312. "alignLevel": null
  6313. }
  6314. }
  6315. ],
  6316. "repeat": null,
  6317. "title": "Per-block metrics",
  6318. "type": "row"
  6319. },
  6320. {
  6321. "collapsed": true,
  6322. "datasource": "${DS_PROMETHEUS}",
  6323. "gridPos": {
  6324. "h": 1,
  6325. "w": 24,
  6326. "x": 0,
  6327. "y": 34
  6328. },
  6329. "id": 61,
  6330. "panels": [
  6331. {
  6332. "aliasColors": {},
  6333. "bars": false,
  6334. "dashLength": 10,
  6335. "dashes": false,
  6336. "datasource": "$datasource",
  6337. "decimals": 2,
  6338. "editable": true,
  6339. "error": false,
  6340. "fieldConfig": {
  6341. "defaults": {
  6342. "custom": {},
  6343. "links": []
  6344. },
  6345. "overrides": []
  6346. },
  6347. "fill": 0,
  6348. "fillGradient": 0,
  6349. "grid": {},
  6350. "gridPos": {
  6351. "h": 10,
  6352. "w": 12,
  6353. "x": 0,
  6354. "y": 35
  6355. },
  6356. "hiddenSeries": false,
  6357. "id": 1,
  6358. "legend": {
  6359. "alignAsTable": true,
  6360. "avg": false,
  6361. "current": false,
  6362. "hideEmpty": true,
  6363. "hideZero": false,
  6364. "max": false,
  6365. "min": false,
  6366. "show": true,
  6367. "total": false,
  6368. "values": false
  6369. },
  6370. "lines": true,
  6371. "linewidth": 2,
  6372. "links": [],
  6373. "nullPointMode": "null",
  6374. "options": {
  6375. "alertThreshold": true
  6376. },
  6377. "percentage": false,
  6378. "pluginVersion": "7.3.7",
  6379. "pointradius": 5,
  6380. "points": false,
  6381. "renderer": "flot",
  6382. "seriesOverrides": [],
  6383. "spaceLength": 10,
  6384. "stack": false,
  6385. "steppedLine": false,
  6386. "targets": [
  6387. {
  6388. "expr": "rate(synapse_util_caches_cache:hits{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])/rate(synapse_util_caches_cache:total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
  6389. "format": "time_series",
  6390. "intervalFactor": 2,
  6391. "legendFormat": "{{name}} {{job}}-{{index}}",
  6392. "refId": "A",
  6393. "step": 20
  6394. }
  6395. ],
  6396. "thresholds": [],
  6397. "timeFrom": null,
  6398. "timeRegions": [],
  6399. "timeShift": null,
  6400. "title": "Cache Hit Ratio",
  6401. "tooltip": {
  6402. "msResolution": true,
  6403. "shared": false,
  6404. "sort": 0,
  6405. "value_type": "cumulative"
  6406. },
  6407. "type": "graph",
  6408. "xaxis": {
  6409. "buckets": null,
  6410. "mode": "time",
  6411. "name": null,
  6412. "show": true,
  6413. "values": []
  6414. },
  6415. "yaxes": [
  6416. {
  6417. "decimals": null,
  6418. "format": "percentunit",
  6419. "label": "",
  6420. "logBase": 1,
  6421. "max": "1",
  6422. "min": 0,
  6423. "show": true
  6424. },
  6425. {
  6426. "format": "short",
  6427. "logBase": 1,
  6428. "max": null,
  6429. "min": null,
  6430. "show": false
  6431. }
  6432. ],
  6433. "yaxis": {
  6434. "align": false,
  6435. "alignLevel": null
  6436. }
  6437. },
  6438. {
  6439. "aliasColors": {},
  6440. "bars": false,
  6441. "dashLength": 10,
  6442. "dashes": false,
  6443. "datasource": "$datasource",
  6444. "editable": true,
  6445. "error": false,
  6446. "fieldConfig": {
  6447. "defaults": {
  6448. "custom": {},
  6449. "links": []
  6450. },
  6451. "overrides": []
  6452. },
  6453. "fill": 1,
  6454. "fillGradient": 0,
  6455. "grid": {},
  6456. "gridPos": {
  6457. "h": 10,
  6458. "w": 12,
  6459. "x": 12,
  6460. "y": 35
  6461. },
  6462. "hiddenSeries": false,
  6463. "id": 8,
  6464. "legend": {
  6465. "alignAsTable": true,
  6466. "avg": false,
  6467. "current": false,
  6468. "hideZero": false,
  6469. "max": false,
  6470. "min": false,
  6471. "show": true,
  6472. "total": false,
  6473. "values": false
  6474. },
  6475. "lines": true,
  6476. "linewidth": 2,
  6477. "links": [],
  6478. "nullPointMode": "connected",
  6479. "options": {
  6480. "alertThreshold": true
  6481. },
  6482. "percentage": false,
  6483. "pluginVersion": "7.3.7",
  6484. "pointradius": 5,
  6485. "points": false,
  6486. "renderer": "flot",
  6487. "seriesOverrides": [],
  6488. "spaceLength": 10,
  6489. "stack": false,
  6490. "steppedLine": false,
  6491. "targets": [
  6492. {
  6493. "expr": "synapse_util_caches_cache:size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
  6494. "format": "time_series",
  6495. "hide": false,
  6496. "interval": "",
  6497. "intervalFactor": 2,
  6498. "legendFormat": "{{name}} {{job}}-{{index}}",
  6499. "refId": "A",
  6500. "step": 20
  6501. }
  6502. ],
  6503. "thresholds": [],
  6504. "timeFrom": null,
  6505. "timeRegions": [],
  6506. "timeShift": null,
  6507. "title": "Cache Size",
  6508. "tooltip": {
  6509. "shared": false,
  6510. "sort": 0,
  6511. "value_type": "cumulative"
  6512. },
  6513. "type": "graph",
  6514. "xaxis": {
  6515. "buckets": null,
  6516. "mode": "time",
  6517. "name": null,
  6518. "show": true,
  6519. "values": []
  6520. },
  6521. "yaxes": [
  6522. {
  6523. "format": "short",
  6524. "logBase": 1,
  6525. "max": null,
  6526. "min": 0,
  6527. "show": true
  6528. },
  6529. {
  6530. "format": "short",
  6531. "logBase": 1,
  6532. "max": null,
  6533. "min": null,
  6534. "show": true
  6535. }
  6536. ],
  6537. "yaxis": {
  6538. "align": false,
  6539. "alignLevel": null
  6540. }
  6541. },
  6542. {
  6543. "aliasColors": {},
  6544. "bars": false,
  6545. "dashLength": 10,
  6546. "dashes": false,
  6547. "datasource": "$datasource",
  6548. "editable": true,
  6549. "error": false,
  6550. "fieldConfig": {
  6551. "defaults": {
  6552. "custom": {},
  6553. "links": []
  6554. },
  6555. "overrides": []
  6556. },
  6557. "fill": 1,
  6558. "fillGradient": 0,
  6559. "grid": {},
  6560. "gridPos": {
  6561. "h": 10,
  6562. "w": 12,
  6563. "x": 0,
  6564. "y": 45
  6565. },
  6566. "hiddenSeries": false,
  6567. "id": 38,
  6568. "legend": {
  6569. "alignAsTable": true,
  6570. "avg": false,
  6571. "current": false,
  6572. "hideZero": false,
  6573. "max": false,
  6574. "min": false,
  6575. "show": true,
  6576. "total": false,
  6577. "values": false
  6578. },
  6579. "lines": true,
  6580. "linewidth": 2,
  6581. "links": [],
  6582. "nullPointMode": "connected",
  6583. "options": {
  6584. "alertThreshold": true
  6585. },
  6586. "percentage": false,
  6587. "pluginVersion": "7.3.7",
  6588. "pointradius": 5,
  6589. "points": false,
  6590. "renderer": "flot",
  6591. "seriesOverrides": [],
  6592. "spaceLength": 10,
  6593. "stack": false,
  6594. "steppedLine": false,
  6595. "targets": [
  6596. {
  6597. "expr": "rate(synapse_util_caches_cache:total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
  6598. "format": "time_series",
  6599. "interval": "",
  6600. "intervalFactor": 2,
  6601. "legendFormat": "{{name}} {{job}}-{{index}}",
  6602. "refId": "A",
  6603. "step": 20
  6604. }
  6605. ],
  6606. "thresholds": [],
  6607. "timeFrom": null,
  6608. "timeRegions": [],
  6609. "timeShift": null,
  6610. "title": "Cache request rate",
  6611. "tooltip": {
  6612. "shared": false,
  6613. "sort": 0,
  6614. "value_type": "cumulative"
  6615. },
  6616. "type": "graph",
  6617. "xaxis": {
  6618. "buckets": null,
  6619. "mode": "time",
  6620. "name": null,
  6621. "show": true,
  6622. "values": []
  6623. },
  6624. "yaxes": [
  6625. {
  6626. "format": "rps",
  6627. "logBase": 1,
  6628. "max": null,
  6629. "min": 0,
  6630. "show": true
  6631. },
  6632. {
  6633. "format": "short",
  6634. "logBase": 1,
  6635. "max": null,
  6636. "min": null,
  6637. "show": true
  6638. }
  6639. ],
  6640. "yaxis": {
  6641. "align": false,
  6642. "alignLevel": null
  6643. }
  6644. },
  6645. {
  6646. "aliasColors": {},
  6647. "bars": false,
  6648. "dashLength": 10,
  6649. "dashes": false,
  6650. "datasource": "$datasource",
  6651. "fieldConfig": {
  6652. "defaults": {
  6653. "custom": {},
  6654. "links": []
  6655. },
  6656. "overrides": []
  6657. },
  6658. "fill": 1,
  6659. "fillGradient": 0,
  6660. "gridPos": {
  6661. "h": 10,
  6662. "w": 12,
  6663. "x": 12,
  6664. "y": 45
  6665. },
  6666. "hiddenSeries": false,
  6667. "id": 39,
  6668. "legend": {
  6669. "alignAsTable": true,
  6670. "avg": false,
  6671. "current": false,
  6672. "max": false,
  6673. "min": false,
  6674. "show": true,
  6675. "total": false,
  6676. "values": false
  6677. },
  6678. "lines": true,
  6679. "linewidth": 1,
  6680. "links": [],
  6681. "nullPointMode": "null",
  6682. "options": {
  6683. "alertThreshold": true
  6684. },
  6685. "percentage": false,
  6686. "pluginVersion": "7.3.7",
  6687. "pointradius": 5,
  6688. "points": false,
  6689. "renderer": "flot",
  6690. "seriesOverrides": [],
  6691. "spaceLength": 10,
  6692. "stack": false,
  6693. "steppedLine": false,
  6694. "targets": [
  6695. {
  6696. "expr": "topk(10, rate(synapse_util_caches_cache:total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]) - rate(synapse_util_caches_cache:hits{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
  6697. "format": "time_series",
  6698. "interval": "",
  6699. "intervalFactor": 2,
  6700. "legendFormat": "{{name}} {{job}}-{{index}}",
  6701. "refId": "A",
  6702. "step": 20
  6703. }
  6704. ],
  6705. "thresholds": [],
  6706. "timeFrom": null,
  6707. "timeRegions": [],
  6708. "timeShift": null,
  6709. "title": "Top 10 cache misses",
  6710. "tooltip": {
  6711. "shared": false,
  6712. "sort": 0,
  6713. "value_type": "individual"
  6714. },
  6715. "type": "graph",
  6716. "xaxis": {
  6717. "buckets": null,
  6718. "mode": "time",
  6719. "name": null,
  6720. "show": true,
  6721. "values": []
  6722. },
  6723. "yaxes": [
  6724. {
  6725. "format": "rps",
  6726. "label": null,
  6727. "logBase": 1,
  6728. "max": null,
  6729. "min": null,
  6730. "show": true
  6731. },
  6732. {
  6733. "format": "short",
  6734. "label": null,
  6735. "logBase": 1,
  6736. "max": null,
  6737. "min": null,
  6738. "show": true
  6739. }
  6740. ],
  6741. "yaxis": {
  6742. "align": false,
  6743. "alignLevel": null
  6744. }
  6745. },
  6746. {
  6747. "aliasColors": {},
  6748. "bars": false,
  6749. "dashLength": 10,
  6750. "dashes": false,
  6751. "datasource": "$datasource",
  6752. "fieldConfig": {
  6753. "defaults": {
  6754. "custom": {},
  6755. "links": []
  6756. },
  6757. "overrides": []
  6758. },
  6759. "fill": 1,
  6760. "fillGradient": 0,
  6761. "gridPos": {
  6762. "h": 9,
  6763. "w": 12,
  6764. "x": 0,
  6765. "y": 55
  6766. },
  6767. "hiddenSeries": false,
  6768. "id": 65,
  6769. "legend": {
  6770. "alignAsTable": true,
  6771. "avg": false,
  6772. "current": false,
  6773. "max": false,
  6774. "min": false,
  6775. "show": true,
  6776. "total": false,
  6777. "values": false
  6778. },
  6779. "lines": true,
  6780. "linewidth": 1,
  6781. "links": [],
  6782. "nullPointMode": "null",
  6783. "options": {
  6784. "alertThreshold": true
  6785. },
  6786. "percentage": false,
  6787. "pluginVersion": "7.3.7",
  6788. "pointradius": 5,
  6789. "points": false,
  6790. "renderer": "flot",
  6791. "seriesOverrides": [],
  6792. "spaceLength": 10,
  6793. "stack": false,
  6794. "steppedLine": false,
  6795. "targets": [
  6796. {
  6797. "expr": "rate(synapse_util_caches_cache:evicted_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
  6798. "format": "time_series",
  6799. "intervalFactor": 1,
  6800. "legendFormat": "{{name}} ({{reason}}) {{job}}-{{index}}",
  6801. "refId": "A"
  6802. }
  6803. ],
  6804. "thresholds": [],
  6805. "timeFrom": null,
  6806. "timeRegions": [],
  6807. "timeShift": null,
  6808. "title": "Cache eviction rate",
  6809. "tooltip": {
  6810. "shared": false,
  6811. "sort": 0,
  6812. "value_type": "individual"
  6813. },
  6814. "type": "graph",
  6815. "xaxis": {
  6816. "buckets": null,
  6817. "mode": "time",
  6818. "name": null,
  6819. "show": true,
  6820. "values": []
  6821. },
  6822. "yaxes": [
  6823. {
  6824. "decimals": null,
  6825. "format": "hertz",
  6826. "label": "entries / second",
  6827. "logBase": 1,
  6828. "max": null,
  6829. "min": null,
  6830. "show": true
  6831. },
  6832. {
  6833. "format": "short",
  6834. "label": null,
  6835. "logBase": 1,
  6836. "max": null,
  6837. "min": null,
  6838. "show": true
  6839. }
  6840. ],
  6841. "yaxis": {
  6842. "align": false,
  6843. "alignLevel": null
  6844. }
  6845. }
  6846. ],
  6847. "repeat": null,
  6848. "title": "Caches",
  6849. "type": "row"
  6850. },
  6851. {
  6852. "collapsed": true,
  6853. "datasource": "${DS_PROMETHEUS}",
  6854. "gridPos": {
  6855. "h": 1,
  6856. "w": 24,
  6857. "x": 0,
  6858. "y": 35
  6859. },
  6860. "id": 148,
  6861. "panels": [
  6862. {
  6863. "aliasColors": {},
  6864. "bars": false,
  6865. "dashLength": 10,
  6866. "dashes": false,
  6867. "datasource": "$datasource",
  6868. "fieldConfig": {
  6869. "defaults": {
  6870. "custom": {},
  6871. "links": []
  6872. },
  6873. "overrides": []
  6874. },
  6875. "fill": 1,
  6876. "fillGradient": 0,
  6877. "gridPos": {
  6878. "h": 8,
  6879. "w": 12,
  6880. "x": 0,
  6881. "y": 29
  6882. },
  6883. "hiddenSeries": false,
  6884. "id": 146,
  6885. "legend": {
  6886. "avg": false,
  6887. "current": false,
  6888. "max": false,
  6889. "min": false,
  6890. "show": true,
  6891. "total": false,
  6892. "values": false
  6893. },
  6894. "lines": true,
  6895. "linewidth": 1,
  6896. "nullPointMode": "null",
  6897. "options": {
  6898. "alertThreshold": true
  6899. },
  6900. "percentage": false,
  6901. "pluginVersion": "7.3.7",
  6902. "pointradius": 2,
  6903. "points": false,
  6904. "renderer": "flot",
  6905. "seriesOverrides": [],
  6906. "spaceLength": 10,
  6907. "stack": false,
  6908. "steppedLine": false,
  6909. "targets": [
  6910. {
  6911. "expr": "synapse_util_caches_response_cache:size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
  6912. "interval": "",
  6913. "legendFormat": "{{name}} {{job}}-{{index}}",
  6914. "refId": "A"
  6915. }
  6916. ],
  6917. "thresholds": [],
  6918. "timeFrom": null,
  6919. "timeRegions": [],
  6920. "timeShift": null,
  6921. "title": "Response cache size",
  6922. "tooltip": {
  6923. "shared": false,
  6924. "sort": 0,
  6925. "value_type": "individual"
  6926. },
  6927. "type": "graph",
  6928. "xaxis": {
  6929. "buckets": null,
  6930. "mode": "time",
  6931. "name": null,
  6932. "show": true,
  6933. "values": []
  6934. },
  6935. "yaxes": [
  6936. {
  6937. "format": "short",
  6938. "label": null,
  6939. "logBase": 1,
  6940. "max": null,
  6941. "min": null,
  6942. "show": true
  6943. },
  6944. {
  6945. "format": "short",
  6946. "label": null,
  6947. "logBase": 1,
  6948. "max": null,
  6949. "min": null,
  6950. "show": true
  6951. }
  6952. ],
  6953. "yaxis": {
  6954. "align": false,
  6955. "alignLevel": null
  6956. }
  6957. },
  6958. {
  6959. "aliasColors": {},
  6960. "bars": false,
  6961. "dashLength": 10,
  6962. "dashes": false,
  6963. "datasource": "$datasource",
  6964. "fieldConfig": {
  6965. "defaults": {
  6966. "custom": {},
  6967. "links": []
  6968. },
  6969. "overrides": []
  6970. },
  6971. "fill": 1,
  6972. "fillGradient": 0,
  6973. "gridPos": {
  6974. "h": 8,
  6975. "w": 12,
  6976. "x": 12,
  6977. "y": 29
  6978. },
  6979. "hiddenSeries": false,
  6980. "id": 150,
  6981. "legend": {
  6982. "avg": false,
  6983. "current": false,
  6984. "max": false,
  6985. "min": false,
  6986. "show": true,
  6987. "total": false,
  6988. "values": false
  6989. },
  6990. "lines": true,
  6991. "linewidth": 1,
  6992. "nullPointMode": "null",
  6993. "options": {
  6994. "alertThreshold": true
  6995. },
  6996. "percentage": false,
  6997. "pluginVersion": "7.3.7",
  6998. "pointradius": 2,
  6999. "points": false,
  7000. "renderer": "flot",
  7001. "seriesOverrides": [],
  7002. "spaceLength": 10,
  7003. "stack": false,
  7004. "steppedLine": false,
  7005. "targets": [
  7006. {
  7007. "expr": "rate(synapse_util_caches_response_cache:hits{instance=\"$instance\", job=~\"$job\", index=~\"$index\"}[$bucket_size])/rate(synapse_util_caches_response_cache:total{instance=\"$instance\", job=~\"$job\", index=~\"$index\"}[$bucket_size])",
  7008. "interval": "",
  7009. "legendFormat": "{{name}} {{job}}-{{index}}",
  7010. "refId": "A"
  7011. },
  7012. {
  7013. "expr": "",
  7014. "interval": "",
  7015. "legendFormat": "",
  7016. "refId": "B"
  7017. }
  7018. ],
  7019. "thresholds": [],
  7020. "timeFrom": null,
  7021. "timeRegions": [],
  7022. "timeShift": null,
  7023. "title": "Response cache hit rate",
  7024. "tooltip": {
  7025. "shared": false,
  7026. "sort": 0,
  7027. "value_type": "individual"
  7028. },
  7029. "type": "graph",
  7030. "xaxis": {
  7031. "buckets": null,
  7032. "mode": "time",
  7033. "name": null,
  7034. "show": true,
  7035. "values": []
  7036. },
  7037. "yaxes": [
  7038. {
  7039. "decimals": null,
  7040. "format": "percentunit",
  7041. "label": null,
  7042. "logBase": 1,
  7043. "max": "1",
  7044. "min": "0",
  7045. "show": true
  7046. },
  7047. {
  7048. "format": "short",
  7049. "label": null,
  7050. "logBase": 1,
  7051. "max": null,
  7052. "min": null,
  7053. "show": true
  7054. }
  7055. ],
  7056. "yaxis": {
  7057. "align": false,
  7058. "alignLevel": null
  7059. }
  7060. }
  7061. ],
  7062. "title": "Response caches",
  7063. "type": "row"
  7064. },
  7065. {
  7066. "collapsed": true,
  7067. "datasource": "${DS_PROMETHEUS}",
  7068. "gridPos": {
  7069. "h": 1,
  7070. "w": 24,
  7071. "x": 0,
  7072. "y": 36
  7073. },
  7074. "id": 62,
  7075. "panels": [
  7076. {
  7077. "aliasColors": {},
  7078. "bars": false,
  7079. "dashLength": 10,
  7080. "dashes": false,
  7081. "datasource": "$datasource",
  7082. "fieldConfig": {
  7083. "defaults": {
  7084. "custom": {},
  7085. "links": []
  7086. },
  7087. "overrides": []
  7088. },
  7089. "fill": 1,
  7090. "fillGradient": 0,
  7091. "gridPos": {
  7092. "h": 9,
  7093. "w": 12,
  7094. "x": 0,
  7095. "y": 30
  7096. },
  7097. "hiddenSeries": false,
  7098. "id": 91,
  7099. "legend": {
  7100. "avg": false,
  7101. "current": false,
  7102. "max": false,
  7103. "min": false,
  7104. "show": true,
  7105. "total": false,
  7106. "values": false
  7107. },
  7108. "lines": true,
  7109. "linewidth": 1,
  7110. "links": [],
  7111. "nullPointMode": "null",
  7112. "options": {
  7113. "alertThreshold": true
  7114. },
  7115. "percentage": false,
  7116. "pluginVersion": "7.3.7",
  7117. "pointradius": 5,
  7118. "points": false,
  7119. "renderer": "flot",
  7120. "seriesOverrides": [],
  7121. "spaceLength": 10,
  7122. "stack": true,
  7123. "steppedLine": false,
  7124. "targets": [
  7125. {
  7126. "expr": "rate(python_gc_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[10m])",
  7127. "format": "time_series",
  7128. "instant": false,
  7129. "intervalFactor": 1,
  7130. "legendFormat": "{{job}}-{{index}} gen {{gen}}",
  7131. "refId": "A"
  7132. }
  7133. ],
  7134. "thresholds": [],
  7135. "timeFrom": null,
  7136. "timeRegions": [],
  7137. "timeShift": null,
  7138. "title": "Total GC time by bucket (10m smoothing)",
  7139. "tooltip": {
  7140. "shared": true,
  7141. "sort": 0,
  7142. "value_type": "individual"
  7143. },
  7144. "type": "graph",
  7145. "xaxis": {
  7146. "buckets": null,
  7147. "mode": "time",
  7148. "name": null,
  7149. "show": true,
  7150. "values": []
  7151. },
  7152. "yaxes": [
  7153. {
  7154. "decimals": null,
  7155. "format": "percentunit",
  7156. "label": null,
  7157. "logBase": 1,
  7158. "max": null,
  7159. "min": "0",
  7160. "show": true
  7161. },
  7162. {
  7163. "format": "short",
  7164. "label": null,
  7165. "logBase": 1,
  7166. "max": null,
  7167. "min": null,
  7168. "show": true
  7169. }
  7170. ],
  7171. "yaxis": {
  7172. "align": false,
  7173. "alignLevel": null
  7174. }
  7175. },
  7176. {
  7177. "aliasColors": {},
  7178. "bars": false,
  7179. "dashLength": 10,
  7180. "dashes": false,
  7181. "datasource": "$datasource",
  7182. "decimals": 3,
  7183. "editable": true,
  7184. "error": false,
  7185. "fieldConfig": {
  7186. "defaults": {
  7187. "custom": {},
  7188. "links": []
  7189. },
  7190. "overrides": []
  7191. },
  7192. "fill": 1,
  7193. "fillGradient": 0,
  7194. "grid": {},
  7195. "gridPos": {
  7196. "h": 9,
  7197. "w": 12,
  7198. "x": 12,
  7199. "y": 30
  7200. },
  7201. "hiddenSeries": false,
  7202. "id": 21,
  7203. "legend": {
  7204. "alignAsTable": true,
  7205. "avg": false,
  7206. "current": false,
  7207. "max": false,
  7208. "min": false,
  7209. "show": true,
  7210. "total": false,
  7211. "values": false
  7212. },
  7213. "lines": true,
  7214. "linewidth": 2,
  7215. "links": [],
  7216. "nullPointMode": "null as zero",
  7217. "options": {
  7218. "alertThreshold": true
  7219. },
  7220. "percentage": false,
  7221. "pluginVersion": "7.3.7",
  7222. "pointradius": 5,
  7223. "points": false,
  7224. "renderer": "flot",
  7225. "seriesOverrides": [],
  7226. "spaceLength": 10,
  7227. "stack": false,
  7228. "steppedLine": false,
  7229. "targets": [
  7230. {
  7231. "expr": "rate(python_gc_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(python_gc_time_count[$bucket_size])",
  7232. "format": "time_series",
  7233. "intervalFactor": 2,
  7234. "legendFormat": "{{job}} {{index}} gen {{gen}} ",
  7235. "refId": "A",
  7236. "step": 20,
  7237. "target": ""
  7238. }
  7239. ],
  7240. "thresholds": [],
  7241. "timeFrom": null,
  7242. "timeRegions": [],
  7243. "timeShift": null,
  7244. "title": "Average GC Time Per Collection",
  7245. "tooltip": {
  7246. "shared": false,
  7247. "sort": 0,
  7248. "value_type": "cumulative"
  7249. },
  7250. "type": "graph",
  7251. "xaxis": {
  7252. "buckets": null,
  7253. "mode": "time",
  7254. "name": null,
  7255. "show": true,
  7256. "values": []
  7257. },
  7258. "yaxes": [
  7259. {
  7260. "format": "s",
  7261. "logBase": 1,
  7262. "max": null,
  7263. "min": null,
  7264. "show": true
  7265. },
  7266. {
  7267. "format": "short",
  7268. "logBase": 1,
  7269. "max": null,
  7270. "min": null,
  7271. "show": true
  7272. }
  7273. ],
  7274. "yaxis": {
  7275. "align": false,
  7276. "alignLevel": null
  7277. }
  7278. },
  7279. {
  7280. "aliasColors": {},
  7281. "bars": false,
  7282. "dashLength": 10,
  7283. "dashes": false,
  7284. "datasource": "$datasource",
  7285. "description": "'gen 0' shows the number of objects allocated since the last gen0 GC.\n'gen 1' / 'gen 2' show the number of gen0/gen1 GCs since the last gen1/gen2 GC.",
  7286. "fieldConfig": {
  7287. "defaults": {
  7288. "custom": {},
  7289. "links": []
  7290. },
  7291. "overrides": []
  7292. },
  7293. "fill": 1,
  7294. "fillGradient": 0,
  7295. "gridPos": {
  7296. "h": 9,
  7297. "w": 12,
  7298. "x": 0,
  7299. "y": 39
  7300. },
  7301. "hiddenSeries": false,
  7302. "id": 89,
  7303. "legend": {
  7304. "avg": false,
  7305. "current": false,
  7306. "hideEmpty": true,
  7307. "hideZero": false,
  7308. "max": false,
  7309. "min": false,
  7310. "show": true,
  7311. "total": false,
  7312. "values": false
  7313. },
  7314. "lines": true,
  7315. "linewidth": 1,
  7316. "links": [],
  7317. "nullPointMode": "null",
  7318. "options": {
  7319. "alertThreshold": true
  7320. },
  7321. "percentage": false,
  7322. "pluginVersion": "7.3.7",
  7323. "pointradius": 5,
  7324. "points": false,
  7325. "renderer": "flot",
  7326. "seriesOverrides": [
  7327. {
  7328. "alias": "/gen 0$/",
  7329. "yaxis": 2
  7330. }
  7331. ],
  7332. "spaceLength": 10,
  7333. "stack": false,
  7334. "steppedLine": false,
  7335. "targets": [
  7336. {
  7337. "expr": "python_gc_counts{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
  7338. "format": "time_series",
  7339. "intervalFactor": 1,
  7340. "legendFormat": "{{job}}-{{index}} gen {{gen}}",
  7341. "refId": "A"
  7342. }
  7343. ],
  7344. "thresholds": [],
  7345. "timeFrom": null,
  7346. "timeRegions": [],
  7347. "timeShift": null,
  7348. "title": "Allocation counts",
  7349. "tooltip": {
  7350. "shared": false,
  7351. "sort": 0,
  7352. "value_type": "individual"
  7353. },
  7354. "type": "graph",
  7355. "xaxis": {
  7356. "buckets": null,
  7357. "mode": "time",
  7358. "name": null,
  7359. "show": true,
  7360. "values": []
  7361. },
  7362. "yaxes": [
  7363. {
  7364. "format": "short",
  7365. "label": "Gen N-1 GCs since last Gen N GC",
  7366. "logBase": 1,
  7367. "max": null,
  7368. "min": null,
  7369. "show": true
  7370. },
  7371. {
  7372. "decimals": null,
  7373. "format": "short",
  7374. "label": "Objects since last Gen 0 GC",
  7375. "logBase": 1,
  7376. "max": null,
  7377. "min": null,
  7378. "show": true
  7379. }
  7380. ],
  7381. "yaxis": {
  7382. "align": false,
  7383. "alignLevel": null
  7384. }
  7385. },
  7386. {
  7387. "aliasColors": {},
  7388. "bars": false,
  7389. "dashLength": 10,
  7390. "dashes": false,
  7391. "datasource": "$datasource",
  7392. "fieldConfig": {
  7393. "defaults": {
  7394. "custom": {},
  7395. "links": []
  7396. },
  7397. "overrides": []
  7398. },
  7399. "fill": 1,
  7400. "fillGradient": 0,
  7401. "gridPos": {
  7402. "h": 9,
  7403. "w": 12,
  7404. "x": 12,
  7405. "y": 39
  7406. },
  7407. "hiddenSeries": false,
  7408. "id": 93,
  7409. "legend": {
  7410. "avg": false,
  7411. "current": false,
  7412. "max": false,
  7413. "min": false,
  7414. "show": true,
  7415. "total": false,
  7416. "values": false
  7417. },
  7418. "lines": true,
  7419. "linewidth": 1,
  7420. "links": [],
  7421. "nullPointMode": "connected",
  7422. "options": {
  7423. "alertThreshold": true
  7424. },
  7425. "percentage": false,
  7426. "pluginVersion": "7.3.7",
  7427. "pointradius": 5,
  7428. "points": false,
  7429. "renderer": "flot",
  7430. "seriesOverrides": [],
  7431. "spaceLength": 10,
  7432. "stack": false,
  7433. "steppedLine": false,
  7434. "targets": [
  7435. {
  7436. "expr": "rate(python_gc_unreachable_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(python_gc_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
  7437. "format": "time_series",
  7438. "intervalFactor": 1,
  7439. "legendFormat": "{{job}}-{{index}} gen {{gen}}",
  7440. "refId": "A"
  7441. }
  7442. ],
  7443. "thresholds": [],
  7444. "timeFrom": null,
  7445. "timeRegions": [],
  7446. "timeShift": null,
  7447. "title": "Object counts per collection",
  7448. "tooltip": {
  7449. "shared": true,
  7450. "sort": 0,
  7451. "value_type": "individual"
  7452. },
  7453. "type": "graph",
  7454. "xaxis": {
  7455. "buckets": null,
  7456. "mode": "time",
  7457. "name": null,
  7458. "show": true,
  7459. "values": []
  7460. },
  7461. "yaxes": [
  7462. {
  7463. "format": "short",
  7464. "label": null,
  7465. "logBase": 1,
  7466. "max": null,
  7467. "min": null,
  7468. "show": true
  7469. },
  7470. {
  7471. "format": "short",
  7472. "label": null,
  7473. "logBase": 1,
  7474. "max": null,
  7475. "min": null,
  7476. "show": true
  7477. }
  7478. ],
  7479. "yaxis": {
  7480. "align": false,
  7481. "alignLevel": null
  7482. }
  7483. },
  7484. {
  7485. "aliasColors": {},
  7486. "bars": false,
  7487. "dashLength": 10,
  7488. "dashes": false,
  7489. "datasource": "$datasource",
  7490. "fieldConfig": {
  7491. "defaults": {
  7492. "custom": {},
  7493. "links": []
  7494. },
  7495. "overrides": []
  7496. },
  7497. "fill": 1,
  7498. "fillGradient": 0,
  7499. "gridPos": {
  7500. "h": 9,
  7501. "w": 12,
  7502. "x": 0,
  7503. "y": 48
  7504. },
  7505. "hiddenSeries": false,
  7506. "id": 95,
  7507. "legend": {
  7508. "avg": false,
  7509. "current": false,
  7510. "max": false,
  7511. "min": false,
  7512. "show": true,
  7513. "total": false,
  7514. "values": false
  7515. },
  7516. "lines": true,
  7517. "linewidth": 1,
  7518. "links": [],
  7519. "nullPointMode": "null",
  7520. "options": {
  7521. "alertThreshold": true
  7522. },
  7523. "percentage": false,
  7524. "pluginVersion": "7.3.7",
  7525. "pointradius": 5,
  7526. "points": false,
  7527. "renderer": "flot",
  7528. "seriesOverrides": [],
  7529. "spaceLength": 10,
  7530. "stack": false,
  7531. "steppedLine": false,
  7532. "targets": [
  7533. {
  7534. "expr": "rate(python_gc_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
  7535. "format": "time_series",
  7536. "intervalFactor": 1,
  7537. "legendFormat": "{{job}}-{{index}} gen {{gen}}",
  7538. "refId": "A"
  7539. }
  7540. ],
  7541. "thresholds": [],
  7542. "timeFrom": null,
  7543. "timeRegions": [],
  7544. "timeShift": null,
  7545. "title": "GC frequency",
  7546. "tooltip": {
  7547. "shared": true,
  7548. "sort": 0,
  7549. "value_type": "individual"
  7550. },
  7551. "type": "graph",
  7552. "xaxis": {
  7553. "buckets": null,
  7554. "mode": "time",
  7555. "name": null,
  7556. "show": true,
  7557. "values": []
  7558. },
  7559. "yaxes": [
  7560. {
  7561. "format": "hertz",
  7562. "label": null,
  7563. "logBase": 1,
  7564. "max": null,
  7565. "min": null,
  7566. "show": true
  7567. },
  7568. {
  7569. "format": "short",
  7570. "label": null,
  7571. "logBase": 1,
  7572. "max": null,
  7573. "min": null,
  7574. "show": true
  7575. }
  7576. ],
  7577. "yaxis": {
  7578. "align": false,
  7579. "alignLevel": null
  7580. }
  7581. },
  7582. {
  7583. "cards": {
  7584. "cardPadding": 0,
  7585. "cardRound": null
  7586. },
  7587. "color": {
  7588. "cardColor": "#b4ff00",
  7589. "colorScale": "sqrt",
  7590. "colorScheme": "interpolateSpectral",
  7591. "exponent": 0.5,
  7592. "max": null,
  7593. "min": 0,
  7594. "mode": "spectrum"
  7595. },
  7596. "dataFormat": "tsbuckets",
  7597. "datasource": "${DS_PROMETHEUS}",
  7598. "fieldConfig": {
  7599. "defaults": {
  7600. "custom": {}
  7601. },
  7602. "overrides": []
  7603. },
  7604. "gridPos": {
  7605. "h": 9,
  7606. "w": 12,
  7607. "x": 12,
  7608. "y": 48
  7609. },
  7610. "heatmap": {},
  7611. "hideZeroBuckets": true,
  7612. "highlightCards": true,
  7613. "id": 87,
  7614. "legend": {
  7615. "show": true
  7616. },
  7617. "links": [],
  7618. "reverseYBuckets": false,
  7619. "targets": [
  7620. {
  7621. "expr": "sum(rate(python_gc_time_bucket[$bucket_size])) by (le)",
  7622. "format": "heatmap",
  7623. "intervalFactor": 1,
  7624. "legendFormat": "{{le}}",
  7625. "refId": "A"
  7626. }
  7627. ],
  7628. "title": "GC durations",
  7629. "tooltip": {
  7630. "show": true,
  7631. "showHistogram": false
  7632. },
  7633. "type": "heatmap",
  7634. "xAxis": {
  7635. "show": true
  7636. },
  7637. "xBucketNumber": null,
  7638. "xBucketSize": null,
  7639. "yAxis": {
  7640. "decimals": null,
  7641. "format": "s",
  7642. "logBase": 1,
  7643. "max": null,
  7644. "min": null,
  7645. "show": true,
  7646. "splitFactor": null
  7647. },
  7648. "yBucketBound": "auto",
  7649. "yBucketNumber": null,
  7650. "yBucketSize": null
  7651. }
  7652. ],
  7653. "repeat": null,
  7654. "title": "GC",
  7655. "type": "row"
  7656. },
  7657. {
  7658. "collapsed": true,
  7659. "datasource": "${DS_PROMETHEUS}",
  7660. "gridPos": {
  7661. "h": 1,
  7662. "w": 24,
  7663. "x": 0,
  7664. "y": 37
  7665. },
  7666. "id": 63,
  7667. "panels": [
  7668. {
  7669. "aliasColors": {},
  7670. "bars": false,
  7671. "dashLength": 10,
  7672. "dashes": false,
  7673. "datasource": "$datasource",
  7674. "fieldConfig": {
  7675. "defaults": {
  7676. "custom": {},
  7677. "links": []
  7678. },
  7679. "overrides": []
  7680. },
  7681. "fill": 1,
  7682. "fillGradient": 0,
  7683. "gridPos": {
  7684. "h": 7,
  7685. "w": 12,
  7686. "x": 0,
  7687. "y": 13
  7688. },
  7689. "hiddenSeries": false,
  7690. "id": 42,
  7691. "legend": {
  7692. "avg": false,
  7693. "current": false,
  7694. "max": false,
  7695. "min": false,
  7696. "show": true,
  7697. "total": false,
  7698. "values": false
  7699. },
  7700. "lines": true,
  7701. "linewidth": 1,
  7702. "links": [],
  7703. "nullPointMode": "null",
  7704. "options": {
  7705. "alertThreshold": true
  7706. },
  7707. "paceLength": 10,
  7708. "percentage": false,
  7709. "pluginVersion": "7.3.7",
  7710. "pointradius": 5,
  7711. "points": false,
  7712. "renderer": "flot",
  7713. "seriesOverrides": [],
  7714. "spaceLength": 10,
  7715. "stack": false,
  7716. "steppedLine": false,
  7717. "targets": [
  7718. {
  7719. "expr": "sum (rate(synapse_replication_tcp_protocol_inbound_commands{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)",
  7720. "format": "time_series",
  7721. "intervalFactor": 2,
  7722. "legendFormat": "{{job}}-{{index}} {{command}}",
  7723. "refId": "A",
  7724. "step": 20
  7725. }
  7726. ],
  7727. "thresholds": [],
  7728. "timeFrom": null,
  7729. "timeRegions": [],
  7730. "timeShift": null,
  7731. "title": "Rate of incoming commands",
  7732. "tooltip": {
  7733. "shared": false,
  7734. "sort": 0,
  7735. "value_type": "individual"
  7736. },
  7737. "type": "graph",
  7738. "xaxis": {
  7739. "buckets": null,
  7740. "mode": "time",
  7741. "name": null,
  7742. "show": true,
  7743. "values": []
  7744. },
  7745. "yaxes": [
  7746. {
  7747. "format": "hertz",
  7748. "label": null,
  7749. "logBase": 1,
  7750. "max": null,
  7751. "min": null,
  7752. "show": true
  7753. },
  7754. {
  7755. "format": "short",
  7756. "label": null,
  7757. "logBase": 1,
  7758. "max": null,
  7759. "min": null,
  7760. "show": true
  7761. }
  7762. ],
  7763. "yaxis": {
  7764. "align": false,
  7765. "alignLevel": null
  7766. }
  7767. },
  7768. {
  7769. "aliasColors": {},
  7770. "bars": false,
  7771. "dashLength": 10,
  7772. "dashes": false,
  7773. "datasource": "${DS_PROMETHEUS}",
  7774. "description": "",
  7775. "fieldConfig": {
  7776. "defaults": {
  7777. "custom": {},
  7778. "links": []
  7779. },
  7780. "overrides": []
  7781. },
  7782. "fill": 1,
  7783. "fillGradient": 0,
  7784. "gridPos": {
  7785. "h": 7,
  7786. "w": 12,
  7787. "x": 12,
  7788. "y": 13
  7789. },
  7790. "hiddenSeries": false,
  7791. "id": 144,
  7792. "legend": {
  7793. "avg": false,
  7794. "current": false,
  7795. "max": false,
  7796. "min": false,
  7797. "show": true,
  7798. "total": false,
  7799. "values": false
  7800. },
  7801. "lines": true,
  7802. "linewidth": 1,
  7803. "nullPointMode": "null",
  7804. "options": {
  7805. "alertThreshold": true
  7806. },
  7807. "percentage": false,
  7808. "pluginVersion": "7.3.7",
  7809. "pointradius": 2,
  7810. "points": false,
  7811. "renderer": "flot",
  7812. "seriesOverrides": [],
  7813. "spaceLength": 10,
  7814. "stack": false,
  7815. "steppedLine": false,
  7816. "targets": [
  7817. {
  7818. "expr": "synapse_replication_tcp_command_queue{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
  7819. "interval": "",
  7820. "legendFormat": "{{stream_name}} {{job}}-{{index}}",
  7821. "refId": "A"
  7822. }
  7823. ],
  7824. "thresholds": [],
  7825. "timeFrom": null,
  7826. "timeRegions": [],
  7827. "timeShift": null,
  7828. "title": "Queued incoming RDATA commands, by stream",
  7829. "tooltip": {
  7830. "shared": false,
  7831. "sort": 0,
  7832. "value_type": "individual"
  7833. },
  7834. "type": "graph",
  7835. "xaxis": {
  7836. "buckets": null,
  7837. "mode": "time",
  7838. "name": null,
  7839. "show": true,
  7840. "values": []
  7841. },
  7842. "yaxes": [
  7843. {
  7844. "format": "short",
  7845. "label": null,
  7846. "logBase": 1,
  7847. "max": null,
  7848. "min": null,
  7849. "show": true
  7850. },
  7851. {
  7852. "format": "short",
  7853. "label": null,
  7854. "logBase": 1,
  7855. "max": null,
  7856. "min": null,
  7857. "show": true
  7858. }
  7859. ],
  7860. "yaxis": {
  7861. "align": false,
  7862. "alignLevel": null
  7863. }
  7864. },
  7865. {
  7866. "aliasColors": {},
  7867. "bars": false,
  7868. "dashLength": 10,
  7869. "dashes": false,
  7870. "datasource": "$datasource",
  7871. "fieldConfig": {
  7872. "defaults": {
  7873. "custom": {},
  7874. "links": []
  7875. },
  7876. "overrides": []
  7877. },
  7878. "fill": 1,
  7879. "fillGradient": 0,
  7880. "gridPos": {
  7881. "h": 7,
  7882. "w": 12,
  7883. "x": 0,
  7884. "y": 20
  7885. },
  7886. "hiddenSeries": false,
  7887. "id": 43,
  7888. "legend": {
  7889. "avg": false,
  7890. "current": false,
  7891. "max": false,
  7892. "min": false,
  7893. "show": true,
  7894. "total": false,
  7895. "values": false
  7896. },
  7897. "lines": true,
  7898. "linewidth": 1,
  7899. "links": [],
  7900. "nullPointMode": "null",
  7901. "options": {
  7902. "alertThreshold": true
  7903. },
  7904. "paceLength": 10,
  7905. "percentage": false,
  7906. "pluginVersion": "7.3.7",
  7907. "pointradius": 5,
  7908. "points": false,
  7909. "renderer": "flot",
  7910. "seriesOverrides": [],
  7911. "spaceLength": 10,
  7912. "stack": false,
  7913. "steppedLine": false,
  7914. "targets": [
  7915. {
  7916. "expr": "sum (rate(synapse_replication_tcp_protocol_outbound_commands{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)",
  7917. "format": "time_series",
  7918. "intervalFactor": 2,
  7919. "legendFormat": "{{job}}-{{index}} {{command}}",
  7920. "refId": "A",
  7921. "step": 20
  7922. }
  7923. ],
  7924. "thresholds": [],
  7925. "timeFrom": null,
  7926. "timeRegions": [],
  7927. "timeShift": null,
  7928. "title": "Rate of outgoing commands",
  7929. "tooltip": {
  7930. "shared": false,
  7931. "sort": 0,
  7932. "value_type": "individual"
  7933. },
  7934. "type": "graph",
  7935. "xaxis": {
  7936. "buckets": null,
  7937. "mode": "time",
  7938. "name": null,
  7939. "show": true,
  7940. "values": []
  7941. },
  7942. "yaxes": [
  7943. {
  7944. "format": "hertz",
  7945. "label": null,
  7946. "logBase": 1,
  7947. "max": null,
  7948. "min": null,
  7949. "show": true
  7950. },
  7951. {
  7952. "format": "short",
  7953. "label": null,
  7954. "logBase": 1,
  7955. "max": null,
  7956. "min": null,
  7957. "show": true
  7958. }
  7959. ],
  7960. "yaxis": {
  7961. "align": false,
  7962. "alignLevel": null
  7963. }
  7964. },
  7965. {
  7966. "aliasColors": {},
  7967. "bars": false,
  7968. "dashLength": 10,
  7969. "dashes": false,
  7970. "datasource": "$datasource",
  7971. "fieldConfig": {
  7972. "defaults": {
  7973. "custom": {},
  7974. "links": []
  7975. },
  7976. "overrides": []
  7977. },
  7978. "fill": 1,
  7979. "fillGradient": 0,
  7980. "gridPos": {
  7981. "h": 7,
  7982. "w": 12,
  7983. "x": 12,
  7984. "y": 20
  7985. },
  7986. "hiddenSeries": false,
  7987. "id": 41,
  7988. "legend": {
  7989. "avg": false,
  7990. "current": false,
  7991. "max": false,
  7992. "min": false,
  7993. "show": true,
  7994. "total": false,
  7995. "values": false
  7996. },
  7997. "lines": true,
  7998. "linewidth": 1,
  7999. "links": [],
  8000. "nullPointMode": "null",
  8001. "options": {
  8002. "alertThreshold": true
  8003. },
  8004. "paceLength": 10,
  8005. "percentage": false,
  8006. "pluginVersion": "7.3.7",
  8007. "pointradius": 5,
  8008. "points": false,
  8009. "renderer": "flot",
  8010. "seriesOverrides": [],
  8011. "spaceLength": 10,
  8012. "stack": false,
  8013. "steppedLine": false,
  8014. "targets": [
  8015. {
  8016. "expr": "rate(synapse_replication_tcp_resource_stream_updates{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
  8017. "format": "time_series",
  8018. "interval": "",
  8019. "intervalFactor": 2,
  8020. "legendFormat": "{{stream_name}}",
  8021. "refId": "A",
  8022. "step": 20
  8023. }
  8024. ],
  8025. "thresholds": [],
  8026. "timeFrom": null,
  8027. "timeRegions": [],
  8028. "timeShift": null,
  8029. "title": "Outgoing stream updates",
  8030. "tooltip": {
  8031. "shared": false,
  8032. "sort": 0,
  8033. "value_type": "individual"
  8034. },
  8035. "type": "graph",
  8036. "xaxis": {
  8037. "buckets": null,
  8038. "mode": "time",
  8039. "name": null,
  8040. "show": true,
  8041. "values": []
  8042. },
  8043. "yaxes": [
  8044. {
  8045. "format": "hertz",
  8046. "label": null,
  8047. "logBase": 1,
  8048. "max": null,
  8049. "min": null,
  8050. "show": true
  8051. },
  8052. {
  8053. "format": "short",
  8054. "label": null,
  8055. "logBase": 1,
  8056. "max": null,
  8057. "min": null,
  8058. "show": true
  8059. }
  8060. ],
  8061. "yaxis": {
  8062. "align": false,
  8063. "alignLevel": null
  8064. }
  8065. },
  8066. {
  8067. "aliasColors": {},
  8068. "bars": false,
  8069. "dashLength": 10,
  8070. "dashes": false,
  8071. "datasource": "$datasource",
  8072. "fieldConfig": {
  8073. "defaults": {
  8074. "custom": {},
  8075. "links": []
  8076. },
  8077. "overrides": []
  8078. },
  8079. "fill": 1,
  8080. "fillGradient": 0,
  8081. "gridPos": {
  8082. "h": 7,
  8083. "w": 12,
  8084. "x": 0,
  8085. "y": 27
  8086. },
  8087. "hiddenSeries": false,
  8088. "id": 113,
  8089. "legend": {
  8090. "avg": false,
  8091. "current": false,
  8092. "max": false,
  8093. "min": false,
  8094. "show": true,
  8095. "total": false,
  8096. "values": false
  8097. },
  8098. "lines": true,
  8099. "linewidth": 1,
  8100. "links": [],
  8101. "nullPointMode": "null",
  8102. "options": {
  8103. "alertThreshold": true
  8104. },
  8105. "paceLength": 10,
  8106. "percentage": false,
  8107. "pluginVersion": "7.3.7",
  8108. "pointradius": 5,
  8109. "points": false,
  8110. "renderer": "flot",
  8111. "seriesOverrides": [],
  8112. "spaceLength": 10,
  8113. "stack": false,
  8114. "steppedLine": false,
  8115. "targets": [
  8116. {
  8117. "expr": "synapse_replication_tcp_resource_connections_per_stream{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
  8118. "format": "time_series",
  8119. "intervalFactor": 1,
  8120. "legendFormat": "{{job}}-{{index}} {{stream_name}}",
  8121. "refId": "A"
  8122. },
  8123. {
  8124. "expr": "synapse_replication_tcp_resource_total_connections{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
  8125. "format": "time_series",
  8126. "intervalFactor": 1,
  8127. "legendFormat": "{{job}}-{{index}}",
  8128. "refId": "B"
  8129. }
  8130. ],
  8131. "thresholds": [],
  8132. "timeFrom": null,
  8133. "timeRegions": [],
  8134. "timeShift": null,
  8135. "title": "Replication connections",
  8136. "tooltip": {
  8137. "shared": true,
  8138. "sort": 0,
  8139. "value_type": "individual"
  8140. },
  8141. "type": "graph",
  8142. "xaxis": {
  8143. "buckets": null,
  8144. "mode": "time",
  8145. "name": null,
  8146. "show": true,
  8147. "values": []
  8148. },
  8149. "yaxes": [
  8150. {
  8151. "format": "short",
  8152. "label": null,
  8153. "logBase": 1,
  8154. "max": null,
  8155. "min": "0",
  8156. "show": true
  8157. },
  8158. {
  8159. "format": "short",
  8160. "label": null,
  8161. "logBase": 1,
  8162. "max": null,
  8163. "min": null,
  8164. "show": true
  8165. }
  8166. ],
  8167. "yaxis": {
  8168. "align": false,
  8169. "alignLevel": null
  8170. }
  8171. },
  8172. {
  8173. "aliasColors": {},
  8174. "bars": false,
  8175. "dashLength": 10,
  8176. "dashes": false,
  8177. "datasource": "$datasource",
  8178. "fieldConfig": {
  8179. "defaults": {
  8180. "custom": {},
  8181. "links": []
  8182. },
  8183. "overrides": []
  8184. },
  8185. "fill": 1,
  8186. "fillGradient": 0,
  8187. "gridPos": {
  8188. "h": 7,
  8189. "w": 12,
  8190. "x": 12,
  8191. "y": 27
  8192. },
  8193. "hiddenSeries": false,
  8194. "id": 115,
  8195. "legend": {
  8196. "avg": false,
  8197. "current": false,
  8198. "max": false,
  8199. "min": false,
  8200. "show": true,
  8201. "total": false,
  8202. "values": false
  8203. },
  8204. "lines": true,
  8205. "linewidth": 1,
  8206. "links": [],
  8207. "nullPointMode": "null",
  8208. "options": {
  8209. "alertThreshold": true
  8210. },
  8211. "paceLength": 10,
  8212. "percentage": false,
  8213. "pluginVersion": "7.3.7",
  8214. "pointradius": 5,
  8215. "points": false,
  8216. "renderer": "flot",
  8217. "seriesOverrides": [],
  8218. "spaceLength": 10,
  8219. "stack": false,
  8220. "steppedLine": false,
  8221. "targets": [
  8222. {
  8223. "expr": "rate(synapse_replication_tcp_protocol_close_reason{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
  8224. "format": "time_series",
  8225. "intervalFactor": 1,
  8226. "legendFormat": "{{job}}-{{index}} {{reason_type}}",
  8227. "refId": "A"
  8228. }
  8229. ],
  8230. "thresholds": [],
  8231. "timeFrom": null,
  8232. "timeRegions": [],
  8233. "timeShift": null,
  8234. "title": "Replication connection close reasons",
  8235. "tooltip": {
  8236. "shared": true,
  8237. "sort": 0,
  8238. "value_type": "individual"
  8239. },
  8240. "type": "graph",
  8241. "xaxis": {
  8242. "buckets": null,
  8243. "mode": "time",
  8244. "name": null,
  8245. "show": true,
  8246. "values": []
  8247. },
  8248. "yaxes": [
  8249. {
  8250. "format": "hertz",
  8251. "label": null,
  8252. "logBase": 1,
  8253. "max": null,
  8254. "min": null,
  8255. "show": true
  8256. },
  8257. {
  8258. "format": "short",
  8259. "label": null,
  8260. "logBase": 1,
  8261. "max": null,
  8262. "min": null,
  8263. "show": true
  8264. }
  8265. ],
  8266. "yaxis": {
  8267. "align": false,
  8268. "alignLevel": null
  8269. }
  8270. }
  8271. ],
  8272. "repeat": null,
  8273. "title": "Replication",
  8274. "type": "row"
  8275. },
  8276. {
  8277. "collapsed": true,
  8278. "datasource": "${DS_PROMETHEUS}",
  8279. "gridPos": {
  8280. "h": 1,
  8281. "w": 24,
  8282. "x": 0,
  8283. "y": 38
  8284. },
  8285. "id": 69,
  8286. "panels": [
  8287. {
  8288. "aliasColors": {},
  8289. "bars": false,
  8290. "dashLength": 10,
  8291. "dashes": false,
  8292. "datasource": "$datasource",
  8293. "fieldConfig": {
  8294. "defaults": {
  8295. "custom": {},
  8296. "links": []
  8297. },
  8298. "overrides": []
  8299. },
  8300. "fill": 1,
  8301. "fillGradient": 0,
  8302. "gridPos": {
  8303. "h": 9,
  8304. "w": 12,
  8305. "x": 0,
  8306. "y": 41
  8307. },
  8308. "hiddenSeries": false,
  8309. "id": 67,
  8310. "legend": {
  8311. "avg": false,
  8312. "current": false,
  8313. "max": false,
  8314. "min": false,
  8315. "show": true,
  8316. "total": false,
  8317. "values": false
  8318. },
  8319. "lines": true,
  8320. "linewidth": 1,
  8321. "links": [],
  8322. "nullPointMode": "connected",
  8323. "options": {
  8324. "alertThreshold": true
  8325. },
  8326. "paceLength": 10,
  8327. "percentage": false,
  8328. "pluginVersion": "7.3.7",
  8329. "pointradius": 5,
  8330. "points": false,
  8331. "renderer": "flot",
  8332. "seriesOverrides": [],
  8333. "spaceLength": 10,
  8334. "stack": false,
  8335. "steppedLine": false,
  8336. "targets": [
  8337. {
  8338. "expr": "max(synapse_event_persisted_position{instance=\"$instance\"}) - on() group_right() synapse_event_processing_positions{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
  8339. "format": "time_series",
  8340. "interval": "",
  8341. "intervalFactor": 1,
  8342. "legendFormat": "{{job}}-{{index}} {{name}}",
  8343. "refId": "A"
  8344. }
  8345. ],
  8346. "thresholds": [],
  8347. "timeFrom": null,
  8348. "timeRegions": [],
  8349. "timeShift": null,
  8350. "title": "Event processing lag",
  8351. "tooltip": {
  8352. "shared": true,
  8353. "sort": 0,
  8354. "value_type": "individual"
  8355. },
  8356. "type": "graph",
  8357. "xaxis": {
  8358. "buckets": null,
  8359. "mode": "time",
  8360. "name": null,
  8361. "show": true,
  8362. "values": []
  8363. },
  8364. "yaxes": [
  8365. {
  8366. "format": "short",
  8367. "label": "events",
  8368. "logBase": 1,
  8369. "max": null,
  8370. "min": "0",
  8371. "show": true
  8372. },
  8373. {
  8374. "format": "short",
  8375. "label": null,
  8376. "logBase": 1,
  8377. "max": null,
  8378. "min": null,
  8379. "show": true
  8380. }
  8381. ],
  8382. "yaxis": {
  8383. "align": false,
  8384. "alignLevel": null
  8385. }
  8386. },
  8387. {
  8388. "aliasColors": {},
  8389. "bars": false,
  8390. "dashLength": 10,
  8391. "dashes": false,
  8392. "datasource": "$datasource",
  8393. "fieldConfig": {
  8394. "defaults": {
  8395. "custom": {},
  8396. "links": []
  8397. },
  8398. "overrides": []
  8399. },
  8400. "fill": 1,
  8401. "fillGradient": 0,
  8402. "gridPos": {
  8403. "h": 9,
  8404. "w": 12,
  8405. "x": 12,
  8406. "y": 41
  8407. },
  8408. "hiddenSeries": false,
  8409. "id": 71,
  8410. "legend": {
  8411. "avg": false,
  8412. "current": false,
  8413. "max": false,
  8414. "min": false,
  8415. "show": true,
  8416. "total": false,
  8417. "values": false
  8418. },
  8419. "lines": true,
  8420. "linewidth": 1,
  8421. "links": [],
  8422. "nullPointMode": "connected",
  8423. "options": {
  8424. "alertThreshold": true
  8425. },
  8426. "paceLength": 10,
  8427. "percentage": false,
  8428. "pluginVersion": "7.3.7",
  8429. "pointradius": 5,
  8430. "points": false,
  8431. "renderer": "flot",
  8432. "seriesOverrides": [],
  8433. "spaceLength": 10,
  8434. "stack": false,
  8435. "steppedLine": false,
  8436. "targets": [
  8437. {
  8438. "expr": "time()*1000-synapse_event_processing_last_ts{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
  8439. "format": "time_series",
  8440. "hide": false,
  8441. "interval": "",
  8442. "intervalFactor": 1,
  8443. "legendFormat": "{{job}}-{{index}} {{name}}",
  8444. "refId": "B"
  8445. }
  8446. ],
  8447. "thresholds": [],
  8448. "timeFrom": null,
  8449. "timeRegions": [],
  8450. "timeShift": null,
  8451. "title": "Age of last processed event",
  8452. "tooltip": {
  8453. "shared": true,
  8454. "sort": 0,
  8455. "value_type": "individual"
  8456. },
  8457. "type": "graph",
  8458. "xaxis": {
  8459. "buckets": null,
  8460. "mode": "time",
  8461. "name": null,
  8462. "show": true,
  8463. "values": []
  8464. },
  8465. "yaxes": [
  8466. {
  8467. "format": "ms",
  8468. "label": null,
  8469. "logBase": 1,
  8470. "max": null,
  8471. "min": "0",
  8472. "show": true
  8473. },
  8474. {
  8475. "format": "short",
  8476. "label": null,
  8477. "logBase": 1,
  8478. "max": null,
  8479. "min": null,
  8480. "show": true
  8481. }
  8482. ],
  8483. "yaxis": {
  8484. "align": false,
  8485. "alignLevel": null
  8486. }
  8487. },
  8488. {
  8489. "aliasColors": {},
  8490. "bars": false,
  8491. "dashLength": 10,
  8492. "dashes": false,
  8493. "datasource": "$datasource",
  8494. "fieldConfig": {
  8495. "defaults": {
  8496. "custom": {},
  8497. "links": []
  8498. },
  8499. "overrides": []
  8500. },
  8501. "fill": 1,
  8502. "fillGradient": 0,
  8503. "gridPos": {
  8504. "h": 9,
  8505. "w": 12,
  8506. "x": 0,
  8507. "y": 50
  8508. },
  8509. "hiddenSeries": false,
  8510. "id": 121,
  8511. "interval": "",
  8512. "legend": {
  8513. "avg": false,
  8514. "current": false,
  8515. "max": false,
  8516. "min": false,
  8517. "show": true,
  8518. "total": false,
  8519. "values": false
  8520. },
  8521. "lines": true,
  8522. "linewidth": 1,
  8523. "links": [],
  8524. "nullPointMode": "connected",
  8525. "options": {
  8526. "alertThreshold": true
  8527. },
  8528. "paceLength": 10,
  8529. "percentage": false,
  8530. "pluginVersion": "7.3.7",
  8531. "pointradius": 5,
  8532. "points": false,
  8533. "renderer": "flot",
  8534. "seriesOverrides": [],
  8535. "spaceLength": 10,
  8536. "stack": false,
  8537. "steppedLine": false,
  8538. "targets": [
  8539. {
  8540. "expr": "deriv(synapse_event_processing_last_ts{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/1000 - 1",
  8541. "format": "time_series",
  8542. "hide": false,
  8543. "interval": "",
  8544. "intervalFactor": 1,
  8545. "legendFormat": "{{job}}-{{index}} {{name}}",
  8546. "refId": "B"
  8547. }
  8548. ],
  8549. "thresholds": [],
  8550. "timeFrom": null,
  8551. "timeRegions": [],
  8552. "timeShift": null,
  8553. "title": "Event processing catchup rate",
  8554. "tooltip": {
  8555. "shared": true,
  8556. "sort": 0,
  8557. "value_type": "individual"
  8558. },
  8559. "type": "graph",
  8560. "xaxis": {
  8561. "buckets": null,
  8562. "mode": "time",
  8563. "name": null,
  8564. "show": true,
  8565. "values": []
  8566. },
  8567. "yaxes": [
  8568. {
  8569. "decimals": null,
  8570. "format": "none",
  8571. "label": "fallbehind(-) / catchup(+): s/sec",
  8572. "logBase": 1,
  8573. "max": null,
  8574. "min": null,
  8575. "show": true
  8576. },
  8577. {
  8578. "format": "short",
  8579. "label": null,
  8580. "logBase": 1,
  8581. "max": null,
  8582. "min": null,
  8583. "show": true
  8584. }
  8585. ],
  8586. "yaxis": {
  8587. "align": false,
  8588. "alignLevel": null
  8589. }
  8590. }
  8591. ],
  8592. "title": "Event processing loop positions",
  8593. "type": "row"
  8594. },
  8595. {
  8596. "collapsed": true,
  8597. "datasource": "${DS_PROMETHEUS}",
  8598. "gridPos": {
  8599. "h": 1,
  8600. "w": 24,
  8601. "x": 0,
  8602. "y": 39
  8603. },
  8604. "id": 126,
  8605. "panels": [
  8606. {
  8607. "cards": {
  8608. "cardPadding": 0,
  8609. "cardRound": null
  8610. },
  8611. "color": {
  8612. "cardColor": "#B877D9",
  8613. "colorScale": "sqrt",
  8614. "colorScheme": "interpolateInferno",
  8615. "exponent": 0.5,
  8616. "max": null,
  8617. "min": 0,
  8618. "mode": "opacity"
  8619. },
  8620. "dataFormat": "tsbuckets",
  8621. "datasource": "$datasource",
  8622. "description": "Colour reflects the number of rooms with the given number of forward extremities, or fewer.\n\nThis is only updated once an hour.",
  8623. "fieldConfig": {
  8624. "defaults": {
  8625. "custom": {}
  8626. },
  8627. "overrides": []
  8628. },
  8629. "gridPos": {
  8630. "h": 8,
  8631. "w": 12,
  8632. "x": 0,
  8633. "y": 42
  8634. },
  8635. "heatmap": {},
  8636. "hideZeroBuckets": true,
  8637. "highlightCards": true,
  8638. "id": 122,
  8639. "legend": {
  8640. "show": true
  8641. },
  8642. "links": [],
  8643. "reverseYBuckets": false,
  8644. "targets": [
  8645. {
  8646. "expr": "synapse_forward_extremities_bucket{instance=\"$instance\"} and on (index, instance, job) (synapse_storage_events_persisted_events > 0)",
  8647. "format": "heatmap",
  8648. "intervalFactor": 1,
  8649. "legendFormat": "{{le}}",
  8650. "refId": "A"
  8651. }
  8652. ],
  8653. "timeFrom": null,
  8654. "timeShift": null,
  8655. "title": "Number of rooms, by number of forward extremities in room",
  8656. "tooltip": {
  8657. "show": true,
  8658. "showHistogram": true
  8659. },
  8660. "type": "heatmap",
  8661. "xAxis": {
  8662. "show": true
  8663. },
  8664. "xBucketNumber": null,
  8665. "xBucketSize": null,
  8666. "yAxis": {
  8667. "decimals": 0,
  8668. "format": "short",
  8669. "logBase": 1,
  8670. "max": null,
  8671. "min": null,
  8672. "show": true,
  8673. "splitFactor": null
  8674. },
  8675. "yBucketBound": "auto",
  8676. "yBucketNumber": null,
  8677. "yBucketSize": null
  8678. },
  8679. {
  8680. "aliasColors": {},
  8681. "bars": false,
  8682. "dashLength": 10,
  8683. "dashes": false,
  8684. "datasource": "$datasource",
  8685. "description": "Number of rooms with the given number of forward extremities or fewer.\n\nThis is only updated once an hour.",
  8686. "fieldConfig": {
  8687. "defaults": {
  8688. "custom": {},
  8689. "links": []
  8690. },
  8691. "overrides": []
  8692. },
  8693. "fill": 0,
  8694. "fillGradient": 0,
  8695. "gridPos": {
  8696. "h": 8,
  8697. "w": 12,
  8698. "x": 12,
  8699. "y": 42
  8700. },
  8701. "hiddenSeries": false,
  8702. "id": 124,
  8703. "interval": "",
  8704. "legend": {
  8705. "avg": false,
  8706. "current": false,
  8707. "max": false,
  8708. "min": false,
  8709. "show": true,
  8710. "total": false,
  8711. "values": false
  8712. },
  8713. "lines": true,
  8714. "linewidth": 1,
  8715. "links": [],
  8716. "nullPointMode": "connected",
  8717. "percentage": false,
  8718. "pluginVersion": "7.1.3",
  8719. "pointradius": 2,
  8720. "points": false,
  8721. "renderer": "flot",
  8722. "seriesOverrides": [],
  8723. "spaceLength": 10,
  8724. "stack": false,
  8725. "steppedLine": false,
  8726. "targets": [
  8727. {
  8728. "expr": "synapse_forward_extremities_bucket{instance=\"$instance\"} > 0",
  8729. "format": "heatmap",
  8730. "interval": "",
  8731. "intervalFactor": 1,
  8732. "legendFormat": "{{le}}",
  8733. "refId": "A"
  8734. }
  8735. ],
  8736. "thresholds": [],
  8737. "timeFrom": null,
  8738. "timeRegions": [],
  8739. "timeShift": null,
  8740. "title": "Room counts, by number of extremities",
  8741. "tooltip": {
  8742. "shared": true,
  8743. "sort": 2,
  8744. "value_type": "individual"
  8745. },
  8746. "type": "graph",
  8747. "xaxis": {
  8748. "buckets": null,
  8749. "mode": "time",
  8750. "name": null,
  8751. "show": true,
  8752. "values": []
  8753. },
  8754. "yaxes": [
  8755. {
  8756. "decimals": null,
  8757. "format": "none",
  8758. "label": "Number of rooms",
  8759. "logBase": 10,
  8760. "max": null,
  8761. "min": null,
  8762. "show": true
  8763. },
  8764. {
  8765. "format": "short",
  8766. "label": null,
  8767. "logBase": 1,
  8768. "max": null,
  8769. "min": null,
  8770. "show": false
  8771. }
  8772. ],
  8773. "yaxis": {
  8774. "align": false,
  8775. "alignLevel": null
  8776. }
  8777. },
  8778. {
  8779. "cards": {
  8780. "cardPadding": 0,
  8781. "cardRound": null
  8782. },
  8783. "color": {
  8784. "cardColor": "#5794F2",
  8785. "colorScale": "sqrt",
  8786. "colorScheme": "interpolateInferno",
  8787. "exponent": 0.5,
  8788. "min": 0,
  8789. "mode": "opacity"
  8790. },
  8791. "dataFormat": "tsbuckets",
  8792. "datasource": "$datasource",
  8793. "description": "Colour reflects the number of events persisted to rooms with the given number of forward extremities, or fewer.",
  8794. "fieldConfig": {
  8795. "defaults": {
  8796. "custom": {}
  8797. },
  8798. "overrides": []
  8799. },
  8800. "gridPos": {
  8801. "h": 8,
  8802. "w": 12,
  8803. "x": 0,
  8804. "y": 50
  8805. },
  8806. "heatmap": {},
  8807. "hideZeroBuckets": true,
  8808. "highlightCards": true,
  8809. "id": 127,
  8810. "legend": {
  8811. "show": true
  8812. },
  8813. "links": [],
  8814. "reverseYBuckets": false,
  8815. "targets": [
  8816. {
  8817. "expr": "rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0)",
  8818. "format": "heatmap",
  8819. "intervalFactor": 1,
  8820. "legendFormat": "{{le}}",
  8821. "refId": "A"
  8822. }
  8823. ],
  8824. "timeFrom": null,
  8825. "timeShift": null,
  8826. "title": "Events persisted, by number of forward extremities in room (heatmap)",
  8827. "tooltip": {
  8828. "show": true,
  8829. "showHistogram": true
  8830. },
  8831. "type": "heatmap",
  8832. "xAxis": {
  8833. "show": true
  8834. },
  8835. "xBucketNumber": null,
  8836. "xBucketSize": null,
  8837. "yAxis": {
  8838. "decimals": 0,
  8839. "format": "short",
  8840. "logBase": 1,
  8841. "max": null,
  8842. "min": null,
  8843. "show": true,
  8844. "splitFactor": null
  8845. },
  8846. "yBucketBound": "auto",
  8847. "yBucketNumber": null,
  8848. "yBucketSize": null
  8849. },
  8850. {
  8851. "aliasColors": {},
  8852. "bars": false,
  8853. "dashLength": 10,
  8854. "dashes": false,
  8855. "datasource": "$datasource",
  8856. "description": "For a given percentage P, the number X where P% of events were persisted to rooms with X forward extremities or fewer.",
  8857. "fieldConfig": {
  8858. "defaults": {
  8859. "custom": {},
  8860. "links": []
  8861. },
  8862. "overrides": []
  8863. },
  8864. "fill": 1,
  8865. "fillGradient": 0,
  8866. "gridPos": {
  8867. "h": 8,
  8868. "w": 12,
  8869. "x": 12,
  8870. "y": 50
  8871. },
  8872. "hiddenSeries": false,
  8873. "id": 128,
  8874. "legend": {
  8875. "avg": false,
  8876. "current": false,
  8877. "max": false,
  8878. "min": false,
  8879. "show": true,
  8880. "total": false,
  8881. "values": false
  8882. },
  8883. "lines": true,
  8884. "linewidth": 1,
  8885. "links": [],
  8886. "nullPointMode": "null",
  8887. "percentage": false,
  8888. "pluginVersion": "7.1.3",
  8889. "pointradius": 2,
  8890. "points": false,
  8891. "renderer": "flot",
  8892. "seriesOverrides": [],
  8893. "spaceLength": 10,
  8894. "stack": false,
  8895. "steppedLine": false,
  8896. "targets": [
  8897. {
  8898. "expr": "histogram_quantile(0.5, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
  8899. "format": "time_series",
  8900. "intervalFactor": 1,
  8901. "legendFormat": "50%",
  8902. "refId": "A"
  8903. },
  8904. {
  8905. "expr": "histogram_quantile(0.75, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
  8906. "format": "time_series",
  8907. "intervalFactor": 1,
  8908. "legendFormat": "75%",
  8909. "refId": "B"
  8910. },
  8911. {
  8912. "expr": "histogram_quantile(0.90, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
  8913. "format": "time_series",
  8914. "intervalFactor": 1,
  8915. "legendFormat": "90%",
  8916. "refId": "C"
  8917. },
  8918. {
  8919. "expr": "histogram_quantile(0.99, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
  8920. "format": "time_series",
  8921. "intervalFactor": 1,
  8922. "legendFormat": "99%",
  8923. "refId": "D"
  8924. }
  8925. ],
  8926. "thresholds": [],
  8927. "timeFrom": null,
  8928. "timeRegions": [],
  8929. "timeShift": null,
  8930. "title": "Events persisted, by number of forward extremities in room (quantiles)",
  8931. "tooltip": {
  8932. "shared": true,
  8933. "sort": 0,
  8934. "value_type": "individual"
  8935. },
  8936. "type": "graph",
  8937. "xaxis": {
  8938. "buckets": null,
  8939. "mode": "time",
  8940. "name": null,
  8941. "show": true,
  8942. "values": []
  8943. },
  8944. "yaxes": [
  8945. {
  8946. "format": "short",
  8947. "label": "Number of extremities in room",
  8948. "logBase": 1,
  8949. "max": null,
  8950. "min": "0",
  8951. "show": true
  8952. },
  8953. {
  8954. "format": "short",
  8955. "label": null,
  8956. "logBase": 1,
  8957. "max": null,
  8958. "min": null,
  8959. "show": true
  8960. }
  8961. ],
  8962. "yaxis": {
  8963. "align": false,
  8964. "alignLevel": null
  8965. }
  8966. },
  8967. {
  8968. "cards": {
  8969. "cardPadding": 0,
  8970. "cardRound": null
  8971. },
  8972. "color": {
  8973. "cardColor": "#FF9830",
  8974. "colorScale": "sqrt",
  8975. "colorScheme": "interpolateInferno",
  8976. "exponent": 0.5,
  8977. "min": 0,
  8978. "mode": "opacity"
  8979. },
  8980. "dataFormat": "tsbuckets",
  8981. "datasource": "$datasource",
  8982. "description": "Colour reflects the number of events persisted to rooms with the given number of stale forward extremities, or fewer.\n\nStale forward extremities are those that were in the previous set of extremities as well as the new.",
  8983. "fieldConfig": {
  8984. "defaults": {
  8985. "custom": {}
  8986. },
  8987. "overrides": []
  8988. },
  8989. "gridPos": {
  8990. "h": 8,
  8991. "w": 12,
  8992. "x": 0,
  8993. "y": 58
  8994. },
  8995. "heatmap": {},
  8996. "hideZeroBuckets": true,
  8997. "highlightCards": true,
  8998. "id": 129,
  8999. "legend": {
  9000. "show": true
  9001. },
  9002. "links": [],
  9003. "reverseYBuckets": false,
  9004. "targets": [
  9005. {
  9006. "expr": "rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0)",
  9007. "format": "heatmap",
  9008. "intervalFactor": 1,
  9009. "legendFormat": "{{le}}",
  9010. "refId": "A"
  9011. }
  9012. ],
  9013. "timeFrom": null,
  9014. "timeShift": null,
  9015. "title": "Events persisted, by number of stale forward extremities in room (heatmap)",
  9016. "tooltip": {
  9017. "show": true,
  9018. "showHistogram": true
  9019. },
  9020. "type": "heatmap",
  9021. "xAxis": {
  9022. "show": true
  9023. },
  9024. "xBucketNumber": null,
  9025. "xBucketSize": null,
  9026. "yAxis": {
  9027. "decimals": 0,
  9028. "format": "short",
  9029. "logBase": 1,
  9030. "max": null,
  9031. "min": null,
  9032. "show": true,
  9033. "splitFactor": null
  9034. },
  9035. "yBucketBound": "auto",
  9036. "yBucketNumber": null,
  9037. "yBucketSize": null
  9038. },
  9039. {
  9040. "aliasColors": {},
  9041. "bars": false,
  9042. "dashLength": 10,
  9043. "dashes": false,
  9044. "datasource": "$datasource",
  9045. "description": "For given percentage P, the number X where P% of events were persisted to rooms with X stale forward extremities or fewer.\n\nStale forward extremities are those that were in the previous set of extremities as well as the new.",
  9046. "fieldConfig": {
  9047. "defaults": {
  9048. "custom": {},
  9049. "links": []
  9050. },
  9051. "overrides": []
  9052. },
  9053. "fill": 1,
  9054. "fillGradient": 0,
  9055. "gridPos": {
  9056. "h": 8,
  9057. "w": 12,
  9058. "x": 12,
  9059. "y": 58
  9060. },
  9061. "hiddenSeries": false,
  9062. "id": 130,
  9063. "legend": {
  9064. "avg": false,
  9065. "current": false,
  9066. "max": false,
  9067. "min": false,
  9068. "show": true,
  9069. "total": false,
  9070. "values": false
  9071. },
  9072. "lines": true,
  9073. "linewidth": 1,
  9074. "links": [],
  9075. "nullPointMode": "null",
  9076. "percentage": false,
  9077. "pluginVersion": "7.1.3",
  9078. "pointradius": 2,
  9079. "points": false,
  9080. "renderer": "flot",
  9081. "seriesOverrides": [],
  9082. "spaceLength": 10,
  9083. "stack": false,
  9084. "steppedLine": false,
  9085. "targets": [
  9086. {
  9087. "expr": "histogram_quantile(0.5, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
  9088. "format": "time_series",
  9089. "intervalFactor": 1,
  9090. "legendFormat": "50%",
  9091. "refId": "A"
  9092. },
  9093. {
  9094. "expr": "histogram_quantile(0.75, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
  9095. "format": "time_series",
  9096. "intervalFactor": 1,
  9097. "legendFormat": "75%",
  9098. "refId": "B"
  9099. },
  9100. {
  9101. "expr": "histogram_quantile(0.90, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
  9102. "format": "time_series",
  9103. "intervalFactor": 1,
  9104. "legendFormat": "90%",
  9105. "refId": "C"
  9106. },
  9107. {
  9108. "expr": "histogram_quantile(0.99, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
  9109. "format": "time_series",
  9110. "intervalFactor": 1,
  9111. "legendFormat": "99%",
  9112. "refId": "D"
  9113. }
  9114. ],
  9115. "thresholds": [],
  9116. "timeFrom": null,
  9117. "timeRegions": [],
  9118. "timeShift": null,
  9119. "title": "Events persisted, by number of stale forward extremities in room (quantiles)",
  9120. "tooltip": {
  9121. "shared": true,
  9122. "sort": 0,
  9123. "value_type": "individual"
  9124. },
  9125. "type": "graph",
  9126. "xaxis": {
  9127. "buckets": null,
  9128. "mode": "time",
  9129. "name": null,
  9130. "show": true,
  9131. "values": []
  9132. },
  9133. "yaxes": [
  9134. {
  9135. "format": "short",
  9136. "label": "Number of stale forward extremities in room",
  9137. "logBase": 1,
  9138. "max": null,
  9139. "min": "0",
  9140. "show": true
  9141. },
  9142. {
  9143. "format": "short",
  9144. "label": null,
  9145. "logBase": 1,
  9146. "max": null,
  9147. "min": null,
  9148. "show": true
  9149. }
  9150. ],
  9151. "yaxis": {
  9152. "align": false,
  9153. "alignLevel": null
  9154. }
  9155. },
  9156. {
  9157. "cards": {
  9158. "cardPadding": 0,
  9159. "cardRound": null
  9160. },
  9161. "color": {
  9162. "cardColor": "#73BF69",
  9163. "colorScale": "sqrt",
  9164. "colorScheme": "interpolateInferno",
  9165. "exponent": 0.5,
  9166. "min": 0,
  9167. "mode": "opacity"
  9168. },
  9169. "dataFormat": "tsbuckets",
  9170. "datasource": "$datasource",
  9171. "description": "Colour reflects the number of state resolution operations performed over the given number of state groups, or fewer.",
  9172. "fieldConfig": {
  9173. "defaults": {
  9174. "custom": {}
  9175. },
  9176. "overrides": []
  9177. },
  9178. "gridPos": {
  9179. "h": 8,
  9180. "w": 12,
  9181. "x": 0,
  9182. "y": 66
  9183. },
  9184. "heatmap": {},
  9185. "hideZeroBuckets": true,
  9186. "highlightCards": true,
  9187. "id": 131,
  9188. "legend": {
  9189. "show": true
  9190. },
  9191. "links": [],
  9192. "reverseYBuckets": false,
  9193. "targets": [
  9194. {
  9195. "expr": "rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
  9196. "format": "heatmap",
  9197. "interval": "",
  9198. "intervalFactor": 1,
  9199. "legendFormat": "{{le}}",
  9200. "refId": "A"
  9201. }
  9202. ],
  9203. "timeFrom": null,
  9204. "timeShift": null,
  9205. "title": "Number of state resolution performed, by number of state groups involved (heatmap)",
  9206. "tooltip": {
  9207. "show": true,
  9208. "showHistogram": true
  9209. },
  9210. "type": "heatmap",
  9211. "xAxis": {
  9212. "show": true
  9213. },
  9214. "xBucketNumber": null,
  9215. "xBucketSize": null,
  9216. "yAxis": {
  9217. "decimals": 0,
  9218. "format": "short",
  9219. "logBase": 1,
  9220. "max": null,
  9221. "min": null,
  9222. "show": true,
  9223. "splitFactor": null
  9224. },
  9225. "yBucketBound": "auto",
  9226. "yBucketNumber": null,
  9227. "yBucketSize": null
  9228. },
  9229. {
  9230. "aliasColors": {},
  9231. "bars": false,
  9232. "dashLength": 10,
  9233. "dashes": false,
  9234. "datasource": "$datasource",
  9235. "description": "For a given percentage P, the number X where P% of state resolution operations took place over X state groups or fewer.",
  9236. "fieldConfig": {
  9237. "defaults": {
  9238. "custom": {},
  9239. "links": []
  9240. },
  9241. "overrides": []
  9242. },
  9243. "fill": 1,
  9244. "fillGradient": 0,
  9245. "gridPos": {
  9246. "h": 8,
  9247. "w": 12,
  9248. "x": 12,
  9249. "y": 66
  9250. },
  9251. "hiddenSeries": false,
  9252. "id": 132,
  9253. "interval": "",
  9254. "legend": {
  9255. "avg": false,
  9256. "current": false,
  9257. "max": false,
  9258. "min": false,
  9259. "show": true,
  9260. "total": false,
  9261. "values": false
  9262. },
  9263. "lines": true,
  9264. "linewidth": 1,
  9265. "links": [],
  9266. "nullPointMode": "null",
  9267. "percentage": false,
  9268. "pluginVersion": "7.1.3",
  9269. "pointradius": 2,
  9270. "points": false,
  9271. "renderer": "flot",
  9272. "seriesOverrides": [],
  9273. "spaceLength": 10,
  9274. "stack": false,
  9275. "steppedLine": false,
  9276. "targets": [
  9277. {
  9278. "expr": "histogram_quantile(0.5, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
  9279. "format": "time_series",
  9280. "interval": "",
  9281. "intervalFactor": 1,
  9282. "legendFormat": "50%",
  9283. "refId": "A"
  9284. },
  9285. {
  9286. "expr": "histogram_quantile(0.75, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
  9287. "format": "time_series",
  9288. "interval": "",
  9289. "intervalFactor": 1,
  9290. "legendFormat": "75%",
  9291. "refId": "B"
  9292. },
  9293. {
  9294. "expr": "histogram_quantile(0.90, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
  9295. "format": "time_series",
  9296. "interval": "",
  9297. "intervalFactor": 1,
  9298. "legendFormat": "90%",
  9299. "refId": "C"
  9300. },
  9301. {
  9302. "expr": "histogram_quantile(0.99, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
  9303. "format": "time_series",
  9304. "interval": "",
  9305. "intervalFactor": 1,
  9306. "legendFormat": "99%",
  9307. "refId": "D"
  9308. }
  9309. ],
  9310. "thresholds": [],
  9311. "timeFrom": null,
  9312. "timeRegions": [],
  9313. "timeShift": null,
  9314. "title": "Number of state resolutions performed, by number of state groups involved (quantiles)",
  9315. "tooltip": {
  9316. "shared": true,
  9317. "sort": 0,
  9318. "value_type": "individual"
  9319. },
  9320. "type": "graph",
  9321. "xaxis": {
  9322. "buckets": null,
  9323. "mode": "time",
  9324. "name": null,
  9325. "show": true,
  9326. "values": []
  9327. },
  9328. "yaxes": [
  9329. {
  9330. "format": "short",
  9331. "label": "Number of state groups",
  9332. "logBase": 1,
  9333. "max": null,
  9334. "min": "0",
  9335. "show": true
  9336. },
  9337. {
  9338. "format": "short",
  9339. "label": null,
  9340. "logBase": 1,
  9341. "max": null,
  9342. "min": null,
  9343. "show": true
  9344. }
  9345. ],
  9346. "yaxis": {
  9347. "align": false,
  9348. "alignLevel": null
  9349. }
  9350. },
  9351. {
  9352. "aliasColors": {},
  9353. "bars": false,
  9354. "dashLength": 10,
  9355. "dashes": false,
  9356. "datasource": "$datasource",
  9357. "description": "When we do a state res while persisting events we try and see if we can prune any stale extremities.",
  9358. "fieldConfig": {
  9359. "defaults": {
  9360. "custom": {}
  9361. },
  9362. "overrides": []
  9363. },
  9364. "fill": 1,
  9365. "fillGradient": 0,
  9366. "gridPos": {
  9367. "h": 8,
  9368. "w": 12,
  9369. "x": 0,
  9370. "y": 74
  9371. },
  9372. "hiddenSeries": false,
  9373. "id": 179,
  9374. "legend": {
  9375. "avg": false,
  9376. "current": false,
  9377. "max": false,
  9378. "min": false,
  9379. "show": true,
  9380. "total": false,
  9381. "values": false
  9382. },
  9383. "lines": true,
  9384. "linewidth": 1,
  9385. "nullPointMode": "null",
  9386. "percentage": false,
  9387. "pluginVersion": "7.1.3",
  9388. "pointradius": 2,
  9389. "points": false,
  9390. "renderer": "flot",
  9391. "seriesOverrides": [],
  9392. "spaceLength": 10,
  9393. "stack": false,
  9394. "steppedLine": false,
  9395. "targets": [
  9396. {
  9397. "expr": "sum(rate(synapse_storage_events_state_resolutions_during_persistence{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
  9398. "interval": "",
  9399. "legendFormat": "State res ",
  9400. "refId": "A"
  9401. },
  9402. {
  9403. "expr": "sum(rate(synapse_storage_events_potential_times_prune_extremities{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
  9404. "interval": "",
  9405. "legendFormat": "Potential to prune",
  9406. "refId": "B"
  9407. },
  9408. {
  9409. "expr": "sum(rate(synapse_storage_events_times_pruned_extremities{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
  9410. "interval": "",
  9411. "legendFormat": "Pruned",
  9412. "refId": "C"
  9413. }
  9414. ],
  9415. "thresholds": [],
  9416. "timeFrom": null,
  9417. "timeRegions": [],
  9418. "timeShift": null,
  9419. "title": "Stale extremity dropping",
  9420. "tooltip": {
  9421. "shared": true,
  9422. "sort": 0,
  9423. "value_type": "individual"
  9424. },
  9425. "type": "graph",
  9426. "xaxis": {
  9427. "buckets": null,
  9428. "mode": "time",
  9429. "name": null,
  9430. "show": true,
  9431. "values": []
  9432. },
  9433. "yaxes": [
  9434. {
  9435. "format": "hertz",
  9436. "label": null,
  9437. "logBase": 1,
  9438. "max": null,
  9439. "min": null,
  9440. "show": true
  9441. },
  9442. {
  9443. "format": "short",
  9444. "label": null,
  9445. "logBase": 1,
  9446. "max": null,
  9447. "min": null,
  9448. "show": true
  9449. }
  9450. ],
  9451. "yaxis": {
  9452. "align": false,
  9453. "alignLevel": null
  9454. }
  9455. }
  9456. ],
  9457. "title": "Extremities",
  9458. "type": "row"
  9459. },
  9460. {
  9461. "collapsed": true,
  9462. "datasource": "${DS_PROMETHEUS}",
  9463. "gridPos": {
  9464. "h": 1,
  9465. "w": 24,
  9466. "x": 0,
  9467. "y": 40
  9468. },
  9469. "id": 158,
  9470. "panels": [
  9471. {
  9472. "aliasColors": {},
  9473. "bars": false,
  9474. "dashLength": 10,
  9475. "dashes": false,
  9476. "datasource": "$datasource",
  9477. "fieldConfig": {
  9478. "defaults": {
  9479. "custom": {},
  9480. "links": []
  9481. },
  9482. "overrides": []
  9483. },
  9484. "fill": 1,
  9485. "fillGradient": 0,
  9486. "gridPos": {
  9487. "h": 8,
  9488. "w": 12,
  9489. "x": 0,
  9490. "y": 41
  9491. },
  9492. "hiddenSeries": false,
  9493. "id": 156,
  9494. "legend": {
  9495. "avg": false,
  9496. "current": false,
  9497. "max": false,
  9498. "min": false,
  9499. "show": true,
  9500. "total": false,
  9501. "values": false
  9502. },
  9503. "lines": true,
  9504. "linewidth": 1,
  9505. "links": [],
  9506. "nullPointMode": "null",
  9507. "options": {
  9508. "alertThreshold": true
  9509. },
  9510. "percentage": false,
  9511. "pluginVersion": "7.3.7",
  9512. "pointradius": 5,
  9513. "points": false,
  9514. "renderer": "flot",
  9515. "seriesOverrides": [
  9516. {
  9517. "alias": "Max",
  9518. "color": "#bf1b00",
  9519. "fill": 0,
  9520. "linewidth": 2
  9521. }
  9522. ],
  9523. "spaceLength": 10,
  9524. "stack": false,
  9525. "steppedLine": false,
  9526. "targets": [
  9527. {
  9528. "expr": "synapse_admin_mau:current{instance=\"$instance\", job=~\"$job\"}",
  9529. "format": "time_series",
  9530. "interval": "",
  9531. "intervalFactor": 1,
  9532. "legendFormat": "Current",
  9533. "refId": "A"
  9534. },
  9535. {
  9536. "expr": "synapse_admin_mau:max{instance=\"$instance\", job=~\"$job\"}",
  9537. "format": "time_series",
  9538. "interval": "",
  9539. "intervalFactor": 1,
  9540. "legendFormat": "Max",
  9541. "refId": "B"
  9542. }
  9543. ],
  9544. "thresholds": [],
  9545. "timeFrom": null,
  9546. "timeRegions": [],
  9547. "timeShift": null,
  9548. "title": "MAU Limits",
  9549. "tooltip": {
  9550. "shared": true,
  9551. "sort": 0,
  9552. "value_type": "individual"
  9553. },
  9554. "type": "graph",
  9555. "xaxis": {
  9556. "buckets": null,
  9557. "mode": "time",
  9558. "name": null,
  9559. "show": true,
  9560. "values": []
  9561. },
  9562. "yaxes": [
  9563. {
  9564. "format": "short",
  9565. "label": null,
  9566. "logBase": 1,
  9567. "max": null,
  9568. "min": "0",
  9569. "show": true
  9570. },
  9571. {
  9572. "format": "short",
  9573. "label": null,
  9574. "logBase": 1,
  9575. "max": null,
  9576. "min": null,
  9577. "show": true
  9578. }
  9579. ],
  9580. "yaxis": {
  9581. "align": false,
  9582. "alignLevel": null
  9583. }
  9584. },
  9585. {
  9586. "aliasColors": {},
  9587. "bars": false,
  9588. "dashLength": 10,
  9589. "dashes": false,
  9590. "datasource": "$datasource",
  9591. "fieldConfig": {
  9592. "defaults": {
  9593. "custom": {}
  9594. },
  9595. "overrides": []
  9596. },
  9597. "fill": 1,
  9598. "fillGradient": 0,
  9599. "gridPos": {
  9600. "h": 8,
  9601. "w": 12,
  9602. "x": 12,
  9603. "y": 41
  9604. },
  9605. "hiddenSeries": false,
  9606. "id": 160,
  9607. "legend": {
  9608. "avg": false,
  9609. "current": false,
  9610. "max": false,
  9611. "min": false,
  9612. "show": true,
  9613. "total": false,
  9614. "values": false
  9615. },
  9616. "lines": true,
  9617. "linewidth": 1,
  9618. "nullPointMode": "null",
  9619. "options": {
  9620. "alertThreshold": true
  9621. },
  9622. "percentage": false,
  9623. "pluginVersion": "7.3.7",
  9624. "pointradius": 2,
  9625. "points": false,
  9626. "renderer": "flot",
  9627. "seriesOverrides": [],
  9628. "spaceLength": 10,
  9629. "stack": false,
  9630. "steppedLine": false,
  9631. "targets": [
  9632. {
  9633. "expr": "synapse_admin_mau_current_mau_by_service{instance=\"$instance\"}",
  9634. "interval": "",
  9635. "legendFormat": "{{ app_service }}",
  9636. "refId": "A"
  9637. }
  9638. ],
  9639. "thresholds": [],
  9640. "timeFrom": null,
  9641. "timeRegions": [],
  9642. "timeShift": null,
  9643. "title": "MAU by Appservice",
  9644. "tooltip": {
  9645. "shared": true,
  9646. "sort": 0,
  9647. "value_type": "individual"
  9648. },
  9649. "type": "graph",
  9650. "xaxis": {
  9651. "buckets": null,
  9652. "mode": "time",
  9653. "name": null,
  9654. "show": true,
  9655. "values": []
  9656. },
  9657. "yaxes": [
  9658. {
  9659. "format": "short",
  9660. "label": null,
  9661. "logBase": 1,
  9662. "max": null,
  9663. "min": null,
  9664. "show": true
  9665. },
  9666. {
  9667. "format": "short",
  9668. "label": null,
  9669. "logBase": 1,
  9670. "max": null,
  9671. "min": null,
  9672. "show": true
  9673. }
  9674. ],
  9675. "yaxis": {
  9676. "align": false,
  9677. "alignLevel": null
  9678. }
  9679. }
  9680. ],
  9681. "title": "MAU",
  9682. "type": "row"
  9683. },
  9684. {
  9685. "collapsed": true,
  9686. "datasource": "${DS_PROMETHEUS}",
  9687. "gridPos": {
  9688. "h": 1,
  9689. "w": 24,
  9690. "x": 0,
  9691. "y": 41
  9692. },
  9693. "id": 177,
  9694. "panels": [
  9695. {
  9696. "aliasColors": {},
  9697. "bars": false,
  9698. "dashLength": 10,
  9699. "dashes": false,
  9700. "datasource": "$datasource",
  9701. "fieldConfig": {
  9702. "defaults": {
  9703. "custom": {},
  9704. "links": []
  9705. },
  9706. "overrides": []
  9707. },
  9708. "fill": 1,
  9709. "fillGradient": 0,
  9710. "gridPos": {
  9711. "h": 7,
  9712. "w": 12,
  9713. "x": 0,
  9714. "y": 1
  9715. },
  9716. "hiddenSeries": false,
  9717. "id": 173,
  9718. "legend": {
  9719. "avg": false,
  9720. "current": false,
  9721. "max": false,
  9722. "min": false,
  9723. "show": true,
  9724. "total": false,
  9725. "values": false
  9726. },
  9727. "lines": true,
  9728. "linewidth": 1,
  9729. "links": [],
  9730. "nullPointMode": "null",
  9731. "percentage": false,
  9732. "pluginVersion": "7.1.3",
  9733. "pointradius": 5,
  9734. "points": false,
  9735. "renderer": "flot",
  9736. "seriesOverrides": [],
  9737. "spaceLength": 10,
  9738. "stack": false,
  9739. "steppedLine": false,
  9740. "targets": [
  9741. {
  9742. "expr": "rate(synapse_notifier_users_woken_by_stream{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
  9743. "format": "time_series",
  9744. "hide": false,
  9745. "intervalFactor": 2,
  9746. "legendFormat": "{{stream}} {{index}}",
  9747. "metric": "synapse_notifier",
  9748. "refId": "A",
  9749. "step": 2
  9750. }
  9751. ],
  9752. "thresholds": [],
  9753. "timeFrom": null,
  9754. "timeRegions": [],
  9755. "timeShift": null,
  9756. "title": "Notifier Streams Woken",
  9757. "tooltip": {
  9758. "shared": true,
  9759. "sort": 2,
  9760. "value_type": "individual"
  9761. },
  9762. "type": "graph",
  9763. "xaxis": {
  9764. "buckets": null,
  9765. "mode": "time",
  9766. "name": null,
  9767. "show": true,
  9768. "values": []
  9769. },
  9770. "yaxes": [
  9771. {
  9772. "format": "hertz",
  9773. "label": null,
  9774. "logBase": 1,
  9775. "max": null,
  9776. "min": null,
  9777. "show": true
  9778. },
  9779. {
  9780. "format": "short",
  9781. "label": null,
  9782. "logBase": 1,
  9783. "max": null,
  9784. "min": null,
  9785. "show": true
  9786. }
  9787. ],
  9788. "yaxis": {
  9789. "align": false,
  9790. "alignLevel": null
  9791. }
  9792. },
  9793. {
  9794. "aliasColors": {},
  9795. "bars": false,
  9796. "dashLength": 10,
  9797. "dashes": false,
  9798. "datasource": "$datasource",
  9799. "fieldConfig": {
  9800. "defaults": {
  9801. "custom": {},
  9802. "links": []
  9803. },
  9804. "overrides": []
  9805. },
  9806. "fill": 1,
  9807. "fillGradient": 0,
  9808. "gridPos": {
  9809. "h": 7,
  9810. "w": 12,
  9811. "x": 12,
  9812. "y": 1
  9813. },
  9814. "hiddenSeries": false,
  9815. "id": 175,
  9816. "legend": {
  9817. "avg": false,
  9818. "current": false,
  9819. "max": false,
  9820. "min": false,
  9821. "show": true,
  9822. "total": false,
  9823. "values": false
  9824. },
  9825. "lines": true,
  9826. "linewidth": 1,
  9827. "links": [],
  9828. "nullPointMode": "null",
  9829. "percentage": false,
  9830. "pluginVersion": "7.1.3",
  9831. "pointradius": 5,
  9832. "points": false,
  9833. "renderer": "flot",
  9834. "seriesOverrides": [],
  9835. "spaceLength": 10,
  9836. "stack": false,
  9837. "steppedLine": false,
  9838. "targets": [
  9839. {
  9840. "expr": "rate(synapse_handler_presence_get_updates{job=~\"$job\",instance=\"$instance\"}[$bucket_size])",
  9841. "format": "time_series",
  9842. "interval": "",
  9843. "intervalFactor": 2,
  9844. "legendFormat": "{{type}} {{index}}",
  9845. "refId": "A",
  9846. "step": 2
  9847. }
  9848. ],
  9849. "thresholds": [],
  9850. "timeFrom": null,
  9851. "timeRegions": [],
  9852. "timeShift": null,
  9853. "title": "Presence Stream Fetch Type Rates",
  9854. "tooltip": {
  9855. "shared": true,
  9856. "sort": 2,
  9857. "value_type": "individual"
  9858. },
  9859. "type": "graph",
  9860. "xaxis": {
  9861. "buckets": null,
  9862. "mode": "time",
  9863. "name": null,
  9864. "show": true,
  9865. "values": []
  9866. },
  9867. "yaxes": [
  9868. {
  9869. "format": "hertz",
  9870. "label": null,
  9871. "logBase": 1,
  9872. "max": null,
  9873. "min": "0",
  9874. "show": true
  9875. },
  9876. {
  9877. "format": "short",
  9878. "label": null,
  9879. "logBase": 1,
  9880. "max": null,
  9881. "min": null,
  9882. "show": true
  9883. }
  9884. ],
  9885. "yaxis": {
  9886. "align": false,
  9887. "alignLevel": null
  9888. }
  9889. }
  9890. ],
  9891. "title": "Notifier",
  9892. "type": "row"
  9893. },
  9894. {
  9895. "collapsed": true,
  9896. "datasource": "${DS_PROMETHEUS}",
  9897. "gridPos": {
  9898. "h": 1,
  9899. "w": 24,
  9900. "x": 0,
  9901. "y": 42
  9902. },
  9903. "id": 170,
  9904. "panels": [
  9905. {
  9906. "aliasColors": {},
  9907. "bars": false,
  9908. "dashLength": 10,
  9909. "dashes": false,
  9910. "datasource": "$datasource",
  9911. "fieldConfig": {
  9912. "defaults": {
  9913. "custom": {}
  9914. },
  9915. "overrides": []
  9916. },
  9917. "fill": 1,
  9918. "fillGradient": 0,
  9919. "gridPos": {
  9920. "h": 8,
  9921. "w": 12,
  9922. "x": 0,
  9923. "y": 43
  9924. },
  9925. "hiddenSeries": false,
  9926. "id": 168,
  9927. "legend": {
  9928. "avg": false,
  9929. "current": false,
  9930. "max": false,
  9931. "min": false,
  9932. "show": true,
  9933. "total": false,
  9934. "values": false
  9935. },
  9936. "lines": true,
  9937. "linewidth": 1,
  9938. "nullPointMode": "null",
  9939. "options": {
  9940. "alertThreshold": true
  9941. },
  9942. "percentage": false,
  9943. "pluginVersion": "7.3.7",
  9944. "pointradius": 2,
  9945. "points": false,
  9946. "renderer": "flot",
  9947. "seriesOverrides": [],
  9948. "spaceLength": 10,
  9949. "stack": false,
  9950. "steppedLine": false,
  9951. "targets": [
  9952. {
  9953. "expr": "rate(synapse_appservice_api_sent_events{instance=\"$instance\"}[$bucket_size])",
  9954. "interval": "",
  9955. "legendFormat": "{{service}}",
  9956. "refId": "A"
  9957. }
  9958. ],
  9959. "thresholds": [],
  9960. "timeFrom": null,
  9961. "timeRegions": [],
  9962. "timeShift": null,
  9963. "title": "Sent Events rate",
  9964. "tooltip": {
  9965. "shared": true,
  9966. "sort": 0,
  9967. "value_type": "individual"
  9968. },
  9969. "type": "graph",
  9970. "xaxis": {
  9971. "buckets": null,
  9972. "mode": "time",
  9973. "name": null,
  9974. "show": true,
  9975. "values": []
  9976. },
  9977. "yaxes": [
  9978. {
  9979. "format": "hertz",
  9980. "label": null,
  9981. "logBase": 1,
  9982. "max": null,
  9983. "min": null,
  9984. "show": true
  9985. },
  9986. {
  9987. "format": "short",
  9988. "label": null,
  9989. "logBase": 1,
  9990. "max": null,
  9991. "min": null,
  9992. "show": true
  9993. }
  9994. ],
  9995. "yaxis": {
  9996. "align": false,
  9997. "alignLevel": null
  9998. }
  9999. },
  10000. {
  10001. "aliasColors": {},
  10002. "bars": false,
  10003. "dashLength": 10,
  10004. "dashes": false,
  10005. "datasource": "$datasource",
  10006. "fieldConfig": {
  10007. "defaults": {
  10008. "custom": {}
  10009. },
  10010. "overrides": []
  10011. },
  10012. "fill": 1,
  10013. "fillGradient": 0,
  10014. "gridPos": {
  10015. "h": 8,
  10016. "w": 12,
  10017. "x": 12,
  10018. "y": 43
  10019. },
  10020. "hiddenSeries": false,
  10021. "id": 171,
  10022. "legend": {
  10023. "avg": false,
  10024. "current": false,
  10025. "max": false,
  10026. "min": false,
  10027. "show": true,
  10028. "total": false,
  10029. "values": false
  10030. },
  10031. "lines": true,
  10032. "linewidth": 1,
  10033. "nullPointMode": "null",
  10034. "options": {
  10035. "alertThreshold": true
  10036. },
  10037. "percentage": false,
  10038. "pluginVersion": "7.3.7",
  10039. "pointradius": 2,
  10040. "points": false,
  10041. "renderer": "flot",
  10042. "seriesOverrides": [],
  10043. "spaceLength": 10,
  10044. "stack": false,
  10045. "steppedLine": false,
  10046. "targets": [
  10047. {
  10048. "expr": "rate(synapse_appservice_api_sent_transactions{instance=\"$instance\"}[$bucket_size])",
  10049. "interval": "",
  10050. "legendFormat": "{{service}}",
  10051. "refId": "A"
  10052. }
  10053. ],
  10054. "thresholds": [],
  10055. "timeFrom": null,
  10056. "timeRegions": [],
  10057. "timeShift": null,
  10058. "title": "Transactions rate",
  10059. "tooltip": {
  10060. "shared": true,
  10061. "sort": 0,
  10062. "value_type": "individual"
  10063. },
  10064. "type": "graph",
  10065. "xaxis": {
  10066. "buckets": null,
  10067. "mode": "time",
  10068. "name": null,
  10069. "show": true,
  10070. "values": []
  10071. },
  10072. "yaxes": [
  10073. {
  10074. "format": "hertz",
  10075. "label": null,
  10076. "logBase": 1,
  10077. "max": null,
  10078. "min": null,
  10079. "show": true
  10080. },
  10081. {
  10082. "format": "short",
  10083. "label": null,
  10084. "logBase": 1,
  10085. "max": null,
  10086. "min": null,
  10087. "show": true
  10088. }
  10089. ],
  10090. "yaxis": {
  10091. "align": false,
  10092. "alignLevel": null
  10093. }
  10094. }
  10095. ],
  10096. "title": "Appservices",
  10097. "type": "row"
  10098. },
  10099. {
  10100. "collapsed": true,
  10101. "datasource": "${DS_PROMETHEUS}",
  10102. "gridPos": {
  10103. "h": 1,
  10104. "w": 24,
  10105. "x": 0,
  10106. "y": 43
  10107. },
  10108. "id": 188,
  10109. "panels": [
  10110. {
  10111. "aliasColors": {},
  10112. "bars": false,
  10113. "dashLength": 10,
  10114. "dashes": false,
  10115. "datasource": "$datasource",
  10116. "fieldConfig": {
  10117. "defaults": {
  10118. "custom": {}
  10119. },
  10120. "overrides": []
  10121. },
  10122. "fill": 1,
  10123. "fillGradient": 0,
  10124. "gridPos": {
  10125. "h": 8,
  10126. "w": 12,
  10127. "x": 0,
  10128. "y": 44
  10129. },
  10130. "hiddenSeries": false,
  10131. "id": 182,
  10132. "legend": {
  10133. "avg": false,
  10134. "current": false,
  10135. "max": false,
  10136. "min": false,
  10137. "show": true,
  10138. "total": false,
  10139. "values": false
  10140. },
  10141. "lines": true,
  10142. "linewidth": 1,
  10143. "nullPointMode": "null",
  10144. "options": {
  10145. "alertThreshold": true
  10146. },
  10147. "percentage": false,
  10148. "pluginVersion": "7.3.7",
  10149. "pointradius": 2,
  10150. "points": false,
  10151. "renderer": "flot",
  10152. "seriesOverrides": [],
  10153. "spaceLength": 10,
  10154. "stack": false,
  10155. "steppedLine": false,
  10156. "targets": [
  10157. {
  10158. "expr": "rate(synapse_handler_presence_notified_presence{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
  10159. "interval": "",
  10160. "legendFormat": "Notified",
  10161. "refId": "A"
  10162. },
  10163. {
  10164. "expr": "rate(synapse_handler_presence_federation_presence_out{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
  10165. "interval": "",
  10166. "legendFormat": "Remote ping",
  10167. "refId": "B"
  10168. },
  10169. {
  10170. "expr": "rate(synapse_handler_presence_presence_updates{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
  10171. "interval": "",
  10172. "legendFormat": "Total updates",
  10173. "refId": "C"
  10174. },
  10175. {
  10176. "expr": "rate(synapse_handler_presence_federation_presence{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
  10177. "interval": "",
  10178. "legendFormat": "Remote updates",
  10179. "refId": "D"
  10180. },
  10181. {
  10182. "expr": "rate(synapse_handler_presence_bump_active_time{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
  10183. "interval": "",
  10184. "legendFormat": "Bump active time",
  10185. "refId": "E"
  10186. }
  10187. ],
  10188. "thresholds": [],
  10189. "timeFrom": null,
  10190. "timeRegions": [],
  10191. "timeShift": null,
  10192. "title": "Presence",
  10193. "tooltip": {
  10194. "shared": true,
  10195. "sort": 2,
  10196. "value_type": "individual"
  10197. },
  10198. "type": "graph",
  10199. "xaxis": {
  10200. "buckets": null,
  10201. "mode": "time",
  10202. "name": null,
  10203. "show": true,
  10204. "values": []
  10205. },
  10206. "yaxes": [
  10207. {
  10208. "format": "hertz",
  10209. "label": null,
  10210. "logBase": 1,
  10211. "max": null,
  10212. "min": null,
  10213. "show": true
  10214. },
  10215. {
  10216. "format": "short",
  10217. "label": null,
  10218. "logBase": 1,
  10219. "max": null,
  10220. "min": null,
  10221. "show": true
  10222. }
  10223. ],
  10224. "yaxis": {
  10225. "align": false,
  10226. "alignLevel": null
  10227. }
  10228. },
  10229. {
  10230. "aliasColors": {},
  10231. "bars": false,
  10232. "dashLength": 10,
  10233. "dashes": false,
  10234. "datasource": "$datasource",
  10235. "fieldConfig": {
  10236. "defaults": {
  10237. "custom": {}
  10238. },
  10239. "overrides": []
  10240. },
  10241. "fill": 1,
  10242. "fillGradient": 0,
  10243. "gridPos": {
  10244. "h": 8,
  10245. "w": 12,
  10246. "x": 12,
  10247. "y": 44
  10248. },
  10249. "hiddenSeries": false,
  10250. "id": 184,
  10251. "legend": {
  10252. "avg": false,
  10253. "current": false,
  10254. "max": false,
  10255. "min": false,
  10256. "show": true,
  10257. "total": false,
  10258. "values": false
  10259. },
  10260. "lines": true,
  10261. "linewidth": 1,
  10262. "nullPointMode": "null",
  10263. "options": {
  10264. "alertThreshold": true
  10265. },
  10266. "percentage": false,
  10267. "pluginVersion": "7.3.7",
  10268. "pointradius": 2,
  10269. "points": false,
  10270. "renderer": "flot",
  10271. "seriesOverrides": [],
  10272. "spaceLength": 10,
  10273. "stack": false,
  10274. "steppedLine": false,
  10275. "targets": [
  10276. {
  10277. "expr": "rate(synapse_handler_presence_state_transition{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
  10278. "interval": "",
  10279. "legendFormat": "{{from}} -> {{to}}",
  10280. "refId": "A"
  10281. }
  10282. ],
  10283. "thresholds": [],
  10284. "timeFrom": null,
  10285. "timeRegions": [],
  10286. "timeShift": null,
  10287. "title": "Presence state transitions",
  10288. "tooltip": {
  10289. "shared": true,
  10290. "sort": 2,
  10291. "value_type": "individual"
  10292. },
  10293. "type": "graph",
  10294. "xaxis": {
  10295. "buckets": null,
  10296. "mode": "time",
  10297. "name": null,
  10298. "show": true,
  10299. "values": []
  10300. },
  10301. "yaxes": [
  10302. {
  10303. "format": "hertz",
  10304. "label": null,
  10305. "logBase": 1,
  10306. "max": null,
  10307. "min": null,
  10308. "show": true
  10309. },
  10310. {
  10311. "format": "short",
  10312. "label": null,
  10313. "logBase": 1,
  10314. "max": null,
  10315. "min": null,
  10316. "show": true
  10317. }
  10318. ],
  10319. "yaxis": {
  10320. "align": false,
  10321. "alignLevel": null
  10322. }
  10323. },
  10324. {
  10325. "aliasColors": {},
  10326. "bars": false,
  10327. "dashLength": 10,
  10328. "dashes": false,
  10329. "datasource": "$datasource",
  10330. "fieldConfig": {
  10331. "defaults": {
  10332. "custom": {}
  10333. },
  10334. "overrides": []
  10335. },
  10336. "fill": 1,
  10337. "fillGradient": 0,
  10338. "gridPos": {
  10339. "h": 8,
  10340. "w": 12,
  10341. "x": 0,
  10342. "y": 52
  10343. },
  10344. "hiddenSeries": false,
  10345. "id": 186,
  10346. "legend": {
  10347. "avg": false,
  10348. "current": false,
  10349. "max": false,
  10350. "min": false,
  10351. "show": true,
  10352. "total": false,
  10353. "values": false
  10354. },
  10355. "lines": true,
  10356. "linewidth": 1,
  10357. "nullPointMode": "null",
  10358. "options": {
  10359. "alertThreshold": true
  10360. },
  10361. "percentage": false,
  10362. "pluginVersion": "7.3.7",
  10363. "pointradius": 2,
  10364. "points": false,
  10365. "renderer": "flot",
  10366. "seriesOverrides": [],
  10367. "spaceLength": 10,
  10368. "stack": false,
  10369. "steppedLine": false,
  10370. "targets": [
  10371. {
  10372. "expr": "rate(synapse_handler_presence_notify_reason{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
  10373. "interval": "",
  10374. "legendFormat": "{{reason}}",
  10375. "refId": "A"
  10376. }
  10377. ],
  10378. "thresholds": [],
  10379. "timeFrom": null,
  10380. "timeRegions": [],
  10381. "timeShift": null,
  10382. "title": "Presence notify reason",
  10383. "tooltip": {
  10384. "shared": true,
  10385. "sort": 2,
  10386. "value_type": "individual"
  10387. },
  10388. "type": "graph",
  10389. "xaxis": {
  10390. "buckets": null,
  10391. "mode": "time",
  10392. "name": null,
  10393. "show": true,
  10394. "values": []
  10395. },
  10396. "yaxes": [
  10397. {
  10398. "format": "hertz",
  10399. "label": null,
  10400. "logBase": 1,
  10401. "max": null,
  10402. "min": null,
  10403. "show": true
  10404. },
  10405. {
  10406. "format": "short",
  10407. "label": null,
  10408. "logBase": 1,
  10409. "max": null,
  10410. "min": null,
  10411. "show": true
  10412. }
  10413. ],
  10414. "yaxis": {
  10415. "align": false,
  10416. "alignLevel": null
  10417. }
  10418. }
  10419. ],
  10420. "title": "Presence",
  10421. "type": "row"
  10422. },
  10423. {
  10424. "collapsed": true,
  10425. "datasource": "${DS_PROMETHEUS}",
  10426. "gridPos": {
  10427. "h": 1,
  10428. "w": 24,
  10429. "x": 0,
  10430. "y": 44
  10431. },
  10432. "id": 197,
  10433. "panels": [
  10434. {
  10435. "aliasColors": {},
  10436. "bars": false,
  10437. "dashLength": 10,
  10438. "dashes": false,
  10439. "datasource": "$datasource",
  10440. "fieldConfig": {
  10441. "defaults": {
  10442. "custom": {}
  10443. },
  10444. "overrides": []
  10445. },
  10446. "fill": 1,
  10447. "fillGradient": 0,
  10448. "gridPos": {
  10449. "h": 8,
  10450. "w": 12,
  10451. "x": 0,
  10452. "y": 1
  10453. },
  10454. "hiddenSeries": false,
  10455. "id": 191,
  10456. "legend": {
  10457. "avg": false,
  10458. "current": false,
  10459. "max": false,
  10460. "min": false,
  10461. "show": true,
  10462. "total": false,
  10463. "values": false
  10464. },
  10465. "lines": true,
  10466. "linewidth": 1,
  10467. "nullPointMode": "null",
  10468. "options": {
  10469. "alertThreshold": true
  10470. },
  10471. "percentage": false,
  10472. "pluginVersion": "7.3.7",
  10473. "pointradius": 2,
  10474. "points": false,
  10475. "renderer": "flot",
  10476. "seriesOverrides": [],
  10477. "spaceLength": 10,
  10478. "stack": false,
  10479. "steppedLine": false,
  10480. "targets": [
  10481. {
  10482. "expr": "rate(synapse_external_cache_set{job=\"$job\", instance=\"$instance\", index=~\"$index\"}[$bucket_size])",
  10483. "interval": "",
  10484. "legendFormat": "{{ cache_name }} {{ index }}",
  10485. "refId": "A"
  10486. }
  10487. ],
  10488. "thresholds": [],
  10489. "timeFrom": null,
  10490. "timeRegions": [],
  10491. "timeShift": null,
  10492. "title": "External Cache Set Rate",
  10493. "tooltip": {
  10494. "shared": true,
  10495. "sort": 2,
  10496. "value_type": "individual"
  10497. },
  10498. "type": "graph",
  10499. "xaxis": {
  10500. "buckets": null,
  10501. "mode": "time",
  10502. "name": null,
  10503. "show": true,
  10504. "values": []
  10505. },
  10506. "yaxes": [
  10507. {
  10508. "format": "hertz",
  10509. "label": null,
  10510. "logBase": 1,
  10511. "max": null,
  10512. "min": null,
  10513. "show": true
  10514. },
  10515. {
  10516. "format": "short",
  10517. "label": null,
  10518. "logBase": 1,
  10519. "max": null,
  10520. "min": null,
  10521. "show": true
  10522. }
  10523. ],
  10524. "yaxis": {
  10525. "align": false,
  10526. "alignLevel": null
  10527. }
  10528. },
  10529. {
  10530. "aliasColors": {},
  10531. "bars": false,
  10532. "dashLength": 10,
  10533. "dashes": false,
  10534. "datasource": "$datasource",
  10535. "description": "",
  10536. "fieldConfig": {
  10537. "defaults": {
  10538. "custom": {}
  10539. },
  10540. "overrides": []
  10541. },
  10542. "fill": 1,
  10543. "fillGradient": 0,
  10544. "gridPos": {
  10545. "h": 8,
  10546. "w": 12,
  10547. "x": 12,
  10548. "y": 1
  10549. },
  10550. "hiddenSeries": false,
  10551. "id": 193,
  10552. "legend": {
  10553. "avg": false,
  10554. "current": false,
  10555. "max": false,
  10556. "min": false,
  10557. "show": true,
  10558. "total": false,
  10559. "values": false
  10560. },
  10561. "lines": true,
  10562. "linewidth": 1,
  10563. "nullPointMode": "null",
  10564. "options": {
  10565. "alertThreshold": true
  10566. },
  10567. "percentage": false,
  10568. "pluginVersion": "7.3.7",
  10569. "pointradius": 2,
  10570. "points": false,
  10571. "renderer": "flot",
  10572. "seriesOverrides": [],
  10573. "spaceLength": 10,
  10574. "stack": false,
  10575. "steppedLine": false,
  10576. "targets": [
  10577. {
  10578. "expr": "rate(synapse_external_cache_get{job=\"$job\", instance=\"$instance\", index=~\"$index\"}[$bucket_size])",
  10579. "interval": "",
  10580. "legendFormat": "{{ cache_name }} {{ index }}",
  10581. "refId": "A"
  10582. }
  10583. ],
  10584. "thresholds": [],
  10585. "timeFrom": null,
  10586. "timeRegions": [],
  10587. "timeShift": null,
  10588. "title": "External Cache Get Rate",
  10589. "tooltip": {
  10590. "shared": true,
  10591. "sort": 2,
  10592. "value_type": "individual"
  10593. },
  10594. "type": "graph",
  10595. "xaxis": {
  10596. "buckets": null,
  10597. "mode": "time",
  10598. "name": null,
  10599. "show": true,
  10600. "values": []
  10601. },
  10602. "yaxes": [
  10603. {
  10604. "format": "hertz",
  10605. "label": null,
  10606. "logBase": 1,
  10607. "max": null,
  10608. "min": null,
  10609. "show": true
  10610. },
  10611. {
  10612. "format": "short",
  10613. "label": null,
  10614. "logBase": 1,
  10615. "max": null,
  10616. "min": null,
  10617. "show": true
  10618. }
  10619. ],
  10620. "yaxis": {
  10621. "align": false,
  10622. "alignLevel": null
  10623. }
  10624. },
  10625. {
  10626. "cards": {
  10627. "cardPadding": -1,
  10628. "cardRound": null
  10629. },
  10630. "color": {
  10631. "cardColor": "#b4ff00",
  10632. "colorScale": "sqrt",
  10633. "colorScheme": "interpolateInferno",
  10634. "exponent": 0.5,
  10635. "min": 0,
  10636. "mode": "spectrum"
  10637. },
  10638. "dataFormat": "tsbuckets",
  10639. "datasource": "$datasource",
  10640. "fieldConfig": {
  10641. "defaults": {
  10642. "custom": {}
  10643. },
  10644. "overrides": []
  10645. },
  10646. "gridPos": {
  10647. "h": 9,
  10648. "w": 12,
  10649. "x": 0,
  10650. "y": 9
  10651. },
  10652. "heatmap": {},
  10653. "hideZeroBuckets": false,
  10654. "highlightCards": true,
  10655. "id": 195,
  10656. "legend": {
  10657. "show": false
  10658. },
  10659. "links": [],
  10660. "reverseYBuckets": false,
  10661. "targets": [
  10662. {
  10663. "expr": "sum(rate(synapse_external_cache_response_time_seconds_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le)",
  10664. "format": "heatmap",
  10665. "instant": false,
  10666. "interval": "",
  10667. "intervalFactor": 1,
  10668. "legendFormat": "{{le}}",
  10669. "refId": "A"
  10670. }
  10671. ],
  10672. "title": "External Cache Response Time",
  10673. "tooltip": {
  10674. "show": true,
  10675. "showHistogram": true
  10676. },
  10677. "tooltipDecimals": 2,
  10678. "type": "heatmap",
  10679. "xAxis": {
  10680. "show": true
  10681. },
  10682. "xBucketNumber": null,
  10683. "xBucketSize": null,
  10684. "yAxis": {
  10685. "decimals": 0,
  10686. "format": "s",
  10687. "logBase": 1,
  10688. "max": null,
  10689. "min": null,
  10690. "show": true,
  10691. "splitFactor": null
  10692. },
  10693. "yBucketBound": "auto",
  10694. "yBucketNumber": null,
  10695. "yBucketSize": null
  10696. }
  10697. ],
  10698. "title": "External Cache",
  10699. "type": "row"
  10700. }
  10701. ],
  10702. "refresh": false,
  10703. "schemaVersion": 26,
  10704. "style": "dark",
  10705. "tags": [
  10706. "matrix"
  10707. ],
  10708. "templating": {
  10709. "list": [
  10710. {
  10711. "current": {
  10712. "selected": false,
  10713. "text": "default",
  10714. "value": "default"
  10715. },
  10716. "error": null,
  10717. "hide": 0,
  10718. "includeAll": false,
  10719. "label": null,
  10720. "multi": false,
  10721. "name": "datasource",
  10722. "options": [],
  10723. "query": "prometheus",
  10724. "queryValue": "",
  10725. "refresh": 1,
  10726. "regex": "",
  10727. "skipUrlSync": false,
  10728. "type": "datasource"
  10729. },
  10730. {
  10731. "allFormat": "glob",
  10732. "auto": true,
  10733. "auto_count": 100,
  10734. "auto_min": "60s",
  10735. "current": {
  10736. "selected": false,
  10737. "text": "auto",
  10738. "value": "$__auto_interval_bucket_size"
  10739. },
  10740. "datasource": null,
  10741. "error": null,
  10742. "hide": 0,
  10743. "includeAll": false,
  10744. "label": "Bucket Size",
  10745. "multi": false,
  10746. "multiFormat": "glob",
  10747. "name": "bucket_size",
  10748. "options": [
  10749. {
  10750. "selected": true,
  10751. "text": "auto",
  10752. "value": "$__auto_interval_bucket_size"
  10753. },
  10754. {
  10755. "selected": false,
  10756. "text": "30s",
  10757. "value": "30s"
  10758. },
  10759. {
  10760. "selected": false,
  10761. "text": "1m",
  10762. "value": "1m"
  10763. },
  10764. {
  10765. "selected": false,
  10766. "text": "2m",
  10767. "value": "2m"
  10768. },
  10769. {
  10770. "selected": false,
  10771. "text": "5m",
  10772. "value": "5m"
  10773. },
  10774. {
  10775. "selected": false,
  10776. "text": "10m",
  10777. "value": "10m"
  10778. },
  10779. {
  10780. "selected": false,
  10781. "text": "15m",
  10782. "value": "15m"
  10783. }
  10784. ],
  10785. "query": "30s,1m,2m,5m,10m,15m",
  10786. "queryValue": "",
  10787. "refresh": 2,
  10788. "skipUrlSync": false,
  10789. "type": "interval"
  10790. },
  10791. {
  10792. "allValue": null,
  10793. "current": {},
  10794. "datasource": "$datasource",
  10795. "definition": "",
  10796. "error": null,
  10797. "hide": 0,
  10798. "includeAll": false,
  10799. "label": null,
  10800. "multi": false,
  10801. "name": "instance",
  10802. "options": [],
  10803. "query": "label_values(synapse_util_metrics_block_ru_utime_seconds, instance)",
  10804. "refresh": 2,
  10805. "regex": "",
  10806. "skipUrlSync": false,
  10807. "sort": 1,
  10808. "tagValuesQuery": "",
  10809. "tags": [],
  10810. "tagsQuery": "",
  10811. "type": "query",
  10812. "useTags": false
  10813. },
  10814. {
  10815. "allFormat": "regex wildcard",
  10816. "allValue": "",
  10817. "current": {},
  10818. "datasource": "$datasource",
  10819. "definition": "",
  10820. "error": null,
  10821. "hide": 0,
  10822. "hideLabel": false,
  10823. "includeAll": true,
  10824. "label": "Job",
  10825. "multi": true,
  10826. "multiFormat": "regex values",
  10827. "name": "job",
  10828. "options": [],
  10829. "query": "label_values(synapse_util_metrics_block_ru_utime_seconds, job)",
  10830. "refresh": 2,
  10831. "refresh_on_load": false,
  10832. "regex": "",
  10833. "skipUrlSync": false,
  10834. "sort": 1,
  10835. "tagValuesQuery": "",
  10836. "tags": [],
  10837. "tagsQuery": "",
  10838. "type": "query",
  10839. "useTags": false
  10840. },
  10841. {
  10842. "allFormat": "regex wildcard",
  10843. "allValue": ".*",
  10844. "current": {},
  10845. "datasource": "$datasource",
  10846. "definition": "",
  10847. "error": null,
  10848. "hide": 0,
  10849. "hideLabel": false,
  10850. "includeAll": true,
  10851. "label": "",
  10852. "multi": true,
  10853. "multiFormat": "regex values",
  10854. "name": "index",
  10855. "options": [],
  10856. "query": "label_values(synapse_util_metrics_block_ru_utime_seconds, index)",
  10857. "refresh": 2,
  10858. "refresh_on_load": false,
  10859. "regex": "",
  10860. "skipUrlSync": false,
  10861. "sort": 3,
  10862. "tagValuesQuery": "",
  10863. "tags": [],
  10864. "tagsQuery": "",
  10865. "type": "query",
  10866. "useTags": false
  10867. }
  10868. ]
  10869. },
  10870. "time": {
  10871. "from": "now-3h",
  10872. "to": "now"
  10873. },
  10874. "timepicker": {
  10875. "now": true,
  10876. "refresh_intervals": [
  10877. "5s",
  10878. "10s",
  10879. "30s",
  10880. "1m",
  10881. "5m",
  10882. "15m",
  10883. "30m",
  10884. "1h",
  10885. "2h",
  10886. "1d"
  10887. ],
  10888. "time_options": [
  10889. "5m",
  10890. "15m",
  10891. "1h",
  10892. "6h",
  10893. "12h",
  10894. "24h",
  10895. "2d",
  10896. "7d",
  10897. "30d"
  10898. ]
  10899. },
  10900. "timezone": "",
  10901. "title": "Synapse",
  10902. "uid": "000000012",
  10903. "version": 100
  10904. }