[{"data":1,"prerenderedAt":3591},["ShallowReactive",2],{"content:\u002F11-observability\u002F02-prometheus-metrics-go":3},{"title":4,"description":5,"path":6,"body":7},"Метрики Prometheus в Go","Лог отвечает на вопрос \"что произошло с конкретным запросом\". Метрика отвечает на вопрос \"как система ведет себя во времени\". Именно метрики обычно первыми показывают, что началась деградация: выросли 5xx, p95 latency, Kafka lag, DB pool wait или возраст последнего курса.","\u002F11-observability\u002F02-prometheus-metrics-go",{"type":8,"value":9,"toc":3570},"minimark",[10,14,17,20,25,28,39,42,45,77,80,127,131,134,175,178,184,187,193,197,265,272,276,279,290,293,304,307,310,372,375,1085,1092,1095,1229,1235,1239,1252,1255,1261,1264,1552,1569,1573,1576,1659,1666,1670,1673,1679,1686,1690,1695,1918,1921,1955,1959,1965,2064,2070,2074,2083,2126,2133,2136,2172,2175,2195,2198,2202,2212,2227,2230,2249,2252,2280,2283,2320,2323,2332,2335,2344,2354,2363,2367,2370,2645,2649,2652,2658,2661,2664,2670,2676,2679,2688,2691,2694,2946,2950,2953,2956,2970,2973,2990,2993,3013,3016,3029,3033,3036,3093,3100,3104,3107,3126,3129,3133,3136,3170,3173,3202,3205,3209,3243,3403,3566],[11,12,4],"h1",{"id":13},"метрики-prometheus-в-go",[15,16,5],"p",{},[15,18,19],{},"Prometheus хорош тем, что модель простая: time series = имя метрики + labels + samples во времени. Но эта простота обманчива: неправильные labels, buckets и alert rules быстро превращают мониторинг в дорогой шум.",[21,22,24],"h2",{"id":23},"data-model","Data model",[15,26,27],{},"Пример time series:",[29,30,36],"pre",{"className":31,"code":33,"language":34,"meta":35},[32],"language-text","http_requests_total{service=\"ratedesk-api\", method=\"POST\", route=\"\u002Fconvert\", code=\"200\"}\n","text","",[37,38,33],"code",{"__ignoreMap":35},[15,40,41],{},"Имя говорит, что измеряем. Labels говорят, в каком разрезе. Каждая новая комбинация labels - новая серия.",[15,43,44],{},"Нельзя класть в labels:",[46,47,48,55,60,65,68,71,74],"ul",{},[49,50,51,54],"li",{},[37,52,53],{},"request_id",";",[49,56,57,54],{},[37,58,59],{},"trace_id",[49,61,62,54],{},[37,63,64],{},"user_id",[49,66,67],{},"email, телефон;",[49,69,70],{},"raw URL path с id;",[49,72,73],{},"полный текст ошибки;",[49,75,76],{},"произвольные значения из внешних систем.",[15,78,79],{},"Можно:",[46,81,82,87,92,97,103,111,116,121],{},[49,83,84,54],{},[37,85,86],{},"service",[49,88,89,54],{},[37,90,91],{},"env",[49,93,94,54],{},[37,95,96],{},"method",[49,98,99,100,54],{},"нормализованный ",[37,101,102],{},"route",[49,104,105,107,108,54],{},[37,106,37],{}," или ",[37,109,110],{},"status_class",[49,112,113,54],{},[37,114,115],{},"dependency",[49,117,118,54],{},[37,119,120],{},"operation",[49,122,123,126],{},[37,124,125],{},"error_kind"," из ограниченного enum.",[21,128,130],{"id":129},"имена-и-единицы","Имена и единицы",[15,132,133],{},"Prometheus-конвенции:",[46,135,136,142,148,158,172],{},[49,137,138,139,54],{},"seconds, не milliseconds: ",[37,140,141],{},"http_request_duration_seconds",[49,143,144,145,54],{},"bytes: ",[37,146,147],{},"process_resident_memory_bytes",[49,149,150,151,154,155,54],{},"counters с ",[37,152,153],{},"_total",": ",[37,156,157],{},"http_requests_total",[49,159,160,161,164,165,168,169,54],{},"ratio как ",[37,162,163],{},"0..1",", а не проценты: ",[37,166,167],{},"0.995",", не ",[37,170,171],{},"99.5",[49,173,174],{},"base unit в имени, если единица важна.",[15,176,177],{},"Плохие имена:",[29,179,182],{"className":180,"code":181,"language":34,"meta":35},[32],"latency\nrequests\ndb_time\nconvert_failed\n",[37,183,181],{"__ignoreMap":35},[15,185,186],{},"Лучше:",[29,188,191],{"className":189,"code":190,"language":34,"meta":35},[32],"http_requests_total\nhttp_request_duration_seconds\ndb_operation_duration_seconds\ndb_operation_errors_total\nratedesk_rates_freshness_seconds\n",[37,192,190],{"__ignoreMap":35},[21,194,196],{"id":195},"типы-метрик","Типы метрик",[198,199,200,217],"table",{},[201,202,203],"thead",{},[204,205,206,211,214],"tr",{},[207,208,210],"th",{"align":209},"left","Тип",[207,212,213],{"align":209},"Для чего",[207,215,216],{"align":209},"Пример",[218,219,220,232,243,254],"tbody",{},[204,221,222,226,229],{},[223,224,225],"td",{"align":209},"Counter",[223,227,228],{"align":209},"Только растет, сбрасывается при рестарте.",[223,230,231],{"align":209},"Количество запросов, ошибок, processed events.",[204,233,234,237,240],{},[223,235,236],{"align":209},"Gauge",[223,238,239],{"align":209},"Может расти и падать.",[223,241,242],{"align":209},"Active connections, queue size, goroutines.",[204,244,245,248,251],{},[223,246,247],{"align":209},"Histogram",[223,249,250],{"align":209},"Распределение значений по buckets.",[223,252,253],{"align":209},"HTTP latency, DB query duration.",[204,255,256,259,262],{},[223,257,258],{"align":209},"Summary",[223,260,261],{"align":209},"Quantiles на клиенте.",[223,263,264],{"align":209},"Редко нужен для multi-instance SLO.",[15,266,267,268,271],{},"Для latency почти всегда берите Histogram. Summary считает quantiles на клиенте и плохо агрегируется между instance. Histogram можно суммировать по pods\u002Finstances и считать ",[37,269,270],{},"histogram_quantile",".",[21,273,275],{"id":274},"red-use-и-golden-signals","RED, USE и Golden Signals",[15,277,278],{},"Для API начинайте с RED:",[46,280,281,284,287],{},[49,282,283],{},"Rate: сколько запросов приходит;",[49,285,286],{},"Errors: сколько запросов завершается ошибкой;",[49,288,289],{},"Duration: сколько времени занимает обработка.",[15,291,292],{},"Для ресурсов и зависимостей - USE:",[46,294,295,298,301],{},[49,296,297],{},"Utilization: насколько занят ресурс;",[49,299,300],{},"Saturation: есть ли очередь\u002Fожидание;",[49,302,303],{},"Errors: сколько ошибок дает ресурс.",[15,305,306],{},"Google SRE часто формулирует похожую рамку как Four Golden Signals: latency, traffic, errors, saturation.",[15,308,309],{},"Для RateDesk:",[198,311,312,322],{},[201,313,314],{},[204,315,316,319],{},[207,317,318],{"align":209},"Область",[207,320,321],{"align":209},"Метрики",[218,323,324,332,340,348,356,364],{},[204,325,326,329],{},[223,327,328],{"align":209},"API",[223,330,331],{"align":209},"RPS, 4xx\u002F5xx, p50\u002Fp95\u002Fp99, request size.",[204,333,334,337],{},[223,335,336],{"align":209},"PostgreSQL",[223,338,339],{"align":209},"operation duration, errors, pool in-use, acquire wait, locks\u002Fslow queries через exporter.",[204,341,342,345],{},[223,343,344],{"align":209},"Redis",[223,346,347],{"align":209},"hit\u002Fmiss ratio, operation duration, unavailable, memory.",[204,349,350,353],{},[223,351,352],{"align":209},"Kafka",[223,354,355],{"align":209},"consumer lag, handler duration, retry\u002FDLQ, rebalance\u002Ferrors.",[204,357,358,361],{},[223,359,360],{"align":209},"Provider API",[223,362,363],{"align":209},"request duration, timeout, circuit breaker state.",[204,365,366,369],{},[223,367,368],{"align":209},"Domain freshness",[223,370,371],{"align":209},"age последнего успешно обновленного курса.",[21,373,374],{"id":374},"client_golang",[29,376,381],{"className":377,"code":378,"language":379,"meta":380,"style":35},"language-go shiki shiki-themes github-dark","package observability\n\nimport (\n    \"net\u002Fhttp\"\n    \"strconv\"\n    \"time\"\n\n    \"github.com\u002Fprometheus\u002Fclient_golang\u002Fprometheus\"\n    \"github.com\u002Fprometheus\u002Fclient_golang\u002Fprometheus\u002Fpromhttp\"\n)\n\nvar (\n    requestsTotal = prometheus.NewCounterVec(\n        prometheus.CounterOpts{\n            Name: \"http_requests_total\",\n            Help: \"Total HTTP requests.\",\n        },\n        []string{\"method\", \"route\", \"code\"},\n    )\n\n    requestDuration = prometheus.NewHistogramVec(\n        prometheus.HistogramOpts{\n            Name: \"http_request_duration_seconds\",\n            Help: \"HTTP request duration in seconds.\",\n            Buckets: []float64{\n                0.025, 0.05, 0.1, 0.2, 0.3,\n                0.5, 1, 2.5, 5,\n            },\n        },\n        []string{\"method\", \"route\", \"code\"},\n    )\n)\n\nfunc RegisterMetrics(reg *prometheus.Registry) {\n    reg.MustRegister(requestsTotal, requestDuration)\n}\n\nfunc Handler(reg *prometheus.Registry) http.Handler {\n    return promhttp.HandlerFor(reg, promhttp.HandlerOpts{})\n}\n\nfunc MetricsMiddleware(route string, next http.Handler) http.Handler {\n    return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n        start := time.Now()\n        rec := &statusRecorder{ResponseWriter: w, status: http.StatusOK}\n\n        next.ServeHTTP(rec, r)\n\n        code := strconv.Itoa(rec.status)\n        requestsTotal.WithLabelValues(r.Method, route, code).Inc()\n        requestDuration.WithLabelValues(r.Method, route, code).\n            Observe(time.Since(start).Seconds())\n    })\n}\n","go","no-run",[37,382,383,396,403,413,426,436,446,451,461,471,477,482,490,508,522,534,545,551,580,586,591,606,618,628,638,649,678,701,707,712,733,738,743,748,778,790,796,801,835,861,866,871,908,951,969,986,991,1003,1008,1025,1042,1053,1074,1080],{"__ignoreMap":35},[384,385,388,392],"span",{"class":386,"line":387},"line",1,[384,389,391],{"class":390},"snl16","package",[384,393,395],{"class":394},"svObZ"," observability\n",[384,397,399],{"class":386,"line":398},2,[384,400,402],{"emptyLinePlaceholder":401},true,"\n",[384,404,406,409],{"class":386,"line":405},3,[384,407,408],{"class":390},"import",[384,410,412],{"class":411},"s95oV"," (\n",[384,414,416,420,423],{"class":386,"line":415},4,[384,417,419],{"class":418},"sU2Wk","    \"",[384,421,422],{"class":394},"net\u002Fhttp",[384,424,425],{"class":418},"\"\n",[384,427,429,431,434],{"class":386,"line":428},5,[384,430,419],{"class":418},[384,432,433],{"class":394},"strconv",[384,435,425],{"class":418},[384,437,439,441,444],{"class":386,"line":438},6,[384,440,419],{"class":418},[384,442,443],{"class":394},"time",[384,445,425],{"class":418},[384,447,449],{"class":386,"line":448},7,[384,450,402],{"emptyLinePlaceholder":401},[384,452,454,456,459],{"class":386,"line":453},8,[384,455,419],{"class":418},[384,457,458],{"class":394},"github.com\u002Fprometheus\u002Fclient_golang\u002Fprometheus",[384,460,425],{"class":418},[384,462,464,466,469],{"class":386,"line":463},9,[384,465,419],{"class":418},[384,467,468],{"class":394},"github.com\u002Fprometheus\u002Fclient_golang\u002Fprometheus\u002Fpromhttp",[384,470,425],{"class":418},[384,472,474],{"class":386,"line":473},10,[384,475,476],{"class":411},")\n",[384,478,480],{"class":386,"line":479},11,[384,481,402],{"emptyLinePlaceholder":401},[384,483,485,488],{"class":386,"line":484},12,[384,486,487],{"class":390},"var",[384,489,412],{"class":411},[384,491,493,496,499,502,505],{"class":386,"line":492},13,[384,494,495],{"class":411},"    requestsTotal ",[384,497,498],{"class":390},"=",[384,500,501],{"class":411}," prometheus.",[384,503,504],{"class":394},"NewCounterVec",[384,506,507],{"class":411},"(\n",[384,509,511,514,516,519],{"class":386,"line":510},14,[384,512,513],{"class":394},"        prometheus",[384,515,271],{"class":411},[384,517,518],{"class":394},"CounterOpts",[384,520,521],{"class":411},"{\n",[384,523,525,528,531],{"class":386,"line":524},15,[384,526,527],{"class":411},"            Name: ",[384,529,530],{"class":418},"\"http_requests_total\"",[384,532,533],{"class":411},",\n",[384,535,537,540,543],{"class":386,"line":536},16,[384,538,539],{"class":411},"            Help: ",[384,541,542],{"class":418},"\"Total HTTP requests.\"",[384,544,533],{"class":411},[384,546,548],{"class":386,"line":547},17,[384,549,550],{"class":411},"        },\n",[384,552,554,557,560,563,566,569,572,574,577],{"class":386,"line":553},18,[384,555,556],{"class":411},"        []",[384,558,559],{"class":390},"string",[384,561,562],{"class":411},"{",[384,564,565],{"class":418},"\"method\"",[384,567,568],{"class":411},", ",[384,570,571],{"class":418},"\"route\"",[384,573,568],{"class":411},[384,575,576],{"class":418},"\"code\"",[384,578,579],{"class":411},"},\n",[384,581,583],{"class":386,"line":582},19,[384,584,585],{"class":411},"    )\n",[384,587,589],{"class":386,"line":588},20,[384,590,402],{"emptyLinePlaceholder":401},[384,592,594,597,599,601,604],{"class":386,"line":593},21,[384,595,596],{"class":411},"    requestDuration ",[384,598,498],{"class":390},[384,600,501],{"class":411},[384,602,603],{"class":394},"NewHistogramVec",[384,605,507],{"class":411},[384,607,609,611,613,616],{"class":386,"line":608},22,[384,610,513],{"class":394},[384,612,271],{"class":411},[384,614,615],{"class":394},"HistogramOpts",[384,617,521],{"class":411},[384,619,621,623,626],{"class":386,"line":620},23,[384,622,527],{"class":411},[384,624,625],{"class":418},"\"http_request_duration_seconds\"",[384,627,533],{"class":411},[384,629,631,633,636],{"class":386,"line":630},24,[384,632,539],{"class":411},[384,634,635],{"class":418},"\"HTTP request duration in seconds.\"",[384,637,533],{"class":411},[384,639,641,644,647],{"class":386,"line":640},25,[384,642,643],{"class":411},"            Buckets: []",[384,645,646],{"class":390},"float64",[384,648,521],{"class":411},[384,650,652,656,658,661,663,666,668,671,673,676],{"class":386,"line":651},26,[384,653,655],{"class":654},"sDLfK","                0.025",[384,657,568],{"class":411},[384,659,660],{"class":654},"0.05",[384,662,568],{"class":411},[384,664,665],{"class":654},"0.1",[384,667,568],{"class":411},[384,669,670],{"class":654},"0.2",[384,672,568],{"class":411},[384,674,675],{"class":654},"0.3",[384,677,533],{"class":411},[384,679,681,684,686,689,691,694,696,699],{"class":386,"line":680},27,[384,682,683],{"class":654},"                0.5",[384,685,568],{"class":411},[384,687,688],{"class":654},"1",[384,690,568],{"class":411},[384,692,693],{"class":654},"2.5",[384,695,568],{"class":411},[384,697,698],{"class":654},"5",[384,700,533],{"class":411},[384,702,704],{"class":386,"line":703},28,[384,705,706],{"class":411},"            },\n",[384,708,710],{"class":386,"line":709},29,[384,711,550],{"class":411},[384,713,715,717,719,721,723,725,727,729,731],{"class":386,"line":714},30,[384,716,556],{"class":411},[384,718,559],{"class":390},[384,720,562],{"class":411},[384,722,565],{"class":418},[384,724,568],{"class":411},[384,726,571],{"class":418},[384,728,568],{"class":411},[384,730,576],{"class":418},[384,732,579],{"class":411},[384,734,736],{"class":386,"line":735},31,[384,737,585],{"class":411},[384,739,741],{"class":386,"line":740},32,[384,742,476],{"class":411},[384,744,746],{"class":386,"line":745},33,[384,747,402],{"emptyLinePlaceholder":401},[384,749,751,754,757,760,764,767,770,772,775],{"class":386,"line":750},34,[384,752,753],{"class":390},"func",[384,755,756],{"class":394}," RegisterMetrics",[384,758,759],{"class":411},"(",[384,761,763],{"class":762},"s9osk","reg",[384,765,766],{"class":390}," *",[384,768,769],{"class":394},"prometheus",[384,771,271],{"class":411},[384,773,774],{"class":394},"Registry",[384,776,777],{"class":411},") {\n",[384,779,781,784,787],{"class":386,"line":780},35,[384,782,783],{"class":411},"    reg.",[384,785,786],{"class":394},"MustRegister",[384,788,789],{"class":411},"(requestsTotal, requestDuration)\n",[384,791,793],{"class":386,"line":792},36,[384,794,795],{"class":411},"}\n",[384,797,799],{"class":386,"line":798},37,[384,800,402],{"emptyLinePlaceholder":401},[384,802,804,806,809,811,813,815,817,819,821,824,827,829,832],{"class":386,"line":803},38,[384,805,753],{"class":390},[384,807,808],{"class":394}," Handler",[384,810,759],{"class":411},[384,812,763],{"class":762},[384,814,766],{"class":390},[384,816,769],{"class":394},[384,818,271],{"class":411},[384,820,774],{"class":394},[384,822,823],{"class":411},") ",[384,825,826],{"class":394},"http",[384,828,271],{"class":411},[384,830,831],{"class":394},"Handler",[384,833,834],{"class":411}," {\n",[384,836,838,841,844,847,850,853,855,858],{"class":386,"line":837},39,[384,839,840],{"class":390},"    return",[384,842,843],{"class":411}," promhttp.",[384,845,846],{"class":394},"HandlerFor",[384,848,849],{"class":411},"(reg, ",[384,851,852],{"class":394},"promhttp",[384,854,271],{"class":411},[384,856,857],{"class":394},"HandlerOpts",[384,859,860],{"class":411},"{})\n",[384,862,864],{"class":386,"line":863},40,[384,865,795],{"class":411},[384,867,869],{"class":386,"line":868},41,[384,870,402],{"emptyLinePlaceholder":401},[384,872,874,876,879,881,883,886,888,891,894,896,898,900,902,904,906],{"class":386,"line":873},42,[384,875,753],{"class":390},[384,877,878],{"class":394}," MetricsMiddleware",[384,880,759],{"class":411},[384,882,102],{"class":762},[384,884,885],{"class":390}," string",[384,887,568],{"class":411},[384,889,890],{"class":762},"next",[384,892,893],{"class":394}," http",[384,895,271],{"class":411},[384,897,831],{"class":394},[384,899,823],{"class":411},[384,901,826],{"class":394},[384,903,271],{"class":411},[384,905,831],{"class":394},[384,907,834],{"class":411},[384,909,911,913,916,919,921,923,925,928,930,932,935,937,940,942,944,946,949],{"class":386,"line":910},43,[384,912,840],{"class":390},[384,914,915],{"class":411}," http.",[384,917,918],{"class":394},"HandlerFunc",[384,920,759],{"class":411},[384,922,753],{"class":390},[384,924,759],{"class":411},[384,926,927],{"class":762},"w",[384,929,893],{"class":394},[384,931,271],{"class":411},[384,933,934],{"class":394},"ResponseWriter",[384,936,568],{"class":411},[384,938,939],{"class":762},"r",[384,941,766],{"class":390},[384,943,826],{"class":394},[384,945,271],{"class":411},[384,947,948],{"class":394},"Request",[384,950,777],{"class":411},[384,952,954,957,960,963,966],{"class":386,"line":953},44,[384,955,956],{"class":411},"        start ",[384,958,959],{"class":390},":=",[384,961,962],{"class":411}," time.",[384,964,965],{"class":394},"Now",[384,967,968],{"class":411},"()\n",[384,970,972,975,977,980,983],{"class":386,"line":971},45,[384,973,974],{"class":411},"        rec ",[384,976,959],{"class":390},[384,978,979],{"class":390}," &",[384,981,982],{"class":394},"statusRecorder",[384,984,985],{"class":411},"{ResponseWriter: w, status: http.StatusOK}\n",[384,987,989],{"class":386,"line":988},46,[384,990,402],{"emptyLinePlaceholder":401},[384,992,994,997,1000],{"class":386,"line":993},47,[384,995,996],{"class":411},"        next.",[384,998,999],{"class":394},"ServeHTTP",[384,1001,1002],{"class":411},"(rec, r)\n",[384,1004,1006],{"class":386,"line":1005},48,[384,1007,402],{"emptyLinePlaceholder":401},[384,1009,1011,1014,1016,1019,1022],{"class":386,"line":1010},49,[384,1012,1013],{"class":411},"        code ",[384,1015,959],{"class":390},[384,1017,1018],{"class":411}," strconv.",[384,1020,1021],{"class":394},"Itoa",[384,1023,1024],{"class":411},"(rec.status)\n",[384,1026,1028,1031,1034,1037,1040],{"class":386,"line":1027},50,[384,1029,1030],{"class":411},"        requestsTotal.",[384,1032,1033],{"class":394},"WithLabelValues",[384,1035,1036],{"class":411},"(r.Method, route, code).",[384,1038,1039],{"class":394},"Inc",[384,1041,968],{"class":411},[384,1043,1045,1048,1050],{"class":386,"line":1044},51,[384,1046,1047],{"class":411},"        requestDuration.",[384,1049,1033],{"class":394},[384,1051,1052],{"class":411},"(r.Method, route, code).\n",[384,1054,1056,1059,1062,1065,1068,1071],{"class":386,"line":1055},52,[384,1057,1058],{"class":394},"            Observe",[384,1060,1061],{"class":411},"(time.",[384,1063,1064],{"class":394},"Since",[384,1066,1067],{"class":411},"(start).",[384,1069,1070],{"class":394},"Seconds",[384,1072,1073],{"class":411},"())\n",[384,1075,1077],{"class":386,"line":1076},53,[384,1078,1079],{"class":411},"    })\n",[384,1081,1083],{"class":386,"line":1082},54,[384,1084,795],{"class":411},[15,1086,1087,1088,1091],{},"В большом приложении лучше использовать свой ",[37,1089,1090],{},"prometheus.Registry",", чтобы контролировать, какие метрики регистрируются, и легче тестировать.",[15,1093,1094],{},"При custom registry явно подключайте стандартные collectors, если хотите видеть Go\u002Fruntime\u002Fprocess\u002Fbuild info:",[29,1096,1098],{"className":377,"code":1097,"language":379,"meta":380,"style":35},"reg := prometheus.NewRegistry()\nreg.MustRegister(\n    collectors.NewGoCollector(),\n    collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),\n    collectors.NewBuildInfoCollector(),\n    requestsTotal,\n    requestDuration,\n)\n\nhttp.Handle(\"\u002Fmetrics\", promhttp.HandlerFor(reg, promhttp.HandlerOpts{\n    Registry:          reg,\n    EnableOpenMetrics: true,\n}))\n",[37,1099,1100,1114,1123,1134,1154,1163,1168,1173,1177,1181,1209,1214,1224],{"__ignoreMap":35},[384,1101,1102,1105,1107,1109,1112],{"class":386,"line":387},[384,1103,1104],{"class":411},"reg ",[384,1106,959],{"class":390},[384,1108,501],{"class":411},[384,1110,1111],{"class":394},"NewRegistry",[384,1113,968],{"class":411},[384,1115,1116,1119,1121],{"class":386,"line":398},[384,1117,1118],{"class":411},"reg.",[384,1120,786],{"class":394},[384,1122,507],{"class":411},[384,1124,1125,1128,1131],{"class":386,"line":405},[384,1126,1127],{"class":411},"    collectors.",[384,1129,1130],{"class":394},"NewGoCollector",[384,1132,1133],{"class":411},"(),\n",[384,1135,1136,1138,1141,1143,1146,1148,1151],{"class":386,"line":415},[384,1137,1127],{"class":411},[384,1139,1140],{"class":394},"NewProcessCollector",[384,1142,759],{"class":411},[384,1144,1145],{"class":394},"collectors",[384,1147,271],{"class":411},[384,1149,1150],{"class":394},"ProcessCollectorOpts",[384,1152,1153],{"class":411},"{}),\n",[384,1155,1156,1158,1161],{"class":386,"line":428},[384,1157,1127],{"class":411},[384,1159,1160],{"class":394},"NewBuildInfoCollector",[384,1162,1133],{"class":411},[384,1164,1165],{"class":386,"line":438},[384,1166,1167],{"class":411},"    requestsTotal,\n",[384,1169,1170],{"class":386,"line":448},[384,1171,1172],{"class":411},"    requestDuration,\n",[384,1174,1175],{"class":386,"line":453},[384,1176,476],{"class":411},[384,1178,1179],{"class":386,"line":463},[384,1180,402],{"emptyLinePlaceholder":401},[384,1182,1183,1186,1189,1191,1194,1197,1199,1201,1203,1205,1207],{"class":386,"line":473},[384,1184,1185],{"class":411},"http.",[384,1187,1188],{"class":394},"Handle",[384,1190,759],{"class":411},[384,1192,1193],{"class":418},"\"\u002Fmetrics\"",[384,1195,1196],{"class":411},", promhttp.",[384,1198,846],{"class":394},[384,1200,849],{"class":411},[384,1202,852],{"class":394},[384,1204,271],{"class":411},[384,1206,857],{"class":394},[384,1208,521],{"class":411},[384,1210,1211],{"class":386,"line":479},[384,1212,1213],{"class":411},"    Registry:          reg,\n",[384,1215,1216,1219,1222],{"class":386,"line":484},[384,1217,1218],{"class":411},"    EnableOpenMetrics: ",[384,1220,1221],{"class":654},"true",[384,1223,533],{"class":411},[384,1225,1226],{"class":386,"line":492},[384,1227,1228],{"class":411},"}))\n",[15,1230,1231,1234],{},[37,1232,1233],{},"EnableOpenMetrics"," нужен, если вы хотите показывать exemplars и переходить из latency bucket к конкретному trace.",[21,1236,1238],{"id":1237},"echo-middleware-и-имена-метрик","Echo middleware и имена метрик",[15,1240,1241,1242,568,1244,1247,1248,1251],{},"В одном проекте не должно быть одновременно ",[37,1243,157],{},[37,1245,1246],{},"http_server_requests_total"," и ",[37,1249,1250],{},"api_requests_total"," для одного и того же события. Выберите контракт и держите его в уроках, dashboards и alerts.",[15,1253,1254],{},"Для RateDesk используем:",[29,1256,1259],{"className":1257,"code":1258,"language":34,"meta":35},[32],"http_requests_total\nhttp_request_duration_seconds\nhttp_requests_in_flight\n",[37,1260,1258],{"__ignoreMap":35},[15,1262,1263],{},"Echo middleware:",[29,1265,1267],{"className":377,"code":1266,"language":379,"meta":380,"style":35},"func EchoMetrics() echo.MiddlewareFunc {\n    return func(next echo.HandlerFunc) echo.HandlerFunc {\n        return func(c echo.Context) error {\n            start := time.Now()\n            err := next(c)\n            if err != nil {\n                c.Error(err)\n            }\n\n            code := strconv.Itoa(c.Response().Status)\n            route := c.Path()\n            if route == \"\" {\n                route = \"unmatched\"\n            }\n            method := c.Request().Method\n\n            requestsTotal.WithLabelValues(method, route, code).Inc()\n            requestDuration.WithLabelValues(method, route, code).\n                Observe(time.Since(start).Seconds())\n\n            return nil\n        }\n    }\n}\n",[37,1268,1269,1289,1317,1343,1356,1369,1385,1396,1401,1405,1425,1440,1455,1465,1469,1483,1487,1501,1511,1526,1530,1538,1543,1548],{"__ignoreMap":35},[384,1270,1271,1273,1276,1279,1282,1284,1287],{"class":386,"line":387},[384,1272,753],{"class":390},[384,1274,1275],{"class":394}," EchoMetrics",[384,1277,1278],{"class":411},"() ",[384,1280,1281],{"class":394},"echo",[384,1283,271],{"class":411},[384,1285,1286],{"class":394},"MiddlewareFunc",[384,1288,834],{"class":411},[384,1290,1291,1293,1296,1298,1300,1303,1305,1307,1309,1311,1313,1315],{"class":386,"line":398},[384,1292,840],{"class":390},[384,1294,1295],{"class":390}," func",[384,1297,759],{"class":411},[384,1299,890],{"class":762},[384,1301,1302],{"class":394}," echo",[384,1304,271],{"class":411},[384,1306,918],{"class":394},[384,1308,823],{"class":411},[384,1310,1281],{"class":394},[384,1312,271],{"class":411},[384,1314,918],{"class":394},[384,1316,834],{"class":411},[384,1318,1319,1322,1324,1326,1329,1331,1333,1336,1338,1341],{"class":386,"line":405},[384,1320,1321],{"class":390},"        return",[384,1323,1295],{"class":390},[384,1325,759],{"class":411},[384,1327,1328],{"class":762},"c",[384,1330,1302],{"class":394},[384,1332,271],{"class":411},[384,1334,1335],{"class":394},"Context",[384,1337,823],{"class":411},[384,1339,1340],{"class":390},"error",[384,1342,834],{"class":411},[384,1344,1345,1348,1350,1352,1354],{"class":386,"line":415},[384,1346,1347],{"class":411},"            start ",[384,1349,959],{"class":390},[384,1351,962],{"class":411},[384,1353,965],{"class":394},[384,1355,968],{"class":411},[384,1357,1358,1361,1363,1366],{"class":386,"line":428},[384,1359,1360],{"class":411},"            err ",[384,1362,959],{"class":390},[384,1364,1365],{"class":394}," next",[384,1367,1368],{"class":411},"(c)\n",[384,1370,1371,1374,1377,1380,1383],{"class":386,"line":438},[384,1372,1373],{"class":390},"            if",[384,1375,1376],{"class":411}," err ",[384,1378,1379],{"class":390},"!=",[384,1381,1382],{"class":654}," nil",[384,1384,834],{"class":411},[384,1386,1387,1390,1393],{"class":386,"line":448},[384,1388,1389],{"class":411},"                c.",[384,1391,1392],{"class":394},"Error",[384,1394,1395],{"class":411},"(err)\n",[384,1397,1398],{"class":386,"line":453},[384,1399,1400],{"class":411},"            }\n",[384,1402,1403],{"class":386,"line":463},[384,1404,402],{"emptyLinePlaceholder":401},[384,1406,1407,1410,1412,1414,1416,1419,1422],{"class":386,"line":473},[384,1408,1409],{"class":411},"            code ",[384,1411,959],{"class":390},[384,1413,1018],{"class":411},[384,1415,1021],{"class":394},[384,1417,1418],{"class":411},"(c.",[384,1420,1421],{"class":394},"Response",[384,1423,1424],{"class":411},"().Status)\n",[384,1426,1427,1430,1432,1435,1438],{"class":386,"line":479},[384,1428,1429],{"class":411},"            route ",[384,1431,959],{"class":390},[384,1433,1434],{"class":411}," c.",[384,1436,1437],{"class":394},"Path",[384,1439,968],{"class":411},[384,1441,1442,1444,1447,1450,1453],{"class":386,"line":484},[384,1443,1373],{"class":390},[384,1445,1446],{"class":411}," route ",[384,1448,1449],{"class":390},"==",[384,1451,1452],{"class":418}," \"\"",[384,1454,834],{"class":411},[384,1456,1457,1460,1462],{"class":386,"line":492},[384,1458,1459],{"class":411},"                route ",[384,1461,498],{"class":390},[384,1463,1464],{"class":418}," \"unmatched\"\n",[384,1466,1467],{"class":386,"line":510},[384,1468,1400],{"class":411},[384,1470,1471,1474,1476,1478,1480],{"class":386,"line":524},[384,1472,1473],{"class":411},"            method ",[384,1475,959],{"class":390},[384,1477,1434],{"class":411},[384,1479,948],{"class":394},[384,1481,1482],{"class":411},"().Method\n",[384,1484,1485],{"class":386,"line":536},[384,1486,402],{"emptyLinePlaceholder":401},[384,1488,1489,1492,1494,1497,1499],{"class":386,"line":547},[384,1490,1491],{"class":411},"            requestsTotal.",[384,1493,1033],{"class":394},[384,1495,1496],{"class":411},"(method, route, code).",[384,1498,1039],{"class":394},[384,1500,968],{"class":411},[384,1502,1503,1506,1508],{"class":386,"line":553},[384,1504,1505],{"class":411},"            requestDuration.",[384,1507,1033],{"class":394},[384,1509,1510],{"class":411},"(method, route, code).\n",[384,1512,1513,1516,1518,1520,1522,1524],{"class":386,"line":582},[384,1514,1515],{"class":394},"                Observe",[384,1517,1061],{"class":411},[384,1519,1064],{"class":394},[384,1521,1067],{"class":411},[384,1523,1070],{"class":394},[384,1525,1073],{"class":411},[384,1527,1528],{"class":386,"line":588},[384,1529,402],{"emptyLinePlaceholder":401},[384,1531,1532,1535],{"class":386,"line":593},[384,1533,1534],{"class":390},"            return",[384,1536,1537],{"class":654}," nil\n",[384,1539,1540],{"class":386,"line":608},[384,1541,1542],{"class":411},"        }\n",[384,1544,1545],{"class":386,"line":620},[384,1546,1547],{"class":411},"    }\n",[384,1549,1550],{"class":386,"line":630},[384,1551,795],{"class":411},[15,1553,1554,1555,1558,1559,568,1562,568,1565,1568],{},"Не используйте ",[37,1556,1557],{},"c.Request().URL.Path",", иначе ",[37,1560,1561],{},"\u002Fusers\u002F1",[37,1563,1564],{},"\u002Fusers\u002F2",[37,1566,1567],{},"\u002Fusers\u002F3"," станут разными series.",[21,1570,1572],{"id":1571},"метрики-на-границах-clean-architecture","Метрики на границах Clean Architecture",[15,1574,1575],{},"Метрики не должны загрязнять domain entities. Обычно их ставят на инфраструктурных границах:",[198,1577,1578,1591],{},[201,1579,1580],{},[204,1581,1582,1585,1588],{},[207,1583,1584],{"align":209},"Граница",[207,1586,1587],{"align":209},"Что мерить",[207,1589,1590],{"align":209},"Где ставить",[218,1592,1593,1604,1615,1626,1637,1648],{},[204,1594,1595,1598,1601],{},[223,1596,1597],{"align":209},"Transport middleware",[223,1599,1600],{"align":209},"RED metrics HTTP\u002FgRPC.",[223,1602,1603],{"align":209},"Echo\u002FgRPC middleware.",[204,1605,1606,1609,1612],{},[223,1607,1608],{"align":209},"Repository",[223,1610,1611],{"align":209},"Query duration, errors, pool wait.",[223,1613,1614],{"align":209},"Postgres adapter.",[204,1616,1617,1620,1623],{},[223,1618,1619],{"align":209},"Cache adapter",[223,1621,1622],{"align":209},"Hit\u002Fmiss, operation duration, unavailable.",[223,1624,1625],{"align":209},"Redis adapter.",[204,1627,1628,1631,1634],{},[223,1629,1630],{"align":209},"Provider client",[223,1632,1633],{"align":209},"Timeout, status class, circuit breaker state.",[223,1635,1636],{"align":209},"HTTP client wrapper.",[204,1638,1639,1642,1645],{},[223,1640,1641],{"align":209},"Kafka consumer",[223,1643,1644],{"align":209},"Lag, handler duration, retries, DLQ.",[223,1646,1647],{"align":209},"Consumer loop\u002Fhandler wrapper.",[204,1649,1650,1653,1656],{},[223,1651,1652],{"align":209},"Usecase",[223,1654,1655],{"align":209},"Domain outcome: stale response, fallback used.",[223,1657,1658],{"align":209},"Application service boundary.",[15,1660,1661,1662,1665],{},"Domain layer возвращает бизнес-смысл, но не обязан знать Prometheus API. Удобный компромисс - маленький порт вроде ",[37,1663,1664],{},"MetricsRecorder",", который реализуется в infrastructure и передается в usecase только для доменных событий.",[21,1667,1669],{"id":1668},"domain-metrics","Domain metrics",[15,1671,1672],{},"RED\u002FUSE недостаточно для продукта с курсами валют. Нужны доменные SLI:",[29,1674,1677],{"className":1675,"code":1676,"language":34,"meta":35},[32],"ratedesk_rate_freshness_seconds{source=\"cbr\", pair=\"USD_RUB\"}\nratedesk_provider_requests_total{provider=\"cbr\", outcome=\"success|timeout|error\"}\nratedesk_provider_fallbacks_total{from=\"cbr\", to=\"stale_cache\"}\nratedesk_stale_responses_total{route=\"\u002Fconvert\", reason=\"provider_timeout\"}\nratedesk_conversions_total{source=\"fresh|stale|fallback\"}\n",[37,1678,1676],{"__ignoreMap":35},[15,1680,1681,1682,1685],{},"Осторожно с labels: ",[37,1683,1684],{},"pair"," должен быть ограниченным справочником, а не произвольным пользовательским вводом. Если валютных пар сотни тысяч, такую детализацию надо пересмотреть.",[21,1687,1689],{"id":1688},"тестирование-метрик","Тестирование метрик",[15,1691,1692,1694],{},[37,1693,374],{}," позволяет тестировать registry без поднятого Prometheus.",[29,1696,1698],{"className":377,"code":1697,"language":379,"meta":380,"style":35},"func TestHTTPMetrics(t *testing.T) {\n    reg := prometheus.NewRegistry()\n    requests := prometheus.NewCounterVec(\n        prometheus.CounterOpts{Name: \"http_requests_total\", Help: \"Total HTTP requests.\"},\n        []string{\"method\", \"route\", \"code\"},\n    )\n    reg.MustRegister(requests)\n\n    requests.WithLabelValues(\"POST\", \"\u002Fconvert\", \"200\").Inc()\n\n    if err := testutil.GatherAndCompare(reg, strings.NewReader(`\n# HELP http_requests_total Total HTTP requests.\n# TYPE http_requests_total counter\nhttp_requests_total{code=\"200\",method=\"POST\",route=\"\u002Fconvert\"} 1\n`), \"http_requests_total\"); err != nil {\n        t.Fatal(err)\n    }\n}\n",[37,1699,1700,1724,1737,1750,1770,1790,1794,1803,1807,1836,1840,1866,1871,1876,1881,1900,1910,1914],{"__ignoreMap":35},[384,1701,1702,1704,1707,1709,1712,1714,1717,1719,1722],{"class":386,"line":387},[384,1703,753],{"class":390},[384,1705,1706],{"class":394}," TestHTTPMetrics",[384,1708,759],{"class":411},[384,1710,1711],{"class":762},"t",[384,1713,766],{"class":390},[384,1715,1716],{"class":394},"testing",[384,1718,271],{"class":411},[384,1720,1721],{"class":394},"T",[384,1723,777],{"class":411},[384,1725,1726,1729,1731,1733,1735],{"class":386,"line":398},[384,1727,1728],{"class":411},"    reg ",[384,1730,959],{"class":390},[384,1732,501],{"class":411},[384,1734,1111],{"class":394},[384,1736,968],{"class":411},[384,1738,1739,1742,1744,1746,1748],{"class":386,"line":405},[384,1740,1741],{"class":411},"    requests ",[384,1743,959],{"class":390},[384,1745,501],{"class":411},[384,1747,504],{"class":394},[384,1749,507],{"class":411},[384,1751,1752,1754,1756,1758,1761,1763,1766,1768],{"class":386,"line":415},[384,1753,513],{"class":394},[384,1755,271],{"class":411},[384,1757,518],{"class":394},[384,1759,1760],{"class":411},"{Name: ",[384,1762,530],{"class":418},[384,1764,1765],{"class":411},", Help: ",[384,1767,542],{"class":418},[384,1769,579],{"class":411},[384,1771,1772,1774,1776,1778,1780,1782,1784,1786,1788],{"class":386,"line":428},[384,1773,556],{"class":411},[384,1775,559],{"class":390},[384,1777,562],{"class":411},[384,1779,565],{"class":418},[384,1781,568],{"class":411},[384,1783,571],{"class":418},[384,1785,568],{"class":411},[384,1787,576],{"class":418},[384,1789,579],{"class":411},[384,1791,1792],{"class":386,"line":438},[384,1793,585],{"class":411},[384,1795,1796,1798,1800],{"class":386,"line":448},[384,1797,783],{"class":411},[384,1799,786],{"class":394},[384,1801,1802],{"class":411},"(requests)\n",[384,1804,1805],{"class":386,"line":453},[384,1806,402],{"emptyLinePlaceholder":401},[384,1808,1809,1812,1814,1816,1819,1821,1824,1826,1829,1832,1834],{"class":386,"line":463},[384,1810,1811],{"class":411},"    requests.",[384,1813,1033],{"class":394},[384,1815,759],{"class":411},[384,1817,1818],{"class":418},"\"POST\"",[384,1820,568],{"class":411},[384,1822,1823],{"class":418},"\"\u002Fconvert\"",[384,1825,568],{"class":411},[384,1827,1828],{"class":418},"\"200\"",[384,1830,1831],{"class":411},").",[384,1833,1039],{"class":394},[384,1835,968],{"class":411},[384,1837,1838],{"class":386,"line":473},[384,1839,402],{"emptyLinePlaceholder":401},[384,1841,1842,1845,1847,1849,1852,1855,1858,1861,1863],{"class":386,"line":479},[384,1843,1844],{"class":390},"    if",[384,1846,1376],{"class":411},[384,1848,959],{"class":390},[384,1850,1851],{"class":411}," testutil.",[384,1853,1854],{"class":394},"GatherAndCompare",[384,1856,1857],{"class":411},"(reg, strings.",[384,1859,1860],{"class":394},"NewReader",[384,1862,759],{"class":411},[384,1864,1865],{"class":418},"`\n",[384,1867,1868],{"class":386,"line":484},[384,1869,1870],{"class":418},"# HELP http_requests_total Total HTTP requests.\n",[384,1872,1873],{"class":386,"line":492},[384,1874,1875],{"class":418},"# TYPE http_requests_total counter\n",[384,1877,1878],{"class":386,"line":510},[384,1879,1880],{"class":418},"http_requests_total{code=\"200\",method=\"POST\",route=\"\u002Fconvert\"} 1\n",[384,1882,1883,1886,1889,1891,1894,1896,1898],{"class":386,"line":524},[384,1884,1885],{"class":418},"`",[384,1887,1888],{"class":411},"), ",[384,1890,530],{"class":418},[384,1892,1893],{"class":411},"); err ",[384,1895,1379],{"class":390},[384,1897,1382],{"class":654},[384,1899,834],{"class":411},[384,1901,1902,1905,1908],{"class":386,"line":536},[384,1903,1904],{"class":411},"        t.",[384,1906,1907],{"class":394},"Fatal",[384,1909,1395],{"class":411},[384,1911,1912],{"class":386,"line":547},[384,1913,1547],{"class":411},[384,1915,1916],{"class":386,"line":553},[384,1917,795],{"class":411},[15,1919,1920],{},"Что проверять:",[46,1922,1923,1933,1944,1949,1952],{},[49,1924,1925,1926,168,1929,1932],{},"route нормализован (",[37,1927,1928],{},"\u002Fusers\u002F:id",[37,1930,1931],{},"\u002Fusers\u002F123",");",[49,1934,1935,1936,568,1938,568,1940,568,1942,54],{},"нет labels ",[37,1937,53],{},[37,1939,59],{},[37,1941,64],{},[37,1943,1340],{},[49,1945,1946,1947,54],{},"buckets содержат границу SLO, например ",[37,1948,675],{},[49,1950,1951],{},"counter растет на expected outcome;",[49,1953,1954],{},"dependency metrics пишутся в adapter, а не в domain entity.",[21,1956,1958],{"id":1957},"exemplars-и-native-histograms","Exemplars и native histograms",[15,1960,1961,1962,1964],{},"Exemplar связывает конкретное наблюдение метрики с trace. Это не label series, поэтому ",[37,1963,59],{}," не взрывает кардинальность.",[29,1966,1968],{"className":377,"code":1967,"language":379,"meta":380,"style":35},"if observer, ok := requestDuration.WithLabelValues(method, route, code).(prometheus.ExemplarObserver); ok {\n    observer.ObserveWithExemplar(seconds, prometheus.Labels{\n        \"trace_id\": traceIDFromContext(ctx),\n    })\n} else {\n    requestDuration.WithLabelValues(method, route, code).Observe(seconds)\n}\n",[37,1969,1970,1998,2018,2031,2035,2045,2060],{"__ignoreMap":35},[384,1971,1972,1975,1978,1980,1983,1985,1988,1990,1992,1995],{"class":386,"line":387},[384,1973,1974],{"class":390},"if",[384,1976,1977],{"class":411}," observer, ok ",[384,1979,959],{"class":390},[384,1981,1982],{"class":411}," requestDuration.",[384,1984,1033],{"class":394},[384,1986,1987],{"class":411},"(method, route, code).(",[384,1989,769],{"class":394},[384,1991,271],{"class":411},[384,1993,1994],{"class":394},"ExemplarObserver",[384,1996,1997],{"class":411},"); ok {\n",[384,1999,2000,2003,2006,2009,2011,2013,2016],{"class":386,"line":398},[384,2001,2002],{"class":411},"    observer.",[384,2004,2005],{"class":394},"ObserveWithExemplar",[384,2007,2008],{"class":411},"(seconds, ",[384,2010,769],{"class":394},[384,2012,271],{"class":411},[384,2014,2015],{"class":394},"Labels",[384,2017,521],{"class":411},[384,2019,2020,2023,2025,2028],{"class":386,"line":405},[384,2021,2022],{"class":418},"        \"trace_id\"",[384,2024,154],{"class":411},[384,2026,2027],{"class":394},"traceIDFromContext",[384,2029,2030],{"class":411},"(ctx),\n",[384,2032,2033],{"class":386,"line":415},[384,2034,1079],{"class":411},[384,2036,2037,2040,2043],{"class":386,"line":428},[384,2038,2039],{"class":411},"} ",[384,2041,2042],{"class":390},"else",[384,2044,834],{"class":411},[384,2046,2047,2050,2052,2054,2057],{"class":386,"line":438},[384,2048,2049],{"class":411},"    requestDuration.",[384,2051,1033],{"class":394},[384,2053,1496],{"class":411},[384,2055,2056],{"class":394},"Observe",[384,2058,2059],{"class":411},"(seconds)\n",[384,2061,2062],{"class":386,"line":448},[384,2063,795],{"class":411},[15,2065,2066,2067,2069],{},"Native histograms в Prometheus 3.x уже можно рассматривать для продакшена, но для учебного SLO на 300ms explicit buckets понятнее: студент видит, почему bucket ",[37,2068,675],{}," нужен для latency SLO. Если включаете native histograms, явно документируйте настройку scrape и совместимость Grafana\u002FPrometheus.",[21,2071,2073],{"id":2072},"buckets-проектируются-вокруг-slo","Buckets проектируются вокруг SLO",[15,2075,2076,2077,2080,2081,271],{},"Если SLO говорит: \"95% ",[37,2078,2079],{},"\u002Fconvert"," быстрее 300ms\", в buckets должна быть граница около ",[37,2082,675],{},[29,2084,2086],{"className":377,"code":2085,"language":379,"meta":380,"style":35},"Buckets: []float64{0.05, 0.1, 0.2, 0.3, 0.5, 1, 2.5}\n",[37,2087,2088],{"__ignoreMap":35},[384,2089,2090,2093,2095,2097,2099,2101,2103,2105,2107,2109,2111,2113,2116,2118,2120,2122,2124],{"class":386,"line":387},[384,2091,2092],{"class":411},"Buckets: []",[384,2094,646],{"class":390},[384,2096,562],{"class":411},[384,2098,660],{"class":654},[384,2100,568],{"class":411},[384,2102,665],{"class":654},[384,2104,568],{"class":411},[384,2106,670],{"class":654},[384,2108,568],{"class":411},[384,2110,675],{"class":654},[384,2112,568],{"class":411},[384,2114,2115],{"class":654},"0.5",[384,2117,568],{"class":411},[384,2119,688],{"class":654},[384,2121,568],{"class":411},[384,2123,693],{"class":654},[384,2125,795],{"class":411},[15,2127,2128,2129,2132],{},"Если все buckets ",[37,2130,2131],{},"0.1, 1, 10",", вы не сможете нормально понять, как часто запрос уложился в 300ms. Buckets - это не декор, а часть измерительной модели.",[15,2134,2135],{},"Latency можно смотреть как percentile:",[29,2137,2141],{"className":2138,"code":2139,"language":2140,"meta":35,"style":35},"language-promql shiki shiki-themes github-dark","histogram_quantile(\n  0.95,\n  sum by (le, route) (\n    rate(http_request_duration_seconds_bucket[5m])\n  )\n)\n","promql",[37,2142,2143,2148,2153,2158,2163,2168],{"__ignoreMap":35},[384,2144,2145],{"class":386,"line":387},[384,2146,2147],{},"histogram_quantile(\n",[384,2149,2150],{"class":386,"line":398},[384,2151,2152],{},"  0.95,\n",[384,2154,2155],{"class":386,"line":405},[384,2156,2157],{},"  sum by (le, route) (\n",[384,2159,2160],{"class":386,"line":415},[384,2161,2162],{},"    rate(http_request_duration_seconds_bucket[5m])\n",[384,2164,2165],{"class":386,"line":428},[384,2166,2167],{},"  )\n",[384,2169,2170],{"class":386,"line":438},[384,2171,476],{},[15,2173,2174],{},"Но для SLO часто лучше считать долю good events:",[29,2176,2178],{"className":2138,"code":2177,"language":2140,"meta":35,"style":35},"sum(rate(http_request_duration_seconds_bucket{route=\"\u002Fconvert\",le=\"0.3\"}[5m]))\n\u002F\nsum(rate(http_request_duration_seconds_count{route=\"\u002Fconvert\"}[5m]))\n",[37,2179,2180,2185,2190],{"__ignoreMap":35},[384,2181,2182],{"class":386,"line":387},[384,2183,2184],{},"sum(rate(http_request_duration_seconds_bucket{route=\"\u002Fconvert\",le=\"0.3\"}[5m]))\n",[384,2186,2187],{"class":386,"line":398},[384,2188,2189],{},"\u002F\n",[384,2191,2192],{"class":386,"line":405},[384,2193,2194],{},"sum(rate(http_request_duration_seconds_count{route=\"\u002Fconvert\"}[5m]))\n",[15,2196,2197],{},"Так вы прямо измеряете: какая доля запросов уложилась в целевой порог.",[21,2199,2201],{"id":2200},"promql-минимум","PromQL минимум",[15,2203,2204,2205,107,2208,2211],{},"Counter читается через ",[37,2206,2207],{},"rate()",[37,2209,2210],{},"increase()",", потому что он растет и сбрасывается при рестарте.",[29,2213,2215],{"className":2138,"code":2214,"language":2140,"meta":35,"style":35},"sum(rate(http_requests_total[5m])) by (route)\nincrease(http_requests_total{code=~\"5..\"}[1h])\n",[37,2216,2217,2222],{"__ignoreMap":35},[384,2218,2219],{"class":386,"line":387},[384,2220,2221],{},"sum(rate(http_requests_total[5m])) by (route)\n",[384,2223,2224],{"class":386,"line":398},[384,2225,2226],{},"increase(http_requests_total{code=~\"5..\"}[1h])\n",[15,2228,2229],{},"Error ratio:",[29,2231,2233],{"className":2138,"code":2232,"language":2140,"meta":35,"style":35},"sum(rate(http_requests_total{code=~\"5..\"}[5m]))\n\u002F\nsum(rate(http_requests_total[5m]))\n",[37,2234,2235,2240,2244],{"__ignoreMap":35},[384,2236,2237],{"class":386,"line":387},[384,2238,2239],{},"sum(rate(http_requests_total{code=~\"5..\"}[5m]))\n",[384,2241,2242],{"class":386,"line":398},[384,2243,2189],{},[384,2245,2246],{"class":386,"line":405},[384,2247,2248],{},"sum(rate(http_requests_total[5m]))\n",[15,2250,2251],{},"p95 latency:",[29,2253,2254],{"className":2138,"code":2139,"language":2140,"meta":35,"style":35},[37,2255,2256,2260,2264,2268,2272,2276],{"__ignoreMap":35},[384,2257,2258],{"class":386,"line":387},[384,2259,2147],{},[384,2261,2262],{"class":386,"line":398},[384,2263,2152],{},[384,2265,2266],{"class":386,"line":405},[384,2267,2157],{},[384,2269,2270],{"class":386,"line":415},[384,2271,2162],{},[384,2273,2274],{"class":386,"line":428},[384,2275,2167],{},[384,2277,2278],{"class":386,"line":438},[384,2279,476],{},[15,2281,2282],{},"Cache hit ratio:",[29,2284,2286],{"className":2138,"code":2285,"language":2140,"meta":35,"style":35},"sum(rate(redis_cache_hits_total[5m]))\n\u002F\n(\n  sum(rate(redis_cache_hits_total[5m]))\n  +\n  sum(rate(redis_cache_misses_total[5m]))\n)\n",[37,2287,2288,2293,2297,2301,2306,2311,2316],{"__ignoreMap":35},[384,2289,2290],{"class":386,"line":387},[384,2291,2292],{},"sum(rate(redis_cache_hits_total[5m]))\n",[384,2294,2295],{"class":386,"line":398},[384,2296,2189],{},[384,2298,2299],{"class":386,"line":405},[384,2300,507],{},[384,2302,2303],{"class":386,"line":415},[384,2304,2305],{},"  sum(rate(redis_cache_hits_total[5m]))\n",[384,2307,2308],{"class":386,"line":428},[384,2309,2310],{},"  +\n",[384,2312,2313],{"class":386,"line":438},[384,2314,2315],{},"  sum(rate(redis_cache_misses_total[5m]))\n",[384,2317,2318],{"class":386,"line":448},[384,2319,476],{},[15,2321,2322],{},"Freshness:",[29,2324,2326],{"className":2138,"code":2325,"language":2140,"meta":35,"style":35},"time() - ratedesk_last_successful_rate_update_timestamp_seconds\n",[37,2327,2328],{"__ignoreMap":35},[384,2329,2330],{"class":386,"line":387},[384,2331,2325],{},[15,2333,2334],{},"Пропавший target:",[29,2336,2338],{"className":2138,"code":2337,"language":2140,"meta":35,"style":35},"absent(up{job=\"ratedesk\"} == 1)\n",[37,2339,2340],{"__ignoreMap":35},[384,2341,2342],{"class":386,"line":387},[384,2343,2337],{},[15,2345,2346,2347,2350,2351,2353],{},"В Grafana используйте ",[37,2348,2349],{},"$__rate_interval",", чтобы окно ",[37,2352,2207],{}," подстраивалось под range и scrape interval:",[29,2355,2357],{"className":2138,"code":2356,"language":2140,"meta":35,"style":35},"sum(rate(http_requests_total[$__rate_interval])) by (route)\n",[37,2358,2359],{"__ignoreMap":35},[384,2360,2361],{"class":386,"line":387},[384,2362,2356],{},[21,2364,2366],{"id":2365},"recording-rules","Recording rules",[15,2368,2369],{},"Дорогие и часто используемые запросы лучше считать заранее. Это ускоряет dashboards и делает alert rules читабельнее.",[29,2371,2375],{"className":2372,"code":2373,"language":2374,"meta":35,"style":35},"language-yaml shiki shiki-themes github-dark","groups:\n  - name: ratedesk:sli\n    interval: 30s\n    rules:\n      - record: job:http_requests:rate5m\n        expr: |\n          sum by (job) (rate(http_requests_total[5m]))\n\n      - record: job:http_errors:ratio_rate5m\n        expr: |\n          sum by (job) (rate(http_requests_total{code=~\"5..\"}[5m]))\n          \u002F\n          sum by (job) (rate(http_requests_total[5m]))\n\n      - record: job:http_errors:ratio_rate30m\n        expr: |\n          sum by (job) (rate(http_requests_total{code=~\"5..\"}[30m]))\n          \u002F\n          sum by (job) (rate(http_requests_total[30m]))\n\n      - record: job:http_errors:ratio_rate1h\n        expr: |\n          sum by (job) (rate(http_requests_total{code=~\"5..\"}[1h]))\n          \u002F\n          sum by (job) (rate(http_requests_total[1h]))\n\n      - record: job:http_errors:ratio_rate6h\n        expr: |\n          sum by (job) (rate(http_requests_total{code=~\"5..\"}[6h]))\n          \u002F\n          sum by (job) (rate(http_requests_total[6h]))\n\n      - record: route:http_request_duration:p95_5m\n        expr: |\n          histogram_quantile(\n            0.95,\n            sum by (route, le) (\n              rate(http_request_duration_seconds_bucket[5m])\n            )\n          )\n","yaml",[37,2376,2377,2386,2399,2409,2416,2429,2439,2444,2448,2459,2467,2472,2477,2481,2485,2496,2504,2509,2513,2518,2522,2533,2541,2546,2550,2555,2559,2570,2578,2583,2587,2592,2596,2607,2615,2620,2625,2630,2635,2640],{"__ignoreMap":35},[384,2378,2379,2383],{"class":386,"line":387},[384,2380,2382],{"class":2381},"s4JwU","groups",[384,2384,2385],{"class":411},":\n",[384,2387,2388,2391,2394,2396],{"class":386,"line":398},[384,2389,2390],{"class":411},"  - ",[384,2392,2393],{"class":2381},"name",[384,2395,154],{"class":411},[384,2397,2398],{"class":418},"ratedesk:sli\n",[384,2400,2401,2404,2406],{"class":386,"line":405},[384,2402,2403],{"class":2381},"    interval",[384,2405,154],{"class":411},[384,2407,2408],{"class":418},"30s\n",[384,2410,2411,2414],{"class":386,"line":415},[384,2412,2413],{"class":2381},"    rules",[384,2415,2385],{"class":411},[384,2417,2418,2421,2424,2426],{"class":386,"line":428},[384,2419,2420],{"class":411},"      - ",[384,2422,2423],{"class":2381},"record",[384,2425,154],{"class":411},[384,2427,2428],{"class":418},"job:http_requests:rate5m\n",[384,2430,2431,2434,2436],{"class":386,"line":438},[384,2432,2433],{"class":2381},"        expr",[384,2435,154],{"class":411},[384,2437,2438],{"class":390},"|\n",[384,2440,2441],{"class":386,"line":448},[384,2442,2443],{"class":418},"          sum by (job) (rate(http_requests_total[5m]))\n",[384,2445,2446],{"class":386,"line":453},[384,2447,402],{"emptyLinePlaceholder":401},[384,2449,2450,2452,2454,2456],{"class":386,"line":463},[384,2451,2420],{"class":411},[384,2453,2423],{"class":2381},[384,2455,154],{"class":411},[384,2457,2458],{"class":418},"job:http_errors:ratio_rate5m\n",[384,2460,2461,2463,2465],{"class":386,"line":473},[384,2462,2433],{"class":2381},[384,2464,154],{"class":411},[384,2466,2438],{"class":390},[384,2468,2469],{"class":386,"line":479},[384,2470,2471],{"class":418},"          sum by (job) (rate(http_requests_total{code=~\"5..\"}[5m]))\n",[384,2473,2474],{"class":386,"line":484},[384,2475,2476],{"class":418},"          \u002F\n",[384,2478,2479],{"class":386,"line":492},[384,2480,2443],{"class":418},[384,2482,2483],{"class":386,"line":510},[384,2484,402],{"emptyLinePlaceholder":401},[384,2486,2487,2489,2491,2493],{"class":386,"line":524},[384,2488,2420],{"class":411},[384,2490,2423],{"class":2381},[384,2492,154],{"class":411},[384,2494,2495],{"class":418},"job:http_errors:ratio_rate30m\n",[384,2497,2498,2500,2502],{"class":386,"line":536},[384,2499,2433],{"class":2381},[384,2501,154],{"class":411},[384,2503,2438],{"class":390},[384,2505,2506],{"class":386,"line":547},[384,2507,2508],{"class":418},"          sum by (job) (rate(http_requests_total{code=~\"5..\"}[30m]))\n",[384,2510,2511],{"class":386,"line":553},[384,2512,2476],{"class":418},[384,2514,2515],{"class":386,"line":582},[384,2516,2517],{"class":418},"          sum by (job) (rate(http_requests_total[30m]))\n",[384,2519,2520],{"class":386,"line":588},[384,2521,402],{"emptyLinePlaceholder":401},[384,2523,2524,2526,2528,2530],{"class":386,"line":593},[384,2525,2420],{"class":411},[384,2527,2423],{"class":2381},[384,2529,154],{"class":411},[384,2531,2532],{"class":418},"job:http_errors:ratio_rate1h\n",[384,2534,2535,2537,2539],{"class":386,"line":608},[384,2536,2433],{"class":2381},[384,2538,154],{"class":411},[384,2540,2438],{"class":390},[384,2542,2543],{"class":386,"line":620},[384,2544,2545],{"class":418},"          sum by (job) (rate(http_requests_total{code=~\"5..\"}[1h]))\n",[384,2547,2548],{"class":386,"line":630},[384,2549,2476],{"class":418},[384,2551,2552],{"class":386,"line":640},[384,2553,2554],{"class":418},"          sum by (job) (rate(http_requests_total[1h]))\n",[384,2556,2557],{"class":386,"line":651},[384,2558,402],{"emptyLinePlaceholder":401},[384,2560,2561,2563,2565,2567],{"class":386,"line":680},[384,2562,2420],{"class":411},[384,2564,2423],{"class":2381},[384,2566,154],{"class":411},[384,2568,2569],{"class":418},"job:http_errors:ratio_rate6h\n",[384,2571,2572,2574,2576],{"class":386,"line":703},[384,2573,2433],{"class":2381},[384,2575,154],{"class":411},[384,2577,2438],{"class":390},[384,2579,2580],{"class":386,"line":709},[384,2581,2582],{"class":418},"          sum by (job) (rate(http_requests_total{code=~\"5..\"}[6h]))\n",[384,2584,2585],{"class":386,"line":714},[384,2586,2476],{"class":418},[384,2588,2589],{"class":386,"line":735},[384,2590,2591],{"class":418},"          sum by (job) (rate(http_requests_total[6h]))\n",[384,2593,2594],{"class":386,"line":740},[384,2595,402],{"emptyLinePlaceholder":401},[384,2597,2598,2600,2602,2604],{"class":386,"line":745},[384,2599,2420],{"class":411},[384,2601,2423],{"class":2381},[384,2603,154],{"class":411},[384,2605,2606],{"class":418},"route:http_request_duration:p95_5m\n",[384,2608,2609,2611,2613],{"class":386,"line":750},[384,2610,2433],{"class":2381},[384,2612,154],{"class":411},[384,2614,2438],{"class":390},[384,2616,2617],{"class":386,"line":780},[384,2618,2619],{"class":418},"          histogram_quantile(\n",[384,2621,2622],{"class":386,"line":792},[384,2623,2624],{"class":418},"            0.95,\n",[384,2626,2627],{"class":386,"line":798},[384,2628,2629],{"class":418},"            sum by (route, le) (\n",[384,2631,2632],{"class":386,"line":803},[384,2633,2634],{"class":418},"              rate(http_request_duration_seconds_bucket[5m])\n",[384,2636,2637],{"class":386,"line":837},[384,2638,2639],{"class":418},"            )\n",[384,2641,2642],{"class":386,"line":863},[384,2643,2644],{"class":418},"          )\n",[21,2646,2648],{"id":2647},"alerts-и-burn-rate","Alerts и burn rate",[15,2650,2651],{},"Плохой alert:",[29,2653,2656],{"className":2654,"code":2655,"language":34,"meta":35},[32],"CPU > 85%\n",[37,2657,2655],{"__ignoreMap":35},[15,2659,2660],{},"Он может быть полезным на dashboard, но сам по себе не говорит, что пользователь страдает.",[15,2662,2663],{},"Лучше alertить по симптомам:",[29,2665,2668],{"className":2666,"code":2667,"language":34,"meta":35},[32],"5xx rate слишком долго выше нормы\np95 latency нарушает SLO\nfreshness курса вышла за допустимое окно\n",[37,2669,2667],{"__ignoreMap":35},[15,2671,2672,2673,271],{},"SLO: 99.9% availability за 30 дней. Error budget = 0.1% = ",[37,2674,2675],{},"0.001",[15,2677,2678],{},"Burn rate:",[29,2680,2682],{"className":2138,"code":2681,"language":2140,"meta":35,"style":35},"error_rate \u002F error_budget\n",[37,2683,2684],{"__ignoreMap":35},[384,2685,2686],{"class":386,"line":387},[384,2687,2681],{},[15,2689,2690],{},"Если burn rate = 10, вы тратите бюджет в 10 раз быстрее нормы.",[15,2692,2693],{},"Multi-window multi-burn-rate alert:",[29,2695,2697],{"className":2372,"code":2696,"language":2374,"meta":35,"style":35},"groups:\n  - name: ratedesk:slo-alerts\n    rules:\n      - alert: RatedeskFastBurn\n        expr: |\n          (\n            job:http_errors:ratio_rate5m{job=\"ratedesk\"} > (14.4 * 0.001)\n          )\n          and\n          (\n            job:http_errors:ratio_rate1h{job=\"ratedesk\"} > (14.4 * 0.001)\n          )\n        for: 2m\n        labels:\n          severity: page\n          slo: availability\n        annotations:\n          summary: \"RateDesk fast SLO burn\"\n          runbook_url: \"https:\u002F\u002Fgitlab.example.com\u002Fratedesk\u002Fdocs\u002Frunbooks\u002Fhigh-5xx\"\n\n      - alert: RatedeskSlowBurn\n        expr: |\n          (\n            job:http_errors:ratio_rate30m{job=\"ratedesk\"} > (6 * 0.001)\n          )\n          and\n          (\n            job:http_errors:ratio_rate6h{job=\"ratedesk\"} > (6 * 0.001)\n          )\n        for: 15m\n        labels:\n          severity: ticket\n          slo: availability\n        annotations:\n          summary: \"RateDesk slow SLO burn\"\n          runbook_url: \"https:\u002F\u002Fgitlab.example.com\u002Fratedesk\u002Fdocs\u002Frunbooks\u002Fhigh-5xx\"\n",[37,2698,2699,2705,2716,2722,2734,2742,2747,2752,2756,2761,2765,2770,2774,2784,2791,2801,2811,2818,2828,2838,2842,2853,2861,2865,2870,2874,2878,2882,2887,2891,2900,2906,2915,2923,2929,2938],{"__ignoreMap":35},[384,2700,2701,2703],{"class":386,"line":387},[384,2702,2382],{"class":2381},[384,2704,2385],{"class":411},[384,2706,2707,2709,2711,2713],{"class":386,"line":398},[384,2708,2390],{"class":411},[384,2710,2393],{"class":2381},[384,2712,154],{"class":411},[384,2714,2715],{"class":418},"ratedesk:slo-alerts\n",[384,2717,2718,2720],{"class":386,"line":405},[384,2719,2413],{"class":2381},[384,2721,2385],{"class":411},[384,2723,2724,2726,2729,2731],{"class":386,"line":415},[384,2725,2420],{"class":411},[384,2727,2728],{"class":2381},"alert",[384,2730,154],{"class":411},[384,2732,2733],{"class":418},"RatedeskFastBurn\n",[384,2735,2736,2738,2740],{"class":386,"line":428},[384,2737,2433],{"class":2381},[384,2739,154],{"class":411},[384,2741,2438],{"class":390},[384,2743,2744],{"class":386,"line":438},[384,2745,2746],{"class":418},"          (\n",[384,2748,2749],{"class":386,"line":448},[384,2750,2751],{"class":418},"            job:http_errors:ratio_rate5m{job=\"ratedesk\"} > (14.4 * 0.001)\n",[384,2753,2754],{"class":386,"line":453},[384,2755,2644],{"class":418},[384,2757,2758],{"class":386,"line":463},[384,2759,2760],{"class":418},"          and\n",[384,2762,2763],{"class":386,"line":473},[384,2764,2746],{"class":418},[384,2766,2767],{"class":386,"line":479},[384,2768,2769],{"class":418},"            job:http_errors:ratio_rate1h{job=\"ratedesk\"} > (14.4 * 0.001)\n",[384,2771,2772],{"class":386,"line":484},[384,2773,2644],{"class":418},[384,2775,2776,2779,2781],{"class":386,"line":492},[384,2777,2778],{"class":2381},"        for",[384,2780,154],{"class":411},[384,2782,2783],{"class":418},"2m\n",[384,2785,2786,2789],{"class":386,"line":510},[384,2787,2788],{"class":2381},"        labels",[384,2790,2385],{"class":411},[384,2792,2793,2796,2798],{"class":386,"line":524},[384,2794,2795],{"class":2381},"          severity",[384,2797,154],{"class":411},[384,2799,2800],{"class":418},"page\n",[384,2802,2803,2806,2808],{"class":386,"line":536},[384,2804,2805],{"class":2381},"          slo",[384,2807,154],{"class":411},[384,2809,2810],{"class":418},"availability\n",[384,2812,2813,2816],{"class":386,"line":547},[384,2814,2815],{"class":2381},"        annotations",[384,2817,2385],{"class":411},[384,2819,2820,2823,2825],{"class":386,"line":553},[384,2821,2822],{"class":2381},"          summary",[384,2824,154],{"class":411},[384,2826,2827],{"class":418},"\"RateDesk fast SLO burn\"\n",[384,2829,2830,2833,2835],{"class":386,"line":582},[384,2831,2832],{"class":2381},"          runbook_url",[384,2834,154],{"class":411},[384,2836,2837],{"class":418},"\"https:\u002F\u002Fgitlab.example.com\u002Fratedesk\u002Fdocs\u002Frunbooks\u002Fhigh-5xx\"\n",[384,2839,2840],{"class":386,"line":588},[384,2841,402],{"emptyLinePlaceholder":401},[384,2843,2844,2846,2848,2850],{"class":386,"line":593},[384,2845,2420],{"class":411},[384,2847,2728],{"class":2381},[384,2849,154],{"class":411},[384,2851,2852],{"class":418},"RatedeskSlowBurn\n",[384,2854,2855,2857,2859],{"class":386,"line":608},[384,2856,2433],{"class":2381},[384,2858,154],{"class":411},[384,2860,2438],{"class":390},[384,2862,2863],{"class":386,"line":620},[384,2864,2746],{"class":418},[384,2866,2867],{"class":386,"line":630},[384,2868,2869],{"class":418},"            job:http_errors:ratio_rate30m{job=\"ratedesk\"} > (6 * 0.001)\n",[384,2871,2872],{"class":386,"line":640},[384,2873,2644],{"class":418},[384,2875,2876],{"class":386,"line":651},[384,2877,2760],{"class":418},[384,2879,2880],{"class":386,"line":680},[384,2881,2746],{"class":418},[384,2883,2884],{"class":386,"line":703},[384,2885,2886],{"class":418},"            job:http_errors:ratio_rate6h{job=\"ratedesk\"} > (6 * 0.001)\n",[384,2888,2889],{"class":386,"line":709},[384,2890,2644],{"class":418},[384,2892,2893,2895,2897],{"class":386,"line":714},[384,2894,2778],{"class":2381},[384,2896,154],{"class":411},[384,2898,2899],{"class":418},"15m\n",[384,2901,2902,2904],{"class":386,"line":735},[384,2903,2788],{"class":2381},[384,2905,2385],{"class":411},[384,2907,2908,2910,2912],{"class":386,"line":740},[384,2909,2795],{"class":2381},[384,2911,154],{"class":411},[384,2913,2914],{"class":418},"ticket\n",[384,2916,2917,2919,2921],{"class":386,"line":745},[384,2918,2805],{"class":2381},[384,2920,154],{"class":411},[384,2922,2810],{"class":418},[384,2924,2925,2927],{"class":386,"line":750},[384,2926,2815],{"class":2381},[384,2928,2385],{"class":411},[384,2930,2931,2933,2935],{"class":386,"line":780},[384,2932,2822],{"class":2381},[384,2934,154],{"class":411},[384,2936,2937],{"class":418},"\"RateDesk slow SLO burn\"\n",[384,2939,2940,2942,2944],{"class":386,"line":792},[384,2941,2832],{"class":2381},[384,2943,154],{"class":411},[384,2945,2837],{"class":418},[21,2947,2949],{"id":2948},"dashboards","Dashboards",[15,2951,2952],{},"Хороший dashboard строится не \"по технологиям\", а по пути диагностики.",[15,2954,2955],{},"Верх:",[46,2957,2958,2961,2964,2967],{},[49,2959,2960],{},"SLO state;",[49,2962,2963],{},"error budget remaining;",[49,2965,2966],{},"current burn rate;",[49,2968,2969],{},"deploy marker.",[15,2971,2972],{},"API:",[46,2974,2975,2978,2981,2984,2987],{},[49,2976,2977],{},"RPS;",[49,2979,2980],{},"4xx\u002F5xx split;",[49,2982,2983],{},"p50\u002Fp95\u002Fp99;",[49,2985,2986],{},"latency heatmap;",[49,2988,2989],{},"top routes by traffic\u002Ferrors.",[15,2991,2992],{},"Dependencies:",[46,2994,2995,2998,3001,3004,3007,3010],{},[49,2996,2997],{},"DB operation duration\u002Ferrors;",[49,2999,3000],{},"DB pool in-use\u002Facquire wait;",[49,3002,3003],{},"Redis hit ratio and operation duration;",[49,3005,3006],{},"Kafka lag and handler duration;",[49,3008,3009],{},"provider timeout\u002Ferror rate;",[49,3011,3012],{},"freshness.",[15,3014,3015],{},"Drilldown:",[46,3017,3018,3021,3026],{},[49,3019,3020],{},"from p95 spike to traces via exemplars;",[49,3022,3023,3024,54],{},"from error route to logs by ",[37,3025,53],{},[49,3027,3028],{},"from dependency error to runbook.",[21,3030,3032],{"id":3031},"scrape-vs-pushgateway","Scrape vs Pushgateway",[15,3034,3035],{},"По умолчанию Prometheus работает pull-моделью:",[29,3037,3039],{"className":2372,"code":3038,"language":2374,"meta":35,"style":35},"scrape_configs:\n  - job_name: ratedesk\n    metrics_path: \u002Fmetrics\n    static_configs:\n      - targets: [\"ratedesk-api:8080\"]\n",[37,3040,3041,3048,3060,3070,3077],{"__ignoreMap":35},[384,3042,3043,3046],{"class":386,"line":387},[384,3044,3045],{"class":2381},"scrape_configs",[384,3047,2385],{"class":411},[384,3049,3050,3052,3055,3057],{"class":386,"line":398},[384,3051,2390],{"class":411},[384,3053,3054],{"class":2381},"job_name",[384,3056,154],{"class":411},[384,3058,3059],{"class":418},"ratedesk\n",[384,3061,3062,3065,3067],{"class":386,"line":405},[384,3063,3064],{"class":2381},"    metrics_path",[384,3066,154],{"class":411},[384,3068,3069],{"class":418},"\u002Fmetrics\n",[384,3071,3072,3075],{"class":386,"line":415},[384,3073,3074],{"class":2381},"    static_configs",[384,3076,2385],{"class":411},[384,3078,3079,3081,3084,3087,3090],{"class":386,"line":428},[384,3080,2420],{"class":411},[384,3082,3083],{"class":2381},"targets",[384,3085,3086],{"class":411},": [",[384,3088,3089],{"class":418},"\"ratedesk-api:8080\"",[384,3091,3092],{"class":411},"]\n",[15,3094,3095,3096,3099],{},"Pushgateway нужен только для service-level batch jobs, которые живут слишком мало для scrape. Не используйте его как универсальный push-вход для всех сервисов: получите stale metrics, потерю ",[37,3097,3098],{},"up",", сложный lifecycle и дополнительную точку отказа.",[21,3101,3103],{"id":3102},"security","Security",[15,3105,3106],{},"Не выставляйте наружу:",[46,3108,3109,3114,3117,3120,3123],{},[49,3110,3111,54],{},[37,3112,3113],{},"\u002Fmetrics",[49,3115,3116],{},"Prometheus UI;",[49,3118,3119],{},"Alertmanager;",[49,3121,3122],{},"Pushgateway;",[49,3124,3125],{},"exporters с admin-информацией.",[15,3127,3128],{},"Доступ должен быть ограничен сетью, reverse proxy, VPN, mTLS или auth. И никогда не кладите secrets\u002FPII в labels: labels индексируются, размножаются и часто живут дольше, чем кажется.",[21,3130,3132],{"id":3131},"практика","Практика",[15,3134,3135],{},"В RateDesk добавьте:",[3137,3138,3139,3143,3146,3149,3152,3155,3158,3164,3167],"ol",{},[49,3140,3141,271],{},[37,3142,3113],{},[49,3144,3145],{},"RED metrics для HTTP.",[49,3147,3148],{},"DB\u002FRedis\u002FKafka\u002Fprovider metrics на границах adapters.",[49,3150,3151],{},"Freshness metric для последнего курса.",[49,3153,3154],{},"Dashboard as code.",[49,3156,3157],{},"2-3 recording rules.",[49,3159,3160,3161,271],{},"2 alert rules с ",[37,3162,3163],{},"runbook_url",[49,3165,3166],{},"README-раздел про допустимые labels.",[49,3168,3169],{},"Тесты на registry, labels и базовые counters\u002Fhistograms.",[15,3171,3172],{},"Acceptance:",[46,3174,3175,3181,3187,3190,3193,3196,3199],{},[49,3176,3177,3178,54],{},"Prometheus видит target ",[37,3179,3180],{},"ratedesk",[49,3182,3183,3184,3186],{},"p95 ",[37,3185,2079],{}," считается через histogram;",[49,3188,3189],{},"имена метрик совпадают в коде, dashboards, recording rules и финальном проекте;",[49,3191,3192],{},"labels не содержат ids, raw URL и произвольный error text;",[49,3194,3195],{},"есть PromQL для RPS, 5xx ratio, p95, cache hit ratio, freshness;",[49,3197,3198],{},"есть хотя бы один SLO\u002Fburn-rate alert;",[49,3200,3201],{},"dashboard позволяет перейти от симптома к logs\u002Ftraces\u002Frunbook.",[3203,3204],"hr",{},[21,3206,3208],{"id":3207},"интерактивная-практика","Интерактивная практика",[3210,3211,3215,3221,3238],"quiz",{"answer":3212,"id":3213,"xp":3214},"2","obs-metrics-q1","10",[15,3216,3217,3218,3220],{},"Почему нельзя добавлять ",[37,3219,64],{}," или raw URL в Prometheus label?",[3222,3223,3224],"template",{"v-slot:options":35},[46,3225,3226,3229,3232,3235],{},[49,3227,3228],{},"Prometheus не поддерживает строки в labels",[49,3230,3231],{},"Это создаёт высокую кардинальность и раздувает series\u002Findex",[49,3233,3234],{},"Labels видны только в Grafana, а не в Prometheus",[49,3236,3237],{},"Так нельзя построить histogram",[3222,3239,3240],{"v-slot:explanation":35},[15,3241,3242],{},"Prometheus хранит time series по комбинациям labels. Небounded значения вроде user id, request id и raw path быстро делают систему дорогой и нестабильной.",[3244,3245,3249,3252,3398],"predict",{"answer":3246,"id":3247,"xp":3248},"counter\\nhistogram","obs-metrics-p1","15",[15,3250,3251],{},"Что выведет программа?",[3222,3253,3254],{"v-slot:code":35},[29,3255,3257],{"className":377,"code":3256,"language":379,"meta":35,"style":35},"package main\n\nimport \"fmt\"\n\nfunc MetricType(signal string) string {\n    if signal == \"duration\" {\n        return \"histogram\"\n    }\n    return \"counter\"\n}\n\nfunc main() {\n    fmt.Println(MetricType(\"requests_total\"))\n    fmt.Println(MetricType(\"duration\"))\n}\n",[37,3258,3259,3266,3270,3282,3286,3306,3320,3327,3331,3338,3342,3346,3356,3377,3394],{"__ignoreMap":35},[384,3260,3261,3263],{"class":386,"line":387},[384,3262,391],{"class":390},[384,3264,3265],{"class":394}," main\n",[384,3267,3268],{"class":386,"line":398},[384,3269,402],{"emptyLinePlaceholder":401},[384,3271,3272,3274,3277,3280],{"class":386,"line":405},[384,3273,408],{"class":390},[384,3275,3276],{"class":418}," \"",[384,3278,3279],{"class":394},"fmt",[384,3281,425],{"class":418},[384,3283,3284],{"class":386,"line":415},[384,3285,402],{"emptyLinePlaceholder":401},[384,3287,3288,3290,3293,3295,3298,3300,3302,3304],{"class":386,"line":428},[384,3289,753],{"class":390},[384,3291,3292],{"class":394}," MetricType",[384,3294,759],{"class":411},[384,3296,3297],{"class":762},"signal",[384,3299,885],{"class":390},[384,3301,823],{"class":411},[384,3303,559],{"class":390},[384,3305,834],{"class":411},[384,3307,3308,3310,3313,3315,3318],{"class":386,"line":438},[384,3309,1844],{"class":390},[384,3311,3312],{"class":411}," signal ",[384,3314,1449],{"class":390},[384,3316,3317],{"class":418}," \"duration\"",[384,3319,834],{"class":411},[384,3321,3322,3324],{"class":386,"line":448},[384,3323,1321],{"class":390},[384,3325,3326],{"class":418}," \"histogram\"\n",[384,3328,3329],{"class":386,"line":453},[384,3330,1547],{"class":411},[384,3332,3333,3335],{"class":386,"line":463},[384,3334,840],{"class":390},[384,3336,3337],{"class":418}," \"counter\"\n",[384,3339,3340],{"class":386,"line":473},[384,3341,795],{"class":411},[384,3343,3344],{"class":386,"line":479},[384,3345,402],{"emptyLinePlaceholder":401},[384,3347,3348,3350,3353],{"class":386,"line":484},[384,3349,753],{"class":390},[384,3351,3352],{"class":394}," main",[384,3354,3355],{"class":411},"() {\n",[384,3357,3358,3361,3364,3366,3369,3371,3374],{"class":386,"line":492},[384,3359,3360],{"class":411},"    fmt.",[384,3362,3363],{"class":394},"Println",[384,3365,759],{"class":411},[384,3367,3368],{"class":394},"MetricType",[384,3370,759],{"class":411},[384,3372,3373],{"class":418},"\"requests_total\"",[384,3375,3376],{"class":411},"))\n",[384,3378,3379,3381,3383,3385,3387,3389,3392],{"class":386,"line":510},[384,3380,3360],{"class":411},[384,3382,3363],{"class":394},[384,3384,759],{"class":411},[384,3386,3368],{"class":394},[384,3388,759],{"class":411},[384,3390,3391],{"class":418},"\"duration\"",[384,3393,3376],{"class":411},[384,3395,3396],{"class":386,"line":524},[384,3397,795],{"class":411},[3222,3399,3400],{"v-slot:hint":35},[15,3401,3402],{},"События считают counter, распределение latency обычно меряют histogram.",[3404,3405,3409,3427,3553],"code-task",{"expected":3406,"id":3407,"xp":3408},"counter\\nhistogram\\ngauge","obs-metrics-ct1","20",[15,3410,3411,3412,3415,3416,3419,3420,3423,3424,271],{},"Реализуй ",[37,3413,3414],{},"MetricKind",": общее число запросов - ",[37,3417,3418],{},"counter",", длительность запроса - ",[37,3421,3422],{},"histogram",", текущие in-flight запросы - ",[37,3425,3426],{},"gauge",[3222,3428,3429],{"v-slot:template":35},[29,3430,3432],{"className":377,"code":3431,"language":379,"meta":35,"style":35},"package main\n\nimport \"fmt\"\n\nfunc MetricKind(signal string) string {\n    return \"todo\"\n}\n\nfunc main() {\n    fmt.Println(MetricKind(\"http_requests_total\"))\n    fmt.Println(MetricKind(\"http_request_duration_seconds\"))\n    fmt.Println(MetricKind(\"http_inflight_requests\"))\n}\n",[37,3433,3434,3440,3444,3454,3458,3477,3484,3488,3492,3500,3516,3532,3549],{"__ignoreMap":35},[384,3435,3436,3438],{"class":386,"line":387},[384,3437,391],{"class":390},[384,3439,3265],{"class":394},[384,3441,3442],{"class":386,"line":398},[384,3443,402],{"emptyLinePlaceholder":401},[384,3445,3446,3448,3450,3452],{"class":386,"line":405},[384,3447,408],{"class":390},[384,3449,3276],{"class":418},[384,3451,3279],{"class":394},[384,3453,425],{"class":418},[384,3455,3456],{"class":386,"line":415},[384,3457,402],{"emptyLinePlaceholder":401},[384,3459,3460,3462,3465,3467,3469,3471,3473,3475],{"class":386,"line":428},[384,3461,753],{"class":390},[384,3463,3464],{"class":394}," MetricKind",[384,3466,759],{"class":411},[384,3468,3297],{"class":762},[384,3470,885],{"class":390},[384,3472,823],{"class":411},[384,3474,559],{"class":390},[384,3476,834],{"class":411},[384,3478,3479,3481],{"class":386,"line":438},[384,3480,840],{"class":390},[384,3482,3483],{"class":418}," \"todo\"\n",[384,3485,3486],{"class":386,"line":448},[384,3487,795],{"class":411},[384,3489,3490],{"class":386,"line":453},[384,3491,402],{"emptyLinePlaceholder":401},[384,3493,3494,3496,3498],{"class":386,"line":463},[384,3495,753],{"class":390},[384,3497,3352],{"class":394},[384,3499,3355],{"class":411},[384,3501,3502,3504,3506,3508,3510,3512,3514],{"class":386,"line":473},[384,3503,3360],{"class":411},[384,3505,3363],{"class":394},[384,3507,759],{"class":411},[384,3509,3414],{"class":394},[384,3511,759],{"class":411},[384,3513,530],{"class":418},[384,3515,3376],{"class":411},[384,3517,3518,3520,3522,3524,3526,3528,3530],{"class":386,"line":479},[384,3519,3360],{"class":411},[384,3521,3363],{"class":394},[384,3523,759],{"class":411},[384,3525,3414],{"class":394},[384,3527,759],{"class":411},[384,3529,625],{"class":418},[384,3531,3376],{"class":411},[384,3533,3534,3536,3538,3540,3542,3544,3547],{"class":386,"line":484},[384,3535,3360],{"class":411},[384,3537,3363],{"class":394},[384,3539,759],{"class":411},[384,3541,3414],{"class":394},[384,3543,759],{"class":411},[384,3545,3546],{"class":418},"\"http_inflight_requests\"",[384,3548,3376],{"class":411},[384,3550,3551],{"class":386,"line":492},[384,3552,795],{"class":411},[3222,3554,3555],{"v-slot:hints":35},[46,3556,3557,3560,3563],{},[49,3558,3559],{},"Counter монотонно растёт.",[49,3561,3562],{},"Histogram нужен для quantiles и SLO buckets.",[49,3564,3565],{},"Gauge показывает текущее значение.",[3567,3568,3569],"style",{},"html pre.shiki code .snl16, html code.shiki .snl16{--shiki-default:#F97583}html pre.shiki code .svObZ, html code.shiki .svObZ{--shiki-default:#B392F0}html pre.shiki code .s95oV, html code.shiki .s95oV{--shiki-default:#E1E4E8}html pre.shiki code .sU2Wk, html code.shiki .sU2Wk{--shiki-default:#9ECBFF}html pre.shiki code .sDLfK, html code.shiki .sDLfK{--shiki-default:#79B8FF}html pre.shiki code .s9osk, html code.shiki .s9osk{--shiki-default:#FFAB70}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html pre.shiki code .s4JwU, html code.shiki .s4JwU{--shiki-default:#85E89D}",{"title":35,"searchDepth":398,"depth":398,"links":3571},[3572,3573,3574,3575,3576,3577,3578,3579,3580,3581,3582,3583,3584,3585,3586,3587,3588,3589,3590],{"id":23,"depth":398,"text":24},{"id":129,"depth":398,"text":130},{"id":195,"depth":398,"text":196},{"id":274,"depth":398,"text":275},{"id":374,"depth":398,"text":374},{"id":1237,"depth":398,"text":1238},{"id":1571,"depth":398,"text":1572},{"id":1668,"depth":398,"text":1669},{"id":1688,"depth":398,"text":1689},{"id":1957,"depth":398,"text":1958},{"id":2072,"depth":398,"text":2073},{"id":2200,"depth":398,"text":2201},{"id":2365,"depth":398,"text":2366},{"id":2647,"depth":398,"text":2648},{"id":2948,"depth":398,"text":2949},{"id":3031,"depth":398,"text":3032},{"id":3102,"depth":398,"text":3103},{"id":3131,"depth":398,"text":3132},{"id":3207,"depth":398,"text":3208},1781022063230]