[{"data":1,"prerenderedAt":2754},["ShallowReactive",2],{"content:\u002F08-kafka\u002F04-consumers-consumer-groups-go":3},{"title":4,"description":5,"path":6,"body":7},"Kafka consumers и consumer groups в Go","Consumer читает records из topic и выполняет side effect: пишет в БД, вызывает другой сервис, обновляет индекс, отправляет письмо. Самая важная часть consumer-кода - не ReadMessage, а когда вы commit offset.","\u002F08-kafka\u002F04-consumers-consumer-groups-go",{"type":8,"value":9,"toc":2741},"minimark",[10,14,28,31,41,44,47,52,55,61,64,67,73,76,78,82,139,142,148,151,153,157,175,1523,1529,1536,1538,1542,1545,1548,1609,1612,1618,1621,1631,1633,1637,1640,1643,1646,1916,1919,1922,1924,1928,1931,1947,1953,2197,2204,2207,2209,2213,2219,2264,2276,2278,2282,2285,2288,2302,2305,2307,2311,2349,2351,2355,2378,2380,2384,2417,2566,2737],[11,12,4],"h1",{"id":13},"kafka-consumers-и-consumer-groups-в-go",[15,16,17,18,22,23,27],"p",{},"Consumer читает records из topic и выполняет side effect: пишет в БД, вызывает другой сервис, обновляет индекс, отправляет письмо. Самая важная часть consumer-кода - не ",[19,20,21],"code",{},"ReadMessage",", а ",[24,25,26],"strong",{},"когда вы commit offset",".",[15,29,30],{},"Offset commit отвечает на вопрос: \"с какого места продолжать после restart или rebalance?\"",[32,33,39],"pre",{"className":34,"code":36,"language":37,"meta":38},[35],"language-text","partition: [100][101][102][103][104]\n                    ^\n                    committed offset means group processed up to here\n","text","",[19,40,36],{"__ignoreMap":38},[15,42,43],{},"Если commit сделать слишком рано, можно потерять сообщение. Если слишком поздно, можно получить duplicate. Поэтому нормальная Kafka-архитектура исходит из at-least-once и делает обработку идемпотентной.",[45,46],"hr",{},[48,49,51],"h2",{"id":50},"consumer-group","Consumer group",[15,53,54],{},"Consumer group - это набор процессов, которые совместно читают topic. Внутри одной group каждая partition назначается только одному active consumer.",[32,56,59],{"className":57,"code":58,"language":37,"meta":38},[35],"topic orders.events, 6 partitions\n\nconsumer group: billing\n\nconsumer A: p0, p3\nconsumer B: p1, p4\nconsumer C: p2, p5\n",[19,60,58],{"__ignoreMap":38},[15,62,63],{},"Если добавить четвёртый consumer, произойдёт rebalance: Kafka перераспределит partitions. Если consumers больше, чем partitions, часть будет простаивать.",[15,65,66],{},"Разные groups независимы:",[32,68,71],{"className":69,"code":70,"language":37,"meta":38},[35],"orders.events\n  group billing       offset A\n  group notifications offset B\n  group analytics     offset C\n",[19,72,70],{"__ignoreMap":38},[15,74,75],{},"Каждая подсистема может читать один поток в своём темпе.",[45,77],{},[48,79,81],{"id":80},"at-least-once-at-most-once-effectively-once","At-least-once, at-most-once, effectively-once",[83,84,85,102],"table",{},[86,87,88],"thead",{},[89,90,91,96,99],"tr",{},[92,93,95],"th",{"align":94},"left","Семантика",[92,97,98],{"align":94},"Когда commit",[92,100,101],{"align":94},"Что может случиться",[103,104,105,117,128],"tbody",{},[89,106,107,111,114],{},[108,109,110],"td",{"align":94},"At-most-once",[108,112,113],{"align":94},"До обработки",[108,115,116],{"align":94},"Сообщение может потеряться при падении после commit.",[89,118,119,122,125],{},[108,120,121],{"align":94},"At-least-once",[108,123,124],{"align":94},"После успешной обработки",[108,126,127],{"align":94},"Сообщение может обработаться повторно.",[89,129,130,133,136],{},[108,131,132],{"align":94},"Effectively-once",[108,134,135],{"align":94},"После идемпотентной обработки",[108,137,138],{"align":94},"Duplicate приходит, но side effect не повторяется.",[15,140,141],{},"На практике для backend чаще всего выбирают:",[32,143,146],{"className":144,"code":145,"language":37,"meta":38},[35],"read message\nprocess idempotently\ncommit offset\n",[19,147,145],{"__ignoreMap":38},[15,149,150],{},"Это at-least-once на уровне доставки и effectively-once на уровне бизнес-эффекта.",[45,152],{},[48,154,156],{"id":155},"manual-commit-в-go","Manual commit в Go",[15,158,159,162,163,166,167,170,171,174],{},[19,160,161],{},"kafka-go"," reader с ",[19,164,165],{},"GroupID"," умеет consumer groups. Метод ",[19,168,169],{},"FetchMessage"," получает сообщение без auto commit, а ",[19,172,173],{},"CommitMessages"," фиксирует offset после успешной обработки.",[32,176,181],{"className":177,"code":178,"language":179,"meta":180,"style":38},"language-go shiki shiki-themes github-dark","package main\n\nimport (\n    \"context\"\n    \"encoding\u002Fjson\"\n    \"errors\"\n    \"fmt\"\n    \"log\u002Fslog\"\n    \"os\"\n    \"os\u002Fsignal\"\n    \"syscall\"\n    \"time\"\n\n    \"github.com\u002Fsegmentio\u002Fkafka-go\"\n)\n\ntype OrderCreated struct {\n    EventID  string `json:\"event_id\"`\n    OrderID  string `json:\"order_id\"`\n    UserID   string `json:\"user_id\"`\n    Amount   int64  `json:\"amount\"`\n    Currency string `json:\"currency\"`\n}\n\ntype Handler interface {\n    HandleOrderCreated(ctx context.Context, event OrderCreated) error\n}\n\ntype LogHandler struct{}\n\nfunc (LogHandler) HandleOrderCreated(ctx context.Context, event OrderCreated) error {\n    select {\n    case \u003C-ctx.Done():\n        return ctx.Err()\n    case \u003C-time.After(50 * time.Millisecond):\n        return nil\n    }\n}\n\nfunc main() {\n    logger := slog.New(slog.NewTextHandler(os.Stdout, nil))\n\n    ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)\n    defer stop()\n\n    reader := kafka.NewReader(kafka.ReaderConfig{\n        Brokers:        []string{\"localhost:9092\"},\n        Topic:          \"orders.events\",\n        GroupID:        \"billing-service\",\n        MinBytes:       1,\n        MaxBytes:       10e6,\n        CommitInterval: 0, \u002F\u002F manual commit\n    })\n    defer func() {\n        if err := reader.Close(); err != nil {\n            logger.Error(\"close kafka reader\", \"error\", err)\n        }\n    }()\n\n    if err := consume(ctx, reader, LogHandler{}, logger); err != nil && !errors.Is(err, context.Canceled) {\n        logger.Error(\"consumer stopped\", \"error\", err)\n        os.Exit(1)\n    }\n}\n\nfunc consume(ctx context.Context, reader *kafka.Reader, handler Handler, logger *slog.Logger) error {\n    for {\n        msg, err := reader.FetchMessage(ctx)\n        if err != nil {\n            return fmt.Errorf(\"fetch message: %w\", err)\n        }\n\n        if err := handleMessage(ctx, handler, msg); err != nil {\n            logger.Error(\"handle message failed\",\n                \"topic\", msg.Topic,\n                \"partition\", msg.Partition,\n                \"offset\", msg.Offset,\n                \"error\", err,\n            )\n            return err\n        }\n\n        if err := reader.CommitMessages(ctx, msg); err != nil {\n            return fmt.Errorf(\"commit offset topic=%s partition=%d offset=%d: %w\",\n                msg.Topic, msg.Partition, msg.Offset, err)\n        }\n    }\n}\n\nfunc handleMessage(ctx context.Context, handler Handler, msg kafka.Message) error {\n    var event OrderCreated\n    if err := json.Unmarshal(msg.Value, &event); err != nil {\n        return fmt.Errorf(\"decode order created: %w\", err)\n    }\n\n    handleCtx, cancel := context.WithTimeout(ctx, 10*time.Second)\n    defer cancel()\n\n    if err := handler.HandleOrderCreated(handleCtx, event); err != nil {\n        return fmt.Errorf(\"handle order created event_id=%s: %w\", event.EventID, err)\n    }\n\n    return nil\n}\n","go","no-run",[19,182,183,196,203,213,226,236,246,256,266,276,286,296,306,311,321,327,332,347,359,370,381,393,404,410,415,428,463,468,473,486,491,531,539,557,572,597,605,611,616,621,632,662,667,690,701,706,733,750,762,773,784,801,816,822,832,860,882,888,894,899,939,958,973,978,983,988,1047,1055,1070,1083,1108,1113,1118,1139,1153,1162,1171,1180,1189,1195,1203,1208,1213,1235,1272,1278,1283,1288,1293,1298,1340,1352,1382,1402,1407,1412,1437,1447,1452,1475,1500,1505,1510,1518],{"__ignoreMap":38},[184,185,188,192],"span",{"class":186,"line":187},"line",1,[184,189,191],{"class":190},"snl16","package",[184,193,195],{"class":194},"svObZ"," main\n",[184,197,199],{"class":186,"line":198},2,[184,200,202],{"emptyLinePlaceholder":201},true,"\n",[184,204,206,209],{"class":186,"line":205},3,[184,207,208],{"class":190},"import",[184,210,212],{"class":211},"s95oV"," (\n",[184,214,216,220,223],{"class":186,"line":215},4,[184,217,219],{"class":218},"sU2Wk","    \"",[184,221,222],{"class":194},"context",[184,224,225],{"class":218},"\"\n",[184,227,229,231,234],{"class":186,"line":228},5,[184,230,219],{"class":218},[184,232,233],{"class":194},"encoding\u002Fjson",[184,235,225],{"class":218},[184,237,239,241,244],{"class":186,"line":238},6,[184,240,219],{"class":218},[184,242,243],{"class":194},"errors",[184,245,225],{"class":218},[184,247,249,251,254],{"class":186,"line":248},7,[184,250,219],{"class":218},[184,252,253],{"class":194},"fmt",[184,255,225],{"class":218},[184,257,259,261,264],{"class":186,"line":258},8,[184,260,219],{"class":218},[184,262,263],{"class":194},"log\u002Fslog",[184,265,225],{"class":218},[184,267,269,271,274],{"class":186,"line":268},9,[184,270,219],{"class":218},[184,272,273],{"class":194},"os",[184,275,225],{"class":218},[184,277,279,281,284],{"class":186,"line":278},10,[184,280,219],{"class":218},[184,282,283],{"class":194},"os\u002Fsignal",[184,285,225],{"class":218},[184,287,289,291,294],{"class":186,"line":288},11,[184,290,219],{"class":218},[184,292,293],{"class":194},"syscall",[184,295,225],{"class":218},[184,297,299,301,304],{"class":186,"line":298},12,[184,300,219],{"class":218},[184,302,303],{"class":194},"time",[184,305,225],{"class":218},[184,307,309],{"class":186,"line":308},13,[184,310,202],{"emptyLinePlaceholder":201},[184,312,314,316,319],{"class":186,"line":313},14,[184,315,219],{"class":218},[184,317,318],{"class":194},"github.com\u002Fsegmentio\u002Fkafka-go",[184,320,225],{"class":218},[184,322,324],{"class":186,"line":323},15,[184,325,326],{"class":211},")\n",[184,328,330],{"class":186,"line":329},16,[184,331,202],{"emptyLinePlaceholder":201},[184,333,335,338,341,344],{"class":186,"line":334},17,[184,336,337],{"class":190},"type",[184,339,340],{"class":194}," OrderCreated",[184,342,343],{"class":190}," struct",[184,345,346],{"class":211}," {\n",[184,348,350,353,356],{"class":186,"line":349},18,[184,351,352],{"class":211},"    EventID  ",[184,354,355],{"class":190},"string",[184,357,358],{"class":218}," `json:\"event_id\"`\n",[184,360,362,365,367],{"class":186,"line":361},19,[184,363,364],{"class":211},"    OrderID  ",[184,366,355],{"class":190},[184,368,369],{"class":218}," `json:\"order_id\"`\n",[184,371,373,376,378],{"class":186,"line":372},20,[184,374,375],{"class":211},"    UserID   ",[184,377,355],{"class":190},[184,379,380],{"class":218}," `json:\"user_id\"`\n",[184,382,384,387,390],{"class":186,"line":383},21,[184,385,386],{"class":211},"    Amount   ",[184,388,389],{"class":190},"int64",[184,391,392],{"class":218},"  `json:\"amount\"`\n",[184,394,396,399,401],{"class":186,"line":395},22,[184,397,398],{"class":211},"    Currency ",[184,400,355],{"class":190},[184,402,403],{"class":218}," `json:\"currency\"`\n",[184,405,407],{"class":186,"line":406},23,[184,408,409],{"class":211},"}\n",[184,411,413],{"class":186,"line":412},24,[184,414,202],{"emptyLinePlaceholder":201},[184,416,418,420,423,426],{"class":186,"line":417},25,[184,419,337],{"class":190},[184,421,422],{"class":194}," Handler",[184,424,425],{"class":190}," interface",[184,427,346],{"class":211},[184,429,431,434,437,441,444,446,449,452,455,457,460],{"class":186,"line":430},26,[184,432,433],{"class":194},"    HandleOrderCreated",[184,435,436],{"class":211},"(",[184,438,440],{"class":439},"s9osk","ctx",[184,442,443],{"class":194}," context",[184,445,27],{"class":211},[184,447,448],{"class":194},"Context",[184,450,451],{"class":211},", ",[184,453,454],{"class":439},"event",[184,456,340],{"class":194},[184,458,459],{"class":211},") ",[184,461,462],{"class":190},"error\n",[184,464,466],{"class":186,"line":465},27,[184,467,409],{"class":211},[184,469,471],{"class":186,"line":470},28,[184,472,202],{"emptyLinePlaceholder":201},[184,474,476,478,481,483],{"class":186,"line":475},29,[184,477,337],{"class":190},[184,479,480],{"class":194}," LogHandler",[184,482,343],{"class":190},[184,484,485],{"class":211},"{}\n",[184,487,489],{"class":186,"line":488},30,[184,490,202],{"emptyLinePlaceholder":201},[184,492,494,497,500,503,505,508,510,512,514,516,518,520,522,524,526,529],{"class":186,"line":493},31,[184,495,496],{"class":190},"func",[184,498,499],{"class":211}," (",[184,501,502],{"class":194},"LogHandler",[184,504,459],{"class":211},[184,506,507],{"class":194},"HandleOrderCreated",[184,509,436],{"class":211},[184,511,440],{"class":439},[184,513,443],{"class":194},[184,515,27],{"class":211},[184,517,448],{"class":194},[184,519,451],{"class":211},[184,521,454],{"class":439},[184,523,340],{"class":194},[184,525,459],{"class":211},[184,527,528],{"class":190},"error",[184,530,346],{"class":211},[184,532,534,537],{"class":186,"line":533},32,[184,535,536],{"class":190},"    select",[184,538,346],{"class":211},[184,540,542,545,548,551,554],{"class":186,"line":541},33,[184,543,544],{"class":190},"    case",[184,546,547],{"class":190}," \u003C-",[184,549,550],{"class":211},"ctx.",[184,552,553],{"class":194},"Done",[184,555,556],{"class":211},"():\n",[184,558,560,563,566,569],{"class":186,"line":559},34,[184,561,562],{"class":190},"        return",[184,564,565],{"class":211}," ctx.",[184,567,568],{"class":194},"Err",[184,570,571],{"class":211},"()\n",[184,573,575,577,579,582,585,587,591,594],{"class":186,"line":574},35,[184,576,544],{"class":190},[184,578,547],{"class":190},[184,580,581],{"class":211},"time.",[184,583,584],{"class":194},"After",[184,586,436],{"class":211},[184,588,590],{"class":589},"sDLfK","50",[184,592,593],{"class":190}," *",[184,595,596],{"class":211}," time.Millisecond):\n",[184,598,600,602],{"class":186,"line":599},36,[184,601,562],{"class":190},[184,603,604],{"class":589}," nil\n",[184,606,608],{"class":186,"line":607},37,[184,609,610],{"class":211},"    }\n",[184,612,614],{"class":186,"line":613},38,[184,615,409],{"class":211},[184,617,619],{"class":186,"line":618},39,[184,620,202],{"emptyLinePlaceholder":201},[184,622,624,626,629],{"class":186,"line":623},40,[184,625,496],{"class":190},[184,627,628],{"class":194}," main",[184,630,631],{"class":211},"() {\n",[184,633,635,638,641,644,647,650,653,656,659],{"class":186,"line":634},41,[184,636,637],{"class":211},"    logger ",[184,639,640],{"class":190},":=",[184,642,643],{"class":211}," slog.",[184,645,646],{"class":194},"New",[184,648,649],{"class":211},"(slog.",[184,651,652],{"class":194},"NewTextHandler",[184,654,655],{"class":211},"(os.Stdout, ",[184,657,658],{"class":589},"nil",[184,660,661],{"class":211},"))\n",[184,663,665],{"class":186,"line":664},42,[184,666,202],{"emptyLinePlaceholder":201},[184,668,670,673,675,678,681,684,687],{"class":186,"line":669},43,[184,671,672],{"class":211},"    ctx, stop ",[184,674,640],{"class":190},[184,676,677],{"class":211}," signal.",[184,679,680],{"class":194},"NotifyContext",[184,682,683],{"class":211},"(context.",[184,685,686],{"class":194},"Background",[184,688,689],{"class":211},"(), os.Interrupt, syscall.SIGTERM)\n",[184,691,693,696,699],{"class":186,"line":692},44,[184,694,695],{"class":190},"    defer",[184,697,698],{"class":194}," stop",[184,700,571],{"class":211},[184,702,704],{"class":186,"line":703},45,[184,705,202],{"emptyLinePlaceholder":201},[184,707,709,712,714,717,720,722,725,727,730],{"class":186,"line":708},46,[184,710,711],{"class":211},"    reader ",[184,713,640],{"class":190},[184,715,716],{"class":211}," kafka.",[184,718,719],{"class":194},"NewReader",[184,721,436],{"class":211},[184,723,724],{"class":194},"kafka",[184,726,27],{"class":211},[184,728,729],{"class":194},"ReaderConfig",[184,731,732],{"class":211},"{\n",[184,734,736,739,741,744,747],{"class":186,"line":735},47,[184,737,738],{"class":211},"        Brokers:        []",[184,740,355],{"class":190},[184,742,743],{"class":211},"{",[184,745,746],{"class":218},"\"localhost:9092\"",[184,748,749],{"class":211},"},\n",[184,751,753,756,759],{"class":186,"line":752},48,[184,754,755],{"class":211},"        Topic:          ",[184,757,758],{"class":218},"\"orders.events\"",[184,760,761],{"class":211},",\n",[184,763,765,768,771],{"class":186,"line":764},49,[184,766,767],{"class":211},"        GroupID:        ",[184,769,770],{"class":218},"\"billing-service\"",[184,772,761],{"class":211},[184,774,776,779,782],{"class":186,"line":775},50,[184,777,778],{"class":211},"        MinBytes:       ",[184,780,781],{"class":589},"1",[184,783,761],{"class":211},[184,785,787,790,793,796,799],{"class":186,"line":786},51,[184,788,789],{"class":211},"        MaxBytes:       ",[184,791,792],{"class":589},"10",[184,794,795],{"class":190},"e",[184,797,798],{"class":589},"6",[184,800,761],{"class":211},[184,802,804,807,810,812],{"class":186,"line":803},52,[184,805,806],{"class":211},"        CommitInterval: ",[184,808,809],{"class":589},"0",[184,811,451],{"class":211},[184,813,815],{"class":814},"sAwPA","\u002F\u002F manual commit\n",[184,817,819],{"class":186,"line":818},53,[184,820,821],{"class":211},"    })\n",[184,823,825,827,830],{"class":186,"line":824},54,[184,826,695],{"class":190},[184,828,829],{"class":190}," func",[184,831,631],{"class":211},[184,833,835,838,841,843,846,849,852,855,858],{"class":186,"line":834},55,[184,836,837],{"class":190},"        if",[184,839,840],{"class":211}," err ",[184,842,640],{"class":190},[184,844,845],{"class":211}," reader.",[184,847,848],{"class":194},"Close",[184,850,851],{"class":211},"(); err ",[184,853,854],{"class":190},"!=",[184,856,857],{"class":589}," nil",[184,859,346],{"class":211},[184,861,863,866,869,871,874,876,879],{"class":186,"line":862},56,[184,864,865],{"class":211},"            logger.",[184,867,868],{"class":194},"Error",[184,870,436],{"class":211},[184,872,873],{"class":218},"\"close kafka reader\"",[184,875,451],{"class":211},[184,877,878],{"class":218},"\"error\"",[184,880,881],{"class":211},", err)\n",[184,883,885],{"class":186,"line":884},57,[184,886,887],{"class":211},"        }\n",[184,889,891],{"class":186,"line":890},58,[184,892,893],{"class":211},"    }()\n",[184,895,897],{"class":186,"line":896},59,[184,898,202],{"emptyLinePlaceholder":201},[184,900,902,905,907,909,912,915,917,920,922,924,927,930,933,936],{"class":186,"line":901},60,[184,903,904],{"class":190},"    if",[184,906,840],{"class":211},[184,908,640],{"class":190},[184,910,911],{"class":194}," consume",[184,913,914],{"class":211},"(ctx, reader, ",[184,916,502],{"class":194},[184,918,919],{"class":211},"{}, logger); err ",[184,921,854],{"class":190},[184,923,857],{"class":589},[184,925,926],{"class":190}," &&",[184,928,929],{"class":190}," !",[184,931,932],{"class":211},"errors.",[184,934,935],{"class":194},"Is",[184,937,938],{"class":211},"(err, context.Canceled) {\n",[184,940,942,945,947,949,952,954,956],{"class":186,"line":941},61,[184,943,944],{"class":211},"        logger.",[184,946,868],{"class":194},[184,948,436],{"class":211},[184,950,951],{"class":218},"\"consumer stopped\"",[184,953,451],{"class":211},[184,955,878],{"class":218},[184,957,881],{"class":211},[184,959,961,964,967,969,971],{"class":186,"line":960},62,[184,962,963],{"class":211},"        os.",[184,965,966],{"class":194},"Exit",[184,968,436],{"class":211},[184,970,781],{"class":589},[184,972,326],{"class":211},[184,974,976],{"class":186,"line":975},63,[184,977,610],{"class":211},[184,979,981],{"class":186,"line":980},64,[184,982,409],{"class":211},[184,984,986],{"class":186,"line":985},65,[184,987,202],{"emptyLinePlaceholder":201},[184,989,991,993,995,997,999,1001,1003,1005,1007,1010,1012,1014,1016,1019,1021,1024,1026,1028,1031,1033,1036,1038,1041,1043,1045],{"class":186,"line":990},66,[184,992,496],{"class":190},[184,994,911],{"class":194},[184,996,436],{"class":211},[184,998,440],{"class":439},[184,1000,443],{"class":194},[184,1002,27],{"class":211},[184,1004,448],{"class":194},[184,1006,451],{"class":211},[184,1008,1009],{"class":439},"reader",[184,1011,593],{"class":190},[184,1013,724],{"class":194},[184,1015,27],{"class":211},[184,1017,1018],{"class":194},"Reader",[184,1020,451],{"class":211},[184,1022,1023],{"class":439},"handler",[184,1025,422],{"class":194},[184,1027,451],{"class":211},[184,1029,1030],{"class":439},"logger",[184,1032,593],{"class":190},[184,1034,1035],{"class":194},"slog",[184,1037,27],{"class":211},[184,1039,1040],{"class":194},"Logger",[184,1042,459],{"class":211},[184,1044,528],{"class":190},[184,1046,346],{"class":211},[184,1048,1050,1053],{"class":186,"line":1049},67,[184,1051,1052],{"class":190},"    for",[184,1054,346],{"class":211},[184,1056,1058,1061,1063,1065,1067],{"class":186,"line":1057},68,[184,1059,1060],{"class":211},"        msg, err ",[184,1062,640],{"class":190},[184,1064,845],{"class":211},[184,1066,169],{"class":194},[184,1068,1069],{"class":211},"(ctx)\n",[184,1071,1073,1075,1077,1079,1081],{"class":186,"line":1072},69,[184,1074,837],{"class":190},[184,1076,840],{"class":211},[184,1078,854],{"class":190},[184,1080,857],{"class":589},[184,1082,346],{"class":211},[184,1084,1086,1089,1092,1095,1097,1100,1103,1106],{"class":186,"line":1085},70,[184,1087,1088],{"class":190},"            return",[184,1090,1091],{"class":211}," fmt.",[184,1093,1094],{"class":194},"Errorf",[184,1096,436],{"class":211},[184,1098,1099],{"class":218},"\"fetch message: ",[184,1101,1102],{"class":589},"%w",[184,1104,1105],{"class":218},"\"",[184,1107,881],{"class":211},[184,1109,1111],{"class":186,"line":1110},71,[184,1112,887],{"class":211},[184,1114,1116],{"class":186,"line":1115},72,[184,1117,202],{"emptyLinePlaceholder":201},[184,1119,1121,1123,1125,1127,1130,1133,1135,1137],{"class":186,"line":1120},73,[184,1122,837],{"class":190},[184,1124,840],{"class":211},[184,1126,640],{"class":190},[184,1128,1129],{"class":194}," handleMessage",[184,1131,1132],{"class":211},"(ctx, handler, msg); err ",[184,1134,854],{"class":190},[184,1136,857],{"class":589},[184,1138,346],{"class":211},[184,1140,1142,1144,1146,1148,1151],{"class":186,"line":1141},74,[184,1143,865],{"class":211},[184,1145,868],{"class":194},[184,1147,436],{"class":211},[184,1149,1150],{"class":218},"\"handle message failed\"",[184,1152,761],{"class":211},[184,1154,1156,1159],{"class":186,"line":1155},75,[184,1157,1158],{"class":218},"                \"topic\"",[184,1160,1161],{"class":211},", msg.Topic,\n",[184,1163,1165,1168],{"class":186,"line":1164},76,[184,1166,1167],{"class":218},"                \"partition\"",[184,1169,1170],{"class":211},", msg.Partition,\n",[184,1172,1174,1177],{"class":186,"line":1173},77,[184,1175,1176],{"class":218},"                \"offset\"",[184,1178,1179],{"class":211},", msg.Offset,\n",[184,1181,1183,1186],{"class":186,"line":1182},78,[184,1184,1185],{"class":218},"                \"error\"",[184,1187,1188],{"class":211},", err,\n",[184,1190,1192],{"class":186,"line":1191},79,[184,1193,1194],{"class":211},"            )\n",[184,1196,1198,1200],{"class":186,"line":1197},80,[184,1199,1088],{"class":190},[184,1201,1202],{"class":211}," err\n",[184,1204,1206],{"class":186,"line":1205},81,[184,1207,887],{"class":211},[184,1209,1211],{"class":186,"line":1210},82,[184,1212,202],{"emptyLinePlaceholder":201},[184,1214,1216,1218,1220,1222,1224,1226,1229,1231,1233],{"class":186,"line":1215},83,[184,1217,837],{"class":190},[184,1219,840],{"class":211},[184,1221,640],{"class":190},[184,1223,845],{"class":211},[184,1225,173],{"class":194},[184,1227,1228],{"class":211},"(ctx, msg); err ",[184,1230,854],{"class":190},[184,1232,857],{"class":589},[184,1234,346],{"class":211},[184,1236,1238,1240,1242,1244,1246,1249,1252,1255,1258,1261,1263,1266,1268,1270],{"class":186,"line":1237},84,[184,1239,1088],{"class":190},[184,1241,1091],{"class":211},[184,1243,1094],{"class":194},[184,1245,436],{"class":211},[184,1247,1248],{"class":218},"\"commit offset topic=",[184,1250,1251],{"class":589},"%s",[184,1253,1254],{"class":218}," partition=",[184,1256,1257],{"class":589},"%d",[184,1259,1260],{"class":218}," offset=",[184,1262,1257],{"class":589},[184,1264,1265],{"class":218},": ",[184,1267,1102],{"class":589},[184,1269,1105],{"class":218},[184,1271,761],{"class":211},[184,1273,1275],{"class":186,"line":1274},85,[184,1276,1277],{"class":211},"                msg.Topic, msg.Partition, msg.Offset, err)\n",[184,1279,1281],{"class":186,"line":1280},86,[184,1282,887],{"class":211},[184,1284,1286],{"class":186,"line":1285},87,[184,1287,610],{"class":211},[184,1289,1291],{"class":186,"line":1290},88,[184,1292,409],{"class":211},[184,1294,1296],{"class":186,"line":1295},89,[184,1297,202],{"emptyLinePlaceholder":201},[184,1299,1301,1303,1305,1307,1309,1311,1313,1315,1317,1319,1321,1323,1326,1329,1331,1334,1336,1338],{"class":186,"line":1300},90,[184,1302,496],{"class":190},[184,1304,1129],{"class":194},[184,1306,436],{"class":211},[184,1308,440],{"class":439},[184,1310,443],{"class":194},[184,1312,27],{"class":211},[184,1314,448],{"class":194},[184,1316,451],{"class":211},[184,1318,1023],{"class":439},[184,1320,422],{"class":194},[184,1322,451],{"class":211},[184,1324,1325],{"class":439},"msg",[184,1327,1328],{"class":194}," kafka",[184,1330,27],{"class":211},[184,1332,1333],{"class":194},"Message",[184,1335,459],{"class":211},[184,1337,528],{"class":190},[184,1339,346],{"class":211},[184,1341,1343,1346,1349],{"class":186,"line":1342},91,[184,1344,1345],{"class":190},"    var",[184,1347,1348],{"class":211}," event ",[184,1350,1351],{"class":194},"OrderCreated\n",[184,1353,1355,1357,1359,1361,1364,1367,1370,1373,1376,1378,1380],{"class":186,"line":1354},92,[184,1356,904],{"class":190},[184,1358,840],{"class":211},[184,1360,640],{"class":190},[184,1362,1363],{"class":211}," json.",[184,1365,1366],{"class":194},"Unmarshal",[184,1368,1369],{"class":211},"(msg.Value, ",[184,1371,1372],{"class":190},"&",[184,1374,1375],{"class":211},"event); err ",[184,1377,854],{"class":190},[184,1379,857],{"class":589},[184,1381,346],{"class":211},[184,1383,1385,1387,1389,1391,1393,1396,1398,1400],{"class":186,"line":1384},93,[184,1386,562],{"class":190},[184,1388,1091],{"class":211},[184,1390,1094],{"class":194},[184,1392,436],{"class":211},[184,1394,1395],{"class":218},"\"decode order created: ",[184,1397,1102],{"class":589},[184,1399,1105],{"class":218},[184,1401,881],{"class":211},[184,1403,1405],{"class":186,"line":1404},94,[184,1406,610],{"class":211},[184,1408,1410],{"class":186,"line":1409},95,[184,1411,202],{"emptyLinePlaceholder":201},[184,1413,1415,1418,1420,1423,1426,1429,1431,1434],{"class":186,"line":1414},96,[184,1416,1417],{"class":211},"    handleCtx, cancel ",[184,1419,640],{"class":190},[184,1421,1422],{"class":211}," context.",[184,1424,1425],{"class":194},"WithTimeout",[184,1427,1428],{"class":211},"(ctx, ",[184,1430,792],{"class":589},[184,1432,1433],{"class":190},"*",[184,1435,1436],{"class":211},"time.Second)\n",[184,1438,1440,1442,1445],{"class":186,"line":1439},97,[184,1441,695],{"class":190},[184,1443,1444],{"class":194}," cancel",[184,1446,571],{"class":211},[184,1448,1450],{"class":186,"line":1449},98,[184,1451,202],{"emptyLinePlaceholder":201},[184,1453,1455,1457,1459,1461,1464,1466,1469,1471,1473],{"class":186,"line":1454},99,[184,1456,904],{"class":190},[184,1458,840],{"class":211},[184,1460,640],{"class":190},[184,1462,1463],{"class":211}," handler.",[184,1465,507],{"class":194},[184,1467,1468],{"class":211},"(handleCtx, event); err ",[184,1470,854],{"class":190},[184,1472,857],{"class":589},[184,1474,346],{"class":211},[184,1476,1478,1480,1482,1484,1486,1489,1491,1493,1495,1497],{"class":186,"line":1477},100,[184,1479,562],{"class":190},[184,1481,1091],{"class":211},[184,1483,1094],{"class":194},[184,1485,436],{"class":211},[184,1487,1488],{"class":218},"\"handle order created event_id=",[184,1490,1251],{"class":589},[184,1492,1265],{"class":218},[184,1494,1102],{"class":589},[184,1496,1105],{"class":218},[184,1498,1499],{"class":211},", event.EventID, err)\n",[184,1501,1503],{"class":186,"line":1502},101,[184,1504,610],{"class":211},[184,1506,1508],{"class":186,"line":1507},102,[184,1509,202],{"emptyLinePlaceholder":201},[184,1511,1513,1516],{"class":186,"line":1512},103,[184,1514,1515],{"class":190},"    return",[184,1517,604],{"class":589},[184,1519,1521],{"class":186,"line":1520},104,[184,1522,409],{"class":211},[15,1524,1525,1526,1528],{},"Если handler успешно завершился, offset commit фиксирует прогресс. Если процесс упал после обработки, но до commit, Kafka отдаст сообщение снова. Поэтому ",[19,1527,507],{}," должен быть идемпотентным.",[15,1530,1531,1532,1535],{},"Обратите внимание на decode errors. Если payload невалидный и вы просто вернёте ошибку из ",[19,1533,1534],{},"consume",", consumer будет снова падать на том же offset после restart. Для production нужен отдельный путь: классифицировать ошибку как non-retriable, записать сообщение в DLQ с metadata, и только после успешной записи в DLQ commit исходный offset.",[45,1537],{},[48,1539,1541],{"id":1540},"идемпотентная-обработка","Идемпотентная обработка",[15,1543,1544],{},"Идемпотентность значит: повторная обработка того же события не меняет результат второй раз.",[15,1546,1547],{},"Типичные техники:",[83,1549,1550,1560],{},[86,1551,1552],{},[89,1553,1554,1557],{},[92,1555,1556],{"align":94},"Техника",[92,1558,1559],{"align":94},"Пример",[103,1561,1562,1576,1586,1601],{},[89,1563,1564,1570],{},[108,1565,1566,1567],{"align":94},"Уникальный ",[19,1568,1569],{},"event_id",[108,1571,1572,1573,27],{"align":94},"Таблица ",[19,1574,1575],{},"processed_events(event_id primary key)",[89,1577,1578,1581],{},[108,1579,1580],{"align":94},"Upsert",[108,1582,1583,27],{"align":94},[19,1584,1585],{},"INSERT ... ON CONFLICT DO UPDATE",[89,1587,1588,1591],{},[108,1589,1590],{"align":94},"State transition guard",[108,1592,1593,1594,1597,1598,27],{"align":94},"Нельзя перевести ",[19,1595,1596],{},"paid"," обратно в ",[19,1599,1600],{},"created",[89,1602,1603,1606],{},[108,1604,1605],{"align":94},"External idempotency key",[108,1607,1608],{"align":94},"При вызове payment\u002Femail API передать idempotency key.",[15,1610,1611],{},"Псевдо-flow для БД:",[32,1613,1616],{"className":1614,"code":1615,"language":37,"meta":38},[35],"BEGIN\n  INSERT INTO processed_events(event_id) VALUES ($1)\n    ON CONFLICT DO NOTHING\n\n  if inserted:\n    apply business changes\nCOMMIT\ncommit kafka offset\n",[19,1617,1615],{"__ignoreMap":38},[15,1619,1620],{},"Kafka offset commit не должен быть единственной защитой от дублей. Offset - это позиция чтения group, а не бизнес-инвариант.",[15,1622,1623,1624,1627,1628,1630],{},"Для RateDesk consumer ",[19,1625,1626],{},"RateUpdated"," может быть идемпотентным по ",[19,1629,1569],{},", но иногда полезен и доменный guard: не применять событие со старой версией курса поверх более новой read model. Это не заменяет таблицу обработанных событий, а защищает от out-of-order replay и ручных backfill.",[45,1632],{},[48,1634,1636],{"id":1635},"backpressure","Backpressure",[15,1638,1639],{},"Consumer должен уметь тормозить чтение, если обработка не успевает. Иначе вы получите рост памяти, timeouts, rebalance и лавину повторов.",[15,1641,1642],{},"Самая простая стратегия: читать следующее сообщение только после обработки предыдущего. Это безопасно для порядка, но ограничивает throughput.",[15,1644,1645],{},"Для параллелизма используют worker pool, но с ограниченной очередью:",[32,1647,1649],{"className":177,"code":1648,"language":179,"meta":180,"style":38},"type Job struct {\n    Message kafka.Message\n}\n\nfunc startWorkers(ctx context.Context, n int, jobs \u003C-chan Job, handler Handler, logger *slog.Logger) {\n    for i := 0; i \u003C n; i++ {\n        go func(workerID int) {\n            for {\n                select {\n                case \u003C-ctx.Done():\n                    return\n                case job, ok := \u003C-jobs:\n                    if !ok {\n                        return\n                    }\n                    if err := handleMessage(ctx, handler, job.Message); err != nil {\n                        logger.Error(\"worker handle failed\", \"worker\", workerID, \"error\", err)\n                    }\n                }\n            }\n        }(i)\n    }\n}\n",[19,1650,1651,1662,1674,1678,1682,1738,1764,1780,1787,1794,1807,1812,1826,1836,1841,1846,1865,1889,1893,1898,1903,1908,1912],{"__ignoreMap":38},[184,1652,1653,1655,1658,1660],{"class":186,"line":187},[184,1654,337],{"class":190},[184,1656,1657],{"class":194}," Job",[184,1659,343],{"class":190},[184,1661,346],{"class":211},[184,1663,1664,1667,1669,1671],{"class":186,"line":198},[184,1665,1666],{"class":211},"    Message ",[184,1668,724],{"class":194},[184,1670,27],{"class":211},[184,1672,1673],{"class":194},"Message\n",[184,1675,1676],{"class":186,"line":205},[184,1677,409],{"class":211},[184,1679,1680],{"class":186,"line":215},[184,1681,202],{"emptyLinePlaceholder":201},[184,1683,1684,1686,1689,1691,1693,1695,1697,1699,1701,1704,1707,1709,1712,1715,1717,1719,1721,1723,1725,1727,1729,1731,1733,1735],{"class":186,"line":228},[184,1685,496],{"class":190},[184,1687,1688],{"class":194}," startWorkers",[184,1690,436],{"class":211},[184,1692,440],{"class":439},[184,1694,443],{"class":194},[184,1696,27],{"class":211},[184,1698,448],{"class":194},[184,1700,451],{"class":211},[184,1702,1703],{"class":439},"n",[184,1705,1706],{"class":190}," int",[184,1708,451],{"class":211},[184,1710,1711],{"class":439},"jobs",[184,1713,1714],{"class":190}," \u003C-chan",[184,1716,1657],{"class":194},[184,1718,451],{"class":211},[184,1720,1023],{"class":439},[184,1722,422],{"class":194},[184,1724,451],{"class":211},[184,1726,1030],{"class":439},[184,1728,593],{"class":190},[184,1730,1035],{"class":194},[184,1732,27],{"class":211},[184,1734,1040],{"class":194},[184,1736,1737],{"class":211},") {\n",[184,1739,1740,1742,1745,1747,1750,1753,1756,1759,1762],{"class":186,"line":238},[184,1741,1052],{"class":190},[184,1743,1744],{"class":211}," i ",[184,1746,640],{"class":190},[184,1748,1749],{"class":589}," 0",[184,1751,1752],{"class":211},"; i ",[184,1754,1755],{"class":190},"\u003C",[184,1757,1758],{"class":211}," n; i",[184,1760,1761],{"class":190},"++",[184,1763,346],{"class":211},[184,1765,1766,1769,1771,1773,1776,1778],{"class":186,"line":248},[184,1767,1768],{"class":190},"        go",[184,1770,829],{"class":190},[184,1772,436],{"class":211},[184,1774,1775],{"class":439},"workerID",[184,1777,1706],{"class":190},[184,1779,1737],{"class":211},[184,1781,1782,1785],{"class":186,"line":258},[184,1783,1784],{"class":190},"            for",[184,1786,346],{"class":211},[184,1788,1789,1792],{"class":186,"line":268},[184,1790,1791],{"class":190},"                select",[184,1793,346],{"class":211},[184,1795,1796,1799,1801,1803,1805],{"class":186,"line":278},[184,1797,1798],{"class":190},"                case",[184,1800,547],{"class":190},[184,1802,550],{"class":211},[184,1804,553],{"class":194},[184,1806,556],{"class":211},[184,1808,1809],{"class":186,"line":288},[184,1810,1811],{"class":190},"                    return\n",[184,1813,1814,1816,1819,1821,1823],{"class":186,"line":298},[184,1815,1798],{"class":190},[184,1817,1818],{"class":211}," job, ok ",[184,1820,640],{"class":190},[184,1822,547],{"class":190},[184,1824,1825],{"class":211},"jobs:\n",[184,1827,1828,1831,1833],{"class":186,"line":308},[184,1829,1830],{"class":190},"                    if",[184,1832,929],{"class":190},[184,1834,1835],{"class":211},"ok {\n",[184,1837,1838],{"class":186,"line":313},[184,1839,1840],{"class":190},"                        return\n",[184,1842,1843],{"class":186,"line":323},[184,1844,1845],{"class":211},"                    }\n",[184,1847,1848,1850,1852,1854,1856,1859,1861,1863],{"class":186,"line":329},[184,1849,1830],{"class":190},[184,1851,840],{"class":211},[184,1853,640],{"class":190},[184,1855,1129],{"class":194},[184,1857,1858],{"class":211},"(ctx, handler, job.Message); err ",[184,1860,854],{"class":190},[184,1862,857],{"class":589},[184,1864,346],{"class":211},[184,1866,1867,1870,1872,1874,1877,1879,1882,1885,1887],{"class":186,"line":334},[184,1868,1869],{"class":211},"                        logger.",[184,1871,868],{"class":194},[184,1873,436],{"class":211},[184,1875,1876],{"class":218},"\"worker handle failed\"",[184,1878,451],{"class":211},[184,1880,1881],{"class":218},"\"worker\"",[184,1883,1884],{"class":211},", workerID, ",[184,1886,878],{"class":218},[184,1888,881],{"class":211},[184,1890,1891],{"class":186,"line":349},[184,1892,1845],{"class":211},[184,1894,1895],{"class":186,"line":361},[184,1896,1897],{"class":211},"                }\n",[184,1899,1900],{"class":186,"line":372},[184,1901,1902],{"class":211},"            }\n",[184,1904,1905],{"class":186,"line":383},[184,1906,1907],{"class":211},"        }(i)\n",[184,1909,1910],{"class":186,"line":395},[184,1911,610],{"class":211},[184,1913,1914],{"class":186,"line":406},[184,1915,409],{"class":211},[15,1917,1918],{},"Но здесь есть подвох: commit offsets становится сложнее. Нельзя просто commit offset 105, если 104 ещё обрабатывается. Для строгого порядка проще держать обработку partition последовательной. Для высокой нагрузки делают partition-aware workers и трекер завершённых offsets.",[15,1920,1921],{},"Если worker pool шардируется по key, очередь каждого shard должна быть ограниченной. Иначе один медленный внешний сервис превратит consumer в неограниченный буфер в памяти, heartbeat начнёт срываться, group уйдёт в rebalance, а после restart вы получите ещё больше дублей.",[45,1923],{},[48,1925,1927],{"id":1926},"graceful-shutdown","Graceful shutdown",[15,1929,1930],{},"При SIGTERM consumer должен:",[1932,1933,1934,1938,1941,1944],"ol",{},[1935,1936,1937],"li",{},"перестать брать новые сообщения;",[1935,1939,1940],{},"дождаться текущей обработки или отменить её по timeout;",[1935,1942,1943],{},"commit только успешно обработанные сообщения;",[1935,1945,1946],{},"закрыть reader, чтобы group быстрее увидела уход consumer.",[15,1948,1949,1952],{},[19,1950,1951],{},"signal.NotifyContext"," помогает связать shutdown с context. Но не надо использовать один бесконечный context без deadlines: если handler зависнет, shutdown тоже зависнет.",[32,1954,1956],{"className":177,"code":1955,"language":179,"meta":180,"style":38},"shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)\ndefer cancel()\n\ndone := make(chan error, 1)\ngo func() {\n    done \u003C- consume(ctx, reader, handler, logger)\n}()\n\nselect {\ncase \u003C-ctx.Done():\n    logger.Info(\"shutdown requested\")\n    _ = reader.Close()\n    select {\n    case err := \u003C-done:\n        logger.Info(\"consumer stopped\", \"error\", err)\n    case \u003C-shutdownCtx.Done():\n        logger.Error(\"consumer shutdown timeout\")\n    }\ncase err := \u003C-done:\n    logger.Error(\"consumer exited\", \"error\", err)\n}\n",[19,1957,1958,1983,1992,1996,2020,2028,2041,2046,2050,2057,2070,2085,2099,2105,2118,2134,2147,2160,2164,2176,2193],{"__ignoreMap":38},[184,1959,1960,1963,1965,1967,1969,1971,1973,1976,1979,1981],{"class":186,"line":187},[184,1961,1962],{"class":211},"shutdownCtx, cancel ",[184,1964,640],{"class":190},[184,1966,1422],{"class":211},[184,1968,1425],{"class":194},[184,1970,683],{"class":211},[184,1972,686],{"class":194},[184,1974,1975],{"class":211},"(), ",[184,1977,1978],{"class":589},"30",[184,1980,1433],{"class":190},[184,1982,1436],{"class":211},[184,1984,1985,1988,1990],{"class":186,"line":198},[184,1986,1987],{"class":190},"defer",[184,1989,1444],{"class":194},[184,1991,571],{"class":211},[184,1993,1994],{"class":186,"line":205},[184,1995,202],{"emptyLinePlaceholder":201},[184,1997,1998,2001,2003,2006,2008,2011,2014,2016,2018],{"class":186,"line":215},[184,1999,2000],{"class":211},"done ",[184,2002,640],{"class":190},[184,2004,2005],{"class":194}," make",[184,2007,436],{"class":211},[184,2009,2010],{"class":190},"chan",[184,2012,2013],{"class":190}," error",[184,2015,451],{"class":211},[184,2017,781],{"class":589},[184,2019,326],{"class":211},[184,2021,2022,2024,2026],{"class":186,"line":228},[184,2023,179],{"class":190},[184,2025,829],{"class":190},[184,2027,631],{"class":211},[184,2029,2030,2033,2036,2038],{"class":186,"line":238},[184,2031,2032],{"class":211},"    done ",[184,2034,2035],{"class":190},"\u003C-",[184,2037,911],{"class":194},[184,2039,2040],{"class":211},"(ctx, reader, handler, logger)\n",[184,2042,2043],{"class":186,"line":248},[184,2044,2045],{"class":211},"}()\n",[184,2047,2048],{"class":186,"line":258},[184,2049,202],{"emptyLinePlaceholder":201},[184,2051,2052,2055],{"class":186,"line":268},[184,2053,2054],{"class":190},"select",[184,2056,346],{"class":211},[184,2058,2059,2062,2064,2066,2068],{"class":186,"line":278},[184,2060,2061],{"class":190},"case",[184,2063,547],{"class":190},[184,2065,550],{"class":211},[184,2067,553],{"class":194},[184,2069,556],{"class":211},[184,2071,2072,2075,2078,2080,2083],{"class":186,"line":288},[184,2073,2074],{"class":211},"    logger.",[184,2076,2077],{"class":194},"Info",[184,2079,436],{"class":211},[184,2081,2082],{"class":218},"\"shutdown requested\"",[184,2084,326],{"class":211},[184,2086,2087,2090,2093,2095,2097],{"class":186,"line":298},[184,2088,2089],{"class":211},"    _ ",[184,2091,2092],{"class":190},"=",[184,2094,845],{"class":211},[184,2096,848],{"class":194},[184,2098,571],{"class":211},[184,2100,2101,2103],{"class":186,"line":308},[184,2102,536],{"class":190},[184,2104,346],{"class":211},[184,2106,2107,2109,2111,2113,2115],{"class":186,"line":313},[184,2108,544],{"class":190},[184,2110,840],{"class":211},[184,2112,640],{"class":190},[184,2114,547],{"class":190},[184,2116,2117],{"class":211},"done:\n",[184,2119,2120,2122,2124,2126,2128,2130,2132],{"class":186,"line":323},[184,2121,944],{"class":211},[184,2123,2077],{"class":194},[184,2125,436],{"class":211},[184,2127,951],{"class":218},[184,2129,451],{"class":211},[184,2131,878],{"class":218},[184,2133,881],{"class":211},[184,2135,2136,2138,2140,2143,2145],{"class":186,"line":329},[184,2137,544],{"class":190},[184,2139,547],{"class":190},[184,2141,2142],{"class":211},"shutdownCtx.",[184,2144,553],{"class":194},[184,2146,556],{"class":211},[184,2148,2149,2151,2153,2155,2158],{"class":186,"line":334},[184,2150,944],{"class":211},[184,2152,868],{"class":194},[184,2154,436],{"class":211},[184,2156,2157],{"class":218},"\"consumer shutdown timeout\"",[184,2159,326],{"class":211},[184,2161,2162],{"class":186,"line":349},[184,2163,610],{"class":211},[184,2165,2166,2168,2170,2172,2174],{"class":186,"line":361},[184,2167,2061],{"class":190},[184,2169,840],{"class":211},[184,2171,640],{"class":190},[184,2173,547],{"class":190},[184,2175,2117],{"class":211},[184,2177,2178,2180,2182,2184,2187,2189,2191],{"class":186,"line":372},[184,2179,2074],{"class":211},[184,2181,868],{"class":194},[184,2183,436],{"class":211},[184,2185,2186],{"class":218},"\"consumer exited\"",[184,2188,451],{"class":211},[184,2190,878],{"class":218},[184,2192,881],{"class":211},[184,2194,2195],{"class":186,"line":383},[184,2196,409],{"class":211},[15,2198,2199,2200,2203],{},"В реальном сервисе shutdown orchestration обычно лежит в ",[19,2201,2202],{},"main",": HTTP server, Kafka consumers, metrics и DB pool закрываются согласованно.",[15,2205,2206],{},"Во время shutdown важно различать отмену процесса и отмену handler. Если Kubernetes даёт 30 секунд на termination, consumer не должен начинать новую долгую обработку на 29-й секунде. Часто используют отдельный draining flag: перестать fetch, дать текущим handlers короткое окно на завершение, затем закрыть reader и разрешить group перераспределить partitions.",[45,2208],{},[48,2210,2212],{"id":2211},"observability-consumer-а","Observability consumer-а",[15,2214,2215,2216,2218],{},"Для consumer полезны не только ",[19,2217,528],{}," logs:",[2220,2221,2222,2229,2232,2235,2238,2241,2244],"ul",{},[1935,2223,2224,2225,2228],{},"lag по ",[19,2226,2227],{},"topic\u002Fgroup\u002Fpartition",", а не только общий lag;",[1935,2230,2231],{},"duration handler-а и commit latency;",[1935,2233,2234],{},"количество duplicates или conflicts в idempotency table;",[1935,2236,2237],{},"decode\u002Fnon-retriable\u002Fretriable errors раздельно;",[1935,2239,2240],{},"rebalance count и время без assignment;",[1935,2242,2243],{},"DLQ write success\u002Ferror rate;",[1935,2245,2246,451,2248,451,2251,451,2254,451,2257,451,2260,2263],{},[19,2247,1569],{},[19,2249,2250],{},"event_type",[19,2252,2253],{},"topic",[19,2255,2256],{},"partition",[19,2258,2259],{},"offset",[19,2261,2262],{},"consumer_group"," в structured logs.",[15,2265,2266,2267,451,2269,451,2272,2275],{},"Следите за cardinality: не делайте ",[19,2268,1569],{},[19,2270,2271],{},"user_id",[19,2273,2274],{},"order_id"," или currency pair label-ами Prometheus-метрик без сильной причины. Такие значения лучше держать в логах и traces.",[45,2277],{},[48,2279,2281],{"id":2280},"poison-messages","Poison messages",[15,2283,2284],{},"Если message всегда падает на обработке, consumer может застрять на одном offset. Это poison message.",[15,2286,2287],{},"Варианты:",[2220,2289,2290,2293,2296,2299],{},[1935,2291,2292],{},"retry несколько раз с backoff;",[1935,2294,2295],{},"отправить событие в DLQ;",[1935,2297,2298],{},"commit исходный offset после успешной записи в DLQ;",[1935,2300,2301],{},"поднять alert, потому что DLQ - не мусорка, а очередь расследования.",[15,2303,2304],{},"Эту тему разберём дальше вместе с retry topics и rebalancing.",[45,2306],{},[48,2308,2310],{"id":2309},"вопросы-на-собеседовании","Вопросы на собеседовании",[2220,2312,2313,2316,2319,2328,2331,2334,2337,2340,2343,2346],{},[1935,2314,2315],{},"Что такое consumer group?",[1935,2317,2318],{},"Почему consumers больше, чем partitions, не увеличивают throughput?",[1935,2320,2321,2322,2324,2325,2327],{},"Чем ",[19,2323,169],{}," + ",[19,2326,173],{}," отличается от auto commit?",[1935,2329,2330],{},"Когда возникает at-least-once delivery?",[1935,2332,2333],{},"Почему at-least-once требует идемпотентности?",[1935,2335,2336],{},"Как worker pool может сломать порядок обработки?",[1935,2338,2339],{},"Что такое backpressure в consumer?",[1935,2341,2342],{},"Как правильно остановить consumer при SIGTERM?",[1935,2344,2345],{},"Почему decode error нельзя бесконечно retry без DLQ?",[1935,2347,2348],{},"Какие consumer metrics нужны для дежурства?",[45,2350],{},[48,2352,2354],{"id":2353},"практика","Практика",[1932,2356,2357,2367,2370,2375],{},[1935,2358,2359,2360,2362,2363,2366],{},"Допишите handler, который сохраняет ",[19,2361,1569],{}," в таблицу ",[19,2364,2365],{},"processed_events"," и пропускает duplicate.",[1935,2368,2369],{},"Объясните, что случится, если commit offset сделать до записи в БД.",[1935,2371,2372,2373,27],{},"Спроектируйте worker pool для topic, где порядок важен только внутри ",[19,2374,2271],{},[1935,2376,2377],{},"Добавьте обработку SIGTERM с timeout 30 секунд для учебного consumer.",[45,2379],{},[48,2381,2383],{"id":2382},"интерактивная-практика","Интерактивная практика",[2385,2386,2389,2392,2412],"quiz",{"answer":2387,"id":2388,"xp":792},"2","kafka-consumer-q1",[15,2390,2391],{},"Когда безопаснее commit-ить offset при at-least-once consumer?",[2393,2394,2395],"template",{"v-slot:options":38},[2220,2396,2397,2400,2403,2409],{},[1935,2398,2399],{},"До обработки сообщения, чтобы не получить duplicate",[1935,2401,2402],{},"После успешной обработки или после успешной записи в DLQ",[1935,2404,2405,2406,2408],{},"Сразу после ",[19,2407,169],{},", независимо от результата",[1935,2410,2411],{},"Никогда: offsets в Kafka не нужны",[2393,2413,2414],{"v-slot:explanation":38},[15,2415,2416],{},"Ранний commit может потерять сообщение. Поздний commit может дать duplicate, поэтому обработчик должен быть идемпотентным.",[2418,2419,2423,2426,2561],"predict",{"answer":2420,"id":2421,"xp":2422},"lost\\nduplicate","kafka-consumer-p1","15",[15,2424,2425],{},"Что выведет программа?",[2393,2427,2428],{"v-slot:code":38},[32,2429,2431],{"className":177,"code":2430,"language":179,"meta":38,"style":38},"package main\n\nimport \"fmt\"\n\nfunc failureMode(commitBeforeHandle bool) string {\n    if commitBeforeHandle {\n        return \"lost\"\n    }\n    return \"duplicate\"\n}\n\nfunc main() {\n    fmt.Println(failureMode(true))\n    fmt.Println(failureMode(false))\n}\n",[19,2432,2433,2439,2443,2454,2458,2479,2486,2493,2497,2504,2508,2512,2520,2540,2557],{"__ignoreMap":38},[184,2434,2435,2437],{"class":186,"line":187},[184,2436,191],{"class":190},[184,2438,195],{"class":194},[184,2440,2441],{"class":186,"line":198},[184,2442,202],{"emptyLinePlaceholder":201},[184,2444,2445,2447,2450,2452],{"class":186,"line":205},[184,2446,208],{"class":190},[184,2448,2449],{"class":218}," \"",[184,2451,253],{"class":194},[184,2453,225],{"class":218},[184,2455,2456],{"class":186,"line":215},[184,2457,202],{"emptyLinePlaceholder":201},[184,2459,2460,2462,2465,2467,2470,2473,2475,2477],{"class":186,"line":228},[184,2461,496],{"class":190},[184,2463,2464],{"class":194}," failureMode",[184,2466,436],{"class":211},[184,2468,2469],{"class":439},"commitBeforeHandle",[184,2471,2472],{"class":190}," bool",[184,2474,459],{"class":211},[184,2476,355],{"class":190},[184,2478,346],{"class":211},[184,2480,2481,2483],{"class":186,"line":238},[184,2482,904],{"class":190},[184,2484,2485],{"class":211}," commitBeforeHandle {\n",[184,2487,2488,2490],{"class":186,"line":248},[184,2489,562],{"class":190},[184,2491,2492],{"class":218}," \"lost\"\n",[184,2494,2495],{"class":186,"line":258},[184,2496,610],{"class":211},[184,2498,2499,2501],{"class":186,"line":268},[184,2500,1515],{"class":190},[184,2502,2503],{"class":218}," \"duplicate\"\n",[184,2505,2506],{"class":186,"line":278},[184,2507,409],{"class":211},[184,2509,2510],{"class":186,"line":288},[184,2511,202],{"emptyLinePlaceholder":201},[184,2513,2514,2516,2518],{"class":186,"line":298},[184,2515,496],{"class":190},[184,2517,628],{"class":194},[184,2519,631],{"class":211},[184,2521,2522,2525,2528,2530,2533,2535,2538],{"class":186,"line":308},[184,2523,2524],{"class":211},"    fmt.",[184,2526,2527],{"class":194},"Println",[184,2529,436],{"class":211},[184,2531,2532],{"class":194},"failureMode",[184,2534,436],{"class":211},[184,2536,2537],{"class":589},"true",[184,2539,661],{"class":211},[184,2541,2542,2544,2546,2548,2550,2552,2555],{"class":186,"line":313},[184,2543,2524],{"class":211},[184,2545,2527],{"class":194},[184,2547,436],{"class":211},[184,2549,2532],{"class":194},[184,2551,436],{"class":211},[184,2553,2554],{"class":589},"false",[184,2556,661],{"class":211},[184,2558,2559],{"class":186,"line":323},[184,2560,409],{"class":211},[2393,2562,2563],{"v-slot:hint":38},[15,2564,2565],{},"Commit до side effect создаёт риск потери. Commit после side effect создаёт риск повтора при падении до commit.",[2567,2568,2572,2579,2724],"code-task",{"expected":2569,"id":2570,"xp":2571},"commit\\nretry\\ncommit","kafka-consumer-ct1","20",[15,2573,2574,2575,2578],{},"Реализуй ",[19,2576,2577],{},"OffsetAction",": commit нужен после успешной обработки или после успешной записи в DLQ; иначе сообщение надо retry.",[2393,2580,2581],{"v-slot:template":38},[32,2582,2584],{"className":177,"code":2583,"language":179,"meta":38,"style":38},"package main\n\nimport \"fmt\"\n\nfunc OffsetAction(processed bool, dlqWritten bool) string {\n    return \"todo\"\n}\n\nfunc main() {\n    fmt.Println(OffsetAction(true, false))\n    fmt.Println(OffsetAction(false, false))\n    fmt.Println(OffsetAction(false, true))\n}\n",[19,2585,2586,2592,2596,2606,2610,2637,2644,2648,2652,2660,2680,2700,2720],{"__ignoreMap":38},[184,2587,2588,2590],{"class":186,"line":187},[184,2589,191],{"class":190},[184,2591,195],{"class":194},[184,2593,2594],{"class":186,"line":198},[184,2595,202],{"emptyLinePlaceholder":201},[184,2597,2598,2600,2602,2604],{"class":186,"line":205},[184,2599,208],{"class":190},[184,2601,2449],{"class":218},[184,2603,253],{"class":194},[184,2605,225],{"class":218},[184,2607,2608],{"class":186,"line":215},[184,2609,202],{"emptyLinePlaceholder":201},[184,2611,2612,2614,2617,2619,2622,2624,2626,2629,2631,2633,2635],{"class":186,"line":228},[184,2613,496],{"class":190},[184,2615,2616],{"class":194}," OffsetAction",[184,2618,436],{"class":211},[184,2620,2621],{"class":439},"processed",[184,2623,2472],{"class":190},[184,2625,451],{"class":211},[184,2627,2628],{"class":439},"dlqWritten",[184,2630,2472],{"class":190},[184,2632,459],{"class":211},[184,2634,355],{"class":190},[184,2636,346],{"class":211},[184,2638,2639,2641],{"class":186,"line":238},[184,2640,1515],{"class":190},[184,2642,2643],{"class":218}," \"todo\"\n",[184,2645,2646],{"class":186,"line":248},[184,2647,409],{"class":211},[184,2649,2650],{"class":186,"line":258},[184,2651,202],{"emptyLinePlaceholder":201},[184,2653,2654,2656,2658],{"class":186,"line":268},[184,2655,496],{"class":190},[184,2657,628],{"class":194},[184,2659,631],{"class":211},[184,2661,2662,2664,2666,2668,2670,2672,2674,2676,2678],{"class":186,"line":278},[184,2663,2524],{"class":211},[184,2665,2527],{"class":194},[184,2667,436],{"class":211},[184,2669,2577],{"class":194},[184,2671,436],{"class":211},[184,2673,2537],{"class":589},[184,2675,451],{"class":211},[184,2677,2554],{"class":589},[184,2679,661],{"class":211},[184,2681,2682,2684,2686,2688,2690,2692,2694,2696,2698],{"class":186,"line":288},[184,2683,2524],{"class":211},[184,2685,2527],{"class":194},[184,2687,436],{"class":211},[184,2689,2577],{"class":194},[184,2691,436],{"class":211},[184,2693,2554],{"class":589},[184,2695,451],{"class":211},[184,2697,2554],{"class":589},[184,2699,661],{"class":211},[184,2701,2702,2704,2706,2708,2710,2712,2714,2716,2718],{"class":186,"line":298},[184,2703,2524],{"class":211},[184,2705,2527],{"class":194},[184,2707,436],{"class":211},[184,2709,2577],{"class":194},[184,2711,436],{"class":211},[184,2713,2554],{"class":589},[184,2715,451],{"class":211},[184,2717,2537],{"class":589},[184,2719,661],{"class":211},[184,2721,2722],{"class":186,"line":308},[184,2723,409],{"class":211},[2393,2725,2726],{"v-slot:hints":38},[2220,2727,2728,2731,2734],{},[1935,2729,2730],{},"Успешная обработка закрывает offset.",[1935,2732,2733],{},"Успешная DLQ-запись тоже закрывает исходный poison message.",[1935,2735,2736],{},"Если ни обработка, ни DLQ не прошли, commit делать нельзя.",[2738,2739,2740],"style",{},"html pre.shiki code .snl16, html code.shiki .snl16{--shiki-default:#F97583}html pre.shiki code .svObZ, html code.shiki .svObZ{--shiki-default:#B392F0}html pre.shiki code .s95oV, html code.shiki .s95oV{--shiki-default:#E1E4E8}html pre.shiki code .sU2Wk, html code.shiki .sU2Wk{--shiki-default:#9ECBFF}html pre.shiki code .s9osk, html code.shiki .s9osk{--shiki-default:#FFAB70}html pre.shiki code .sDLfK, html code.shiki .sDLfK{--shiki-default:#79B8FF}html pre.shiki code .sAwPA, html code.shiki .sAwPA{--shiki-default:#6A737D}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}",{"title":38,"searchDepth":198,"depth":198,"links":2742},[2743,2744,2745,2746,2747,2748,2749,2750,2751,2752,2753],{"id":50,"depth":198,"text":51},{"id":80,"depth":198,"text":81},{"id":155,"depth":198,"text":156},{"id":1540,"depth":198,"text":1541},{"id":1635,"depth":198,"text":1636},{"id":1926,"depth":198,"text":1927},{"id":2211,"depth":198,"text":2212},{"id":2280,"depth":198,"text":2281},{"id":2309,"depth":198,"text":2310},{"id":2353,"depth":198,"text":2354},{"id":2382,"depth":198,"text":2383},1781022066929]