diff --git a/Makefile b/Makefile index a243a86617b75..d80362a72ccfe 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,7 @@ tag := $(shell git describe --exact-match --tags 2>/dev/null) branch := $(shell git rev-parse --abbrev-ref HEAD) commit := $(shell git rev-parse --short=8 HEAD) +upstream_version ?= 1.32.1 ifdef NIGHTLY version := $(next_version) @@ -32,6 +33,13 @@ else ifneq ($(findstring -rc,$(tag)),) deb_version := $(version)~$(rc)-1 deb_iteration := 0 tar_version := $(version)~$(rc) +else ifneq ($(filter bklite-%,$(tag)),) + version := $(patsubst bklite-%,%,$(tag)) + rpm_version := $(version)-1 + rpm_iteration := 1 + deb_version := $(version)-1 + deb_iteration := 1 + tar_version := $(version) else version := $(tag:v%=%) rpm_version := $(version)-1 @@ -46,7 +54,7 @@ GOOS ?= $(shell go env GOOS) GOARCH ?= $(shell go env GOARCH) HOSTGO := env -u GOOS -u GOARCH -u GOARM -- go INTERNAL_PKG=github.com/influxdata/telegraf/internal -LDFLAGS := $(LDFLAGS) -X $(INTERNAL_PKG).Commit=$(commit) -X $(INTERNAL_PKG).Branch=$(branch) +LDFLAGS := $(LDFLAGS) -X $(INTERNAL_PKG).Commit=$(commit) -X $(INTERNAL_PKG).Branch=$(branch) -X $(INTERNAL_PKG).Upstream=$(upstream_version) ifneq ($(tag),) LDFLAGS += -X $(INTERNAL_PKG).Version=$(version) else diff --git a/cmd/telegraf/main.go b/cmd/telegraf/main.go index 6821a6020962f..26f2ab73ca19c 100644 --- a/cmd/telegraf/main.go +++ b/cmd/telegraf/main.go @@ -205,6 +205,9 @@ func runApp(args []string, outputBuffer io.Writer, pprof Server, c TelegrafConfi return fmt.Errorf("%w and %w", err, err2) } return nil + case cCtx.Bool("upstream-version"): + fmt.Fprintf(outputBuffer, "%s\n", internal.FormatUpstreamVersion()) + return nil // DEPRECATED case cCtx.Bool("version"): fmt.Fprintf(outputBuffer, "%s\n", internal.FormatFullVersion()) @@ -356,6 +359,10 @@ func runApp(args []string, outputBuffer io.Writer, pprof Server, c TelegrafConfi Name: "version", Usage: "DEPRECATED: display the version and exit", }, + &cli.BoolFlag{ + Name: "upstream-version", + Usage: "print the upstream Telegraf version compiled into this binary and exit", + }, // Already covered with the subcommand `./telegraf config` &cli.BoolFlag{ Name: "sample-config", diff --git a/cmd/telegraf/main_test.go b/cmd/telegraf/main_test.go index 04c99fcf0cea2..30ea589d17dae 100644 --- a/cmd/telegraf/main_test.go +++ b/cmd/telegraf/main_test.go @@ -435,30 +435,21 @@ func TestCommandVersion(t *testing.T) { ExpectedOutput string }{ { - Version: "v2.0.0", - ExpectedOutput: "Telegraf v2.0.0\n", + Version: "0.5.0", + Branch: "bklite", + Commit: "abcdef", + ExpectedOutput: "0.5.0\n", }, { - ExpectedOutput: "Telegraf unknown\n", - }, - { - Version: "v2.0.0", - Branch: "master", - ExpectedOutput: "Telegraf v2.0.0 (git: master@unknown)\n", - }, - { - Version: "v2.0.0", - Branch: "master", - Commit: "123", - ExpectedOutput: "Telegraf v2.0.0 (git: master@123)\n", - }, - { - Version: "v2.0.0", - Commit: "123", - ExpectedOutput: "Telegraf v2.0.0 (git: unknown@123)\n", + ExpectedOutput: "unknown\n", }, } + origVersion, origBranch, origCommit := internal.Version, internal.Branch, internal.Commit + t.Cleanup(func() { + internal.Version, internal.Branch, internal.Commit = origVersion, origBranch, origCommit + }) + for _, test := range tests { buf := new(bytes.Buffer) args := os.Args[0:1] @@ -481,30 +472,21 @@ func TestFlagVersion(t *testing.T) { ExpectedOutput string }{ { - Version: "v2.0.0", - ExpectedOutput: "Telegraf v2.0.0\n", - }, - { - ExpectedOutput: "Telegraf unknown\n", - }, - { - Version: "v2.0.0", - Branch: "master", - ExpectedOutput: "Telegraf v2.0.0 (git: master@unknown)\n", - }, - { - Version: "v2.0.0", - Branch: "master", - Commit: "123", - ExpectedOutput: "Telegraf v2.0.0 (git: master@123)\n", + Version: "0.5.0", + Branch: "bklite", + Commit: "abcdef", + ExpectedOutput: "0.5.0\n", }, { - Version: "v2.0.0", - Commit: "123", - ExpectedOutput: "Telegraf v2.0.0 (git: unknown@123)\n", + ExpectedOutput: "unknown\n", }, } + origVersion, origBranch, origCommit := internal.Version, internal.Branch, internal.Commit + t.Cleanup(func() { + internal.Version, internal.Branch, internal.Commit = origVersion, origBranch, origCommit + }) + for _, test := range tests { buf := new(bytes.Buffer) args := os.Args[0:1] @@ -518,6 +500,19 @@ func TestFlagVersion(t *testing.T) { } } +func TestFlagUpstreamVersion(t *testing.T) { + orig := internal.Upstream + t.Cleanup(func() { internal.Upstream = orig }) + internal.Upstream = "1.32.1" + internal.Version = "0.5.0" + + buf := new(bytes.Buffer) + args := append(os.Args[0:1], "--upstream-version") + err := runApp(args, buf, NewMockServer(), NewMockConfig(buf), NewMockTelegraf()) + require.NoError(t, err) + require.Equal(t, "1.32.1\n", buf.String()) +} + func TestGlobablBoolFlags(t *testing.T) { commands := []string{ "--debug", diff --git a/internal/internal.go b/internal/internal.go index d82ce04d66cff..8117edc07c504 100644 --- a/internal/internal.go +++ b/internal/internal.go @@ -35,9 +35,10 @@ var ( // Set via LDFLAGS -X var ( - Version = "unknown" - Branch = "" - Commit = "" + Version = "unknown" + Upstream = "1.32.1" + Branch = "" + Commit = "" ) type ReadWaitCloser struct { @@ -45,27 +46,23 @@ type ReadWaitCloser struct { wg sync.WaitGroup } +// FormatFullVersion is the BKLite release version printed by --version. +// Packaging stores this stdout as the node-management version, so it is only +// the x.y.z release number. func FormatFullVersion() string { - var parts = []string{"Telegraf"} - - if Version != "" { - parts = append(parts, Version) - } else { - parts = append(parts, "unknown") + if Version == "" { + return "unknown" } + return Version +} - if Branch != "" || Commit != "" { - if Branch == "" { - Branch = "unknown" - } - if Commit == "" { - Commit = "unknown" - } - git := fmt.Sprintf("(git: %s@%s)", Branch, Commit) - parts = append(parts, git) +// FormatUpstreamVersion is the upstream Telegraf version compiled into the +// binary. It is not part of --version. +func FormatUpstreamVersion() string { + if Upstream == "" { + return "unknown" } - - return strings.Join(parts, " ") + return Upstream } // ProductToken returns a tag for Telegraf that can be used in user agents. diff --git a/plugins/inputs/all/bklite_rocketmq.go b/plugins/inputs/all/bklite_rocketmq.go new file mode 100644 index 0000000000000..12b3faee93ba6 --- /dev/null +++ b/plugins/inputs/all/bklite_rocketmq.go @@ -0,0 +1,5 @@ +//go:build !custom || inputs || inputs.bklite_rocketmq + +package all + +import _ "github.com/influxdata/telegraf/plugins/inputs/bklite_rocketmq" // register plugin diff --git a/plugins/inputs/bklite_rocketmq/README.md b/plugins/inputs/bklite_rocketmq/README.md new file mode 100644 index 0000000000000..5f7b78bbe2afc --- /dev/null +++ b/plugins/inputs/bklite_rocketmq/README.md @@ -0,0 +1,46 @@ +# BKLite RocketMQ Input Plugin + +`bklite_rocketmq` collects RocketMQ broker, producer, consumer, and consumer-group metrics through the NameServer admin protocol. + +## Versions + +The target is **RocketMQ 4.9.x** (4.9.4 through 4.9.8). That is the current 4.x line still running in production. The admin calls are the same remoting protocol used since 4.4, when ACL was added, so 4.4–4.8 clusters that speak this protocol also match. + +RocketMQ **5.x** (current release 5.5.1) is a separate line. Since 5.1 it exposes its own Prometheus `/metrics`. This plugin does not collect that endpoint. A later BK-Lite pull path can select it; the measurement names here stay `bklite_rocketmq_*` so the two paths can share one metric contract. + +`rocketmq-client-go` v2.1.2 can list topics from NameServer. It does not implement consume-stats, broker-runtime, or cluster-info queries. Those numeric fields are emitted when a stats source provides them, and are left absent otherwise. This plugin does not invent values and does not shell out to `mqadmin`. + +## Configuration + +```toml @sample.conf +# RocketMQ 4.9.x admin collection. Nameserver protocol matches 4.4 through 4.9. +# RocketMQ 5.x native /metrics is not collected by this plugin. +[[inputs.bklite_rocketmq]] + ## NameServer addresses, host:port. + namesrv_addrs = ["127.0.0.1:9876"] + + ## Optional ACL. Leave empty when the cluster has ACL disabled. + # access_key = "" + # secret_key = "" + + ## Admin call timeout. + # response_timeout = "10s" + + ## Glob filters. Empty include means all. Used to bound topic and group cardinality. + # topic_include = [] + # topic_exclude = [] + # group_include = [] + # group_exclude = [] +``` + +## Metrics + +| Measurement | Fields | Tags | +|---|---|---| +| `bklite_rocketmq_up` | `up` | `namesrv` | +| `bklite_rocketmq_broker` | `tps`, `qps`, `commitlog_disk_ratio` | `cluster`, `broker`, `broker_ip` | +| `bklite_rocketmq_producer` | `tps`, `message_size`, `offset` | `cluster`, `broker`, `topic` | +| `bklite_rocketmq_consumer` | `tps`, `message_size`, `offset` | `cluster`, `broker`, `group`, `topic` | +| `bklite_rocketmq_group` | `diff`, `ready`, `inflight`, `latency_ms`, `retry_diff`, `dlq` | `cluster`, `group`, `topic` | + +`ready` is messages waiting to be consumed. `inflight` is messages pulled but not acked. When 4.x admin returns only one backlog, it is stored in `diff` and `ready` / `inflight` are omitted. A failed collection sets `up` to 0 and does not fail the Telegraf agent. diff --git a/plugins/inputs/bklite_rocketmq/bklite_rocketmq.go b/plugins/inputs/bklite_rocketmq/bklite_rocketmq.go new file mode 100644 index 0000000000000..1bc46f5d6ec28 --- /dev/null +++ b/plugins/inputs/bklite_rocketmq/bklite_rocketmq.go @@ -0,0 +1,117 @@ +//go:generate ../../../tools/readme_config_includer/generator +package bklite_rocketmq + +import ( + "context" + _ "embed" + "errors" + "strings" + "time" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/config" + "github.com/influxdata/telegraf/filter" + "github.com/influxdata/telegraf/plugins/inputs" +) + +//go:embed sample.conf +var sampleConfig string + +const defaultResponseTimeout = 10 * time.Second + +// RocketMQ collects RocketMQ 4.9.x (admin protocol 4.4–4.9) via NameServer. +// RocketMQ 5.x native Prometheus metrics are a separate pull path. +type RocketMQ struct { + NamesrvAddrs []string `toml:"namesrv_addrs"` + BrokerAddr string `toml:"broker_addr"` + AccessKey string `toml:"access_key"` + SecretKey string `toml:"secret_key"` + ResponseTimeout config.Duration `toml:"response_timeout"` + TopicInclude []string `toml:"topic_include"` + TopicExclude []string `toml:"topic_exclude"` + GroupInclude []string `toml:"group_include"` + GroupExclude []string `toml:"group_exclude"` + Log telegraf.Logger `toml:"-"` + + topicFilter filter.Filter + groupFilter filter.Filter + source statsSource +} + +type statsSource interface { + Collect(ctx context.Context) (snapshot, error) +} + +func (*RocketMQ) SampleConfig() string { + return sampleConfig +} + +func (r *RocketMQ) Init() error { + if len(r.NamesrvAddrs) == 0 { + return errors.New("namesrv_addrs is required") + } + if r.ResponseTimeout <= 0 { + r.ResponseTimeout = config.Duration(defaultResponseTimeout) + } + var err error + r.topicFilter, err = filter.NewIncludeExcludeFilter(r.TopicInclude, r.TopicExclude) + if err != nil { + return err + } + r.groupFilter, err = filter.NewIncludeExcludeFilter(r.GroupInclude, r.GroupExclude) + if err != nil { + return err + } + if r.source == nil { + r.source, err = newMQAdmin(r.NamesrvAddrs, r.BrokerAddr, r.AccessKey, r.SecretKey) + if err != nil { + return err + } + } + return nil +} + +func (r *RocketMQ) Gather(acc telegraf.Accumulator) error { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(r.ResponseTimeout)) + defer cancel() + + snap, err := r.source.Collect(ctx) + if err != nil { + if r.Log != nil { + r.Log.Errorf("rocketmq collect failed: %v", err) + } + acc.AddError(err) + if snap.Namesrv == "" { + acc.AddGauge("bklite_rocketmq_up", map[string]interface{}{"up": int64(0)}, map[string]string{ + "namesrv": r.namesrvTag(), + }) + return nil + } + } + r.emit(acc, snap) + return nil +} + +func (r *RocketMQ) namesrvTag() string { + return strings.Join(r.NamesrvAddrs, ",") +} + +func (r *RocketMQ) allowTopic(name string) bool { + if r.topicFilter == nil { + return true + } + return r.topicFilter.Match(name) +} + +func (r *RocketMQ) allowGroup(name string) bool { + if r.groupFilter == nil { + return true + } + return r.groupFilter.Match(name) +} + +func init() { + inputs.Add("bklite_rocketmq", func() telegraf.Input { + return &RocketMQ{} + }) +} diff --git a/plugins/inputs/bklite_rocketmq/bklite_rocketmq_test.go b/plugins/inputs/bklite_rocketmq/bklite_rocketmq_test.go new file mode 100644 index 0000000000000..89ed2ce0b8990 --- /dev/null +++ b/plugins/inputs/bklite_rocketmq/bklite_rocketmq_test.go @@ -0,0 +1,210 @@ +package bklite_rocketmq + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/influxdata/telegraf/config" + "github.com/influxdata/telegraf/testutil" + "github.com/stretchr/testify/require" +) + +func f64(v float64) *float64 { return &v } + +func TestGatherEmitsOperationalMetricsAndSkipsMissingFields(t *testing.T) { + plugin := &RocketMQ{ + NamesrvAddrs: []string{"10.0.0.1:9876"}, + source: fakeSource{snap: snapshot{ + Namesrv: "10.0.0.1:9876", + Brokers: []brokerSample{{ + Cluster: "c1", Broker: "broker-a", BrokerIP: "10.0.0.2", + TPS: f64(12), QPS: f64(30), CommitLogDiskRatio: f64(0.4), + }}, + Producers: []producerSample{{ + Cluster: "c1", Broker: "broker-a", Topic: "orders", + TPS: f64(8), MessageSize: f64(128), Offset: f64(1000), + }}, + Consumers: []consumerSample{{ + Cluster: "c1", Broker: "broker-a", Group: "billing", Topic: "orders", + TPS: f64(7), MessageSize: f64(128), Offset: f64(900), + }}, + Groups: []groupSample{{ + Cluster: "c1", Group: "billing", Topic: "orders", + Diff: f64(100), LatencyMs: f64(40), RetryDiff: f64(3), DLQ: f64(1), + }}, + }}, + } + require.NoError(t, plugin.Init()) + + acc := &testutil.Accumulator{} + require.NoError(t, plugin.Gather(acc)) + require.Empty(t, acc.Errors) + + acc.AssertContainsTaggedFields(t, "bklite_rocketmq_up", map[string]interface{}{"up": int64(1)}, map[string]string{"namesrv": "10.0.0.1:9876"}) + acc.AssertContainsTaggedFields(t, "bklite_rocketmq_broker", map[string]interface{}{ + "tps": float64(12), "qps": float64(30), "commitlog_disk_ratio": float64(0.4), + }, map[string]string{"cluster": "c1", "broker": "broker-a", "broker_ip": "10.0.0.2"}) + acc.AssertContainsTaggedFields(t, "bklite_rocketmq_group", map[string]interface{}{ + "diff": float64(100), "latency_ms": float64(40), "retry_diff": float64(3), "dlq": float64(1), + }, map[string]string{"cluster": "c1", "group": "billing", "topic": "orders"}) + + group, ok := acc.Get("bklite_rocketmq_group") + require.True(t, ok) + require.NotContains(t, group.Fields, "ready") + require.NotContains(t, group.Fields, "inflight") +} + +func TestGatherFiltersTopicAndGroup(t *testing.T) { + plugin := &RocketMQ{ + NamesrvAddrs: []string{"10.0.0.1:9876"}, + TopicInclude: []string{"orders"}, + GroupExclude: []string{"billing"}, + source: fakeSource{snap: snapshot{ + Producers: []producerSample{ + {Cluster: "c1", Broker: "b", Topic: "orders", TPS: f64(1)}, + {Cluster: "c1", Broker: "b", Topic: "audit", TPS: f64(1)}, + }, + Groups: []groupSample{ + {Cluster: "c1", Group: "billing", Topic: "orders", Diff: f64(1)}, + {Cluster: "c1", Group: "search", Topic: "orders", Diff: f64(2)}, + {Cluster: "c1", Group: "search", Topic: "audit", Diff: f64(3)}, + }, + }}, + } + require.NoError(t, plugin.Init()) + + acc := &testutil.Accumulator{} + require.NoError(t, plugin.Gather(acc)) + + require.Len(t, acc.Metrics, 3) + acc.AssertContainsTaggedFields(t, "bklite_rocketmq_producer", map[string]interface{}{"tps": float64(1)}, map[string]string{ + "cluster": "c1", "broker": "b", "topic": "orders", + }) + acc.AssertContainsTaggedFields(t, "bklite_rocketmq_group", map[string]interface{}{"diff": float64(2)}, map[string]string{ + "cluster": "c1", "group": "search", "topic": "orders", + }) +} + +func TestGatherFailureKeepsTelegrafUp(t *testing.T) { + plugin := &RocketMQ{ + NamesrvAddrs: []string{"10.0.0.1:9876"}, + ResponseTimeout: config.Duration(time.Second), + source: fakeSource{err: errors.New("nameserver down")}, + } + require.NoError(t, plugin.Init()) + + acc := &testutil.Accumulator{} + require.NoError(t, plugin.Gather(acc)) + require.Len(t, acc.Errors, 1) + acc.AssertContainsTaggedFields(t, "bklite_rocketmq_up", map[string]interface{}{"up": int64(0)}, map[string]string{"namesrv": "10.0.0.1:9876"}) +} + +func TestParseRuntimeAndTopicOffset(t *testing.T) { + tps := firstTPS("1.5 0.4 0.1") + require.NotNil(t, tps) + require.Equal(t, 1.5, *tps) + disk := parseFloat("0.84") + require.NotNil(t, disk) + require.Equal(t, 0.84, *disk) + + offset, ok := sumMaxOffset([]byte(`{"offsetTable":{"q0":{"maxOffset":4},"q1":{"maxOffset":6}}}`)) + require.True(t, ok) + require.Equal(t, float64(10), offset) + + info := clusterInfo{BrokerAddrTable: map[string]struct { + Cluster string `json:"cluster"` + BrokerName string `json:"brokerName"` + BrokerAddrs map[string]string `json:"brokerAddrs"` + }{ + "broker-a": {Cluster: "DefaultCluster", BrokerName: "broker-a", BrokerAddrs: map[string]string{"0": "10.0.0.2:10911"}}, + }} + masters := info.masters() + require.Len(t, masters, 1) + require.Equal(t, "10.0.0.2:10911", masters[0].dial) +} + +func TestParse49ConsumeStatsObjectKeys(t *testing.T) { + body := []byte(`{"consumeTps":1.2,"offsetTable":{{"brokerName":"broker-a","queueId":0,"topic":"weopsx_monitor_demo"}:{"brokerOffset":123,"consumerOffset":120,"lastTimestamp":1710000000000},{"topic":"weopsx_monitor_demo","queueId":1,"brokerName":"broker-a"}:{"lastTimestamp":1710000000001,"consumerOffset":10,"brokerOffset":15}}}`) + queues, groupTPS := parseConsumeQueues(body) + require.Len(t, queues, 2) + require.NotNil(t, groupTPS) + require.Equal(t, 1.2, *groupTPS) + + topics := consumeTopics(queues) + require.Equal(t, []string{"weopsx_monitor_demo"}, topics) + groups, consumers := buildConsumeSamples("DefaultCluster", "broker-a", "weopsx_monitor_cg", queues, map[string]*float64{ + topics[0]: groupTPS, + }) + require.Len(t, groups, 1) + require.Equal(t, "weopsx_monitor_demo", groups[0].Topic) + require.Equal(t, float64(8), *groups[0].Diff) + require.Nil(t, groups[0].Ready) + require.Nil(t, groups[0].Inflight) + require.Len(t, consumers, 1) + require.Equal(t, float64(130), *consumers[0].Offset) + require.Equal(t, 1.2, *consumers[0].TPS) + require.Nil(t, consumers[0].MessageSize) + + plugin := &RocketMQ{NamesrvAddrs: []string{"10.0.0.1:9876"}} + require.NoError(t, plugin.Init()) + acc := &testutil.Accumulator{} + plugin.emit(acc, snapshot{ + Namesrv: "10.0.0.1:9876", + Groups: groups, + Consumers: consumers, + }) + acc.AssertContainsTaggedFields(t, "bklite_rocketmq_group", map[string]interface{}{"diff": float64(8)}, map[string]string{ + "cluster": "DefaultCluster", "group": "weopsx_monitor_cg", "topic": "weopsx_monitor_demo", + }) + acc.AssertContainsTaggedFields(t, "bklite_rocketmq_consumer", map[string]interface{}{ + "offset": float64(130), "tps": 1.2, + }, map[string]string{ + "cluster": "DefaultCluster", "broker": "broker-a", "group": "weopsx_monitor_cg", "topic": "weopsx_monitor_demo", + }) + got, ok := acc.Get("bklite_rocketmq_group") + require.True(t, ok) + require.NotContains(t, got.Fields, "ready") + require.NotContains(t, got.Fields, "inflight") +} + +func TestGroupTPSNotCopiedAcrossTopics(t *testing.T) { + body := []byte(`{"consumeTps":9.9,"offsetTable":{{"brokerName":"broker-a","queueId":0,"topic":"orders"}:{"brokerOffset":5,"consumerOffset":1},{"brokerName":"broker-a","queueId":0,"topic":"audit"}:{"consumerOffset":4,"brokerOffset":4}}}`) + queues, groupTPS := parseConsumeQueues(body) + require.NotNil(t, groupTPS) + require.Len(t, consumeTopics(queues), 2) + _, consumers := buildConsumeSamples("DefaultCluster", "broker-a", "weopsx_monitor_cg", queues, nil) + require.Len(t, consumers, 2) + for _, consumer := range consumers { + require.Nil(t, consumer.TPS) + require.Nil(t, consumer.MessageSize) + } +} + +func TestDLQFallbackCarriesTopic(t *testing.T) { + sample := dlqGroupSample("DefaultCluster", "weopsx_monitor_cg_fail", 4) + require.Equal(t, "%DLQ%weopsx_monitor_cg_fail", sample.Topic) + require.Equal(t, "weopsx_monitor_cg_fail", sample.Group) + require.Equal(t, float64(4), *sample.DLQ) + require.Nil(t, sample.Ready) + require.Nil(t, sample.Inflight) + require.Nil(t, sample.Diff) +} + +func TestInitRequiresNamesrv(t *testing.T) { + plugin := &RocketMQ{} + require.Error(t, plugin.Init()) +} + +type fakeSource struct { + snap snapshot + err error +} + +func (f fakeSource) Collect(context.Context) (snapshot, error) { + if f.err != nil { + return snapshot{}, f.err + } + return f.snap, nil +} diff --git a/plugins/inputs/bklite_rocketmq/client.go b/plugins/inputs/bklite_rocketmq/client.go new file mode 100644 index 0000000000000..5fb5da73cdbac --- /dev/null +++ b/plugins/inputs/bklite_rocketmq/client.go @@ -0,0 +1,486 @@ +package bklite_rocketmq + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + "strconv" + "strings" +) + +// mqAdmin speaks the RocketMQ 4.x admin remoting protocol. +// brokerAddr overrides the address registered in NameServer. On a laptop that +// address is usually a cluster DNS name, so local collection points it at a +// port-forwarded broker. +type mqAdmin struct { + namesrv []string + namesrvTag string + brokerAddr string +} + +func newMQAdmin(addrs []string, brokerAddr, _, _ string) (*mqAdmin, error) { + return &mqAdmin{ + namesrv: addrs, + namesrvTag: joinAddrs(addrs), + brokerAddr: brokerAddr, + }, nil +} + +func (m *mqAdmin) Collect(ctx context.Context) (snapshot, error) { + snap := snapshot{Namesrv: m.namesrvTag} + body, namesrv, err := m.invokeNamesrv(ctx, codeGetBrokerClusterInfo, nil) + if err != nil { + return snapshot{}, fmt.Errorf("broker cluster info: %w", err) + } + var cluster clusterInfo + if err = json.Unmarshal(relaxRocketMQJSON(body), &cluster); err != nil { + return snapshot{}, fmt.Errorf("broker cluster info decode: %w", err) + } + brokers := cluster.masters() + if m.brokerAddr != "" { + for i := range brokers { + brokers[i].dial = m.brokerAddr + } + } + + topicsBody, _, err := m.invokeNamesrv(ctx, codeGetAllTopicList, nil) + if err != nil { + return snap, fmt.Errorf("topic list: %w", err) + } + var topics topicList + if err = json.Unmarshal(topicsBody, &topics); err != nil { + return snap, fmt.Errorf("topic list decode: %w", err) + } + _ = namesrv + + var firstErr error + for _, broker := range brokers { + runtimeBody, err := invoke(ctx, broker.dial, codeGetBrokerRuntimeInfo, nil) + if err != nil { + firstErr = fmt.Errorf("broker runtime %s: %w", broker.dial, err) + continue + } + var kv kvTable + if err = json.Unmarshal(runtimeBody, &kv); err != nil { + firstErr = err + continue + } + snap.Brokers = append(snap.Brokers, brokerSample{ + Cluster: broker.cluster, + Broker: broker.name, + BrokerIP: broker.registered, + TPS: firstTPS(kv.Table["putTps"]), + QPS: firstTPS(firstNonEmpty(kv.Table["getTotalTps"], kv.Table["getTransferredTps"], kv.Table["getFoundTps"])), + CommitLogDiskRatio: parseFloat(kv.Table["commitLogDiskRatio"]), + }) + snap.Producers = append(snap.Producers, m.topicOffsets(ctx, broker, topics.TopicList)...) + groups, consumers := m.groupStats(ctx, broker) + snap.Groups = append(snap.Groups, groups...) + snap.Consumers = append(snap.Consumers, consumers...) + } + if len(snap.Brokers) == 0 && firstErr != nil { + return snap, firstErr + } + return snap, nil +} + +func (m *mqAdmin) topicOffsets(ctx context.Context, broker brokerEndpoint, topics []string) []producerSample { + var out []producerSample + for _, topic := range topics { + if topic == "" || strings.HasPrefix(topic, "%") { + continue + } + body, err := invoke(ctx, broker.dial, codeGetTopicStatsInfo, map[string]string{"topic": topic}) + if err != nil { + continue + } + offset, ok := sumMaxOffset(body) + if !ok { + continue + } + out = append(out, producerSample{ + Cluster: broker.cluster, + Broker: broker.name, + Topic: topic, + Offset: &offset, + }) + } + return out +} + +func (m *mqAdmin) groupStats(ctx context.Context, broker brokerEndpoint) ([]groupSample, []consumerSample) { + body, err := invoke(ctx, broker.dial, codeGetAllSubscriptionGroup, nil) + if err != nil { + return nil, nil + } + var groups subscriptionGroups + if err = json.Unmarshal(body, &groups); err != nil { + return nil, nil + } + var groupOut []groupSample + var consumerOut []consumerSample + for name := range groups.SubscriptionGroupTable { + statsBody, err := invoke(ctx, broker.dial, codeGetConsumeStats, map[string]string{"consumerGroup": name}) + if err != nil { + continue + } + queues, groupTPS := parseConsumeQueues(statsBody) + perTopicTPS := m.perTopicConsumeTPS(ctx, broker, name, queues, groupTPS) + samples, consumers := buildConsumeSamples(broker.cluster, broker.name, name, queues, perTopicTPS) + groupOut = append(groupOut, samples...) + consumerOut = append(consumerOut, consumers...) + dlq, hasDLQ := m.dlqDepth(ctx, broker, name) + if !hasDLQ { + continue + } + if len(samples) == 0 { + groupOut = append(groupOut, dlqGroupSample(broker.cluster, name, dlq)) + continue + } + groupOut = append(groupOut, dlqGroupSample(broker.cluster, name, dlq)) + } + return groupOut, consumerOut +} + +func (m *mqAdmin) perTopicConsumeTPS(ctx context.Context, broker brokerEndpoint, group string, queues []consumeQueue, groupTPS *float64) map[string]*float64 { + topics := consumeTopics(queues) + if len(topics) == 1 { + if groupTPS == nil { + return nil + } + return map[string]*float64{topics[0]: groupTPS} + } + if len(topics) < 2 { + return nil + } + out := map[string]*float64{} + for _, topic := range topics { + body, err := invoke(ctx, broker.dial, codeGetConsumeStats, map[string]string{ + "consumerGroup": group, + "topic": topic, + }) + if err != nil { + continue + } + _, tps := parseConsumeQueues(body) + if tps != nil { + out[topic] = tps + } + } + if len(out) == 0 { + return nil + } + return out +} + +func (m *mqAdmin) dlqDepth(ctx context.Context, broker brokerEndpoint, group string) (float64, bool) { + body, err := invoke(ctx, broker.dial, codeGetTopicStatsInfo, map[string]string{"topic": "%DLQ%" + group}) + if err != nil { + return 0, false + } + return sumMaxOffset(body) +} + +func (m *mqAdmin) invokeNamesrv(ctx context.Context, code int, ext map[string]string) ([]byte, string, error) { + var last error + for _, addr := range m.namesrv { + body, err := invoke(ctx, addr, code, ext) + if err == nil { + return body, addr, nil + } + last = err + } + if last == nil { + last = fmt.Errorf("no nameserver address") + } + return nil, "", last +} + +type brokerEndpoint struct { + cluster string + name string + registered string + dial string +} + +type clusterInfo struct { + BrokerAddrTable map[string]struct { + Cluster string `json:"cluster"` + BrokerName string `json:"brokerName"` + BrokerAddrs map[string]string `json:"brokerAddrs"` + } `json:"brokerAddrTable"` +} + +func (c clusterInfo) masters() []brokerEndpoint { + var out []brokerEndpoint + for _, broker := range c.BrokerAddrTable { + addr := broker.BrokerAddrs["0"] + if addr == "" { + continue + } + name := broker.BrokerName + if name == "" { + name = addr + } + out = append(out, brokerEndpoint{ + cluster: broker.Cluster, + name: name, + registered: addr, + dial: addr, + }) + } + return out +} + +type kvTable struct { + Table map[string]string `json:"table"` +} + +type topicList struct { + TopicList []string `json:"topicList"` +} + +type subscriptionGroups struct { + SubscriptionGroupTable map[string]json.RawMessage `json:"subscriptionGroupTable"` +} + +var maxOffsetPattern = regexp.MustCompile(`"maxOffset"\s*:\s*(-?\d+)`) + +func sumMaxOffset(body []byte) (float64, bool) { + matches := maxOffsetPattern.FindAllSubmatch(body, -1) + if len(matches) == 0 { + return 0, false + } + var sum float64 + for _, match := range matches { + value, err := strconv.ParseFloat(string(match[1]), 64) + if err != nil { + continue + } + sum += value + } + return sum, true +} + +type consumeQueue struct { + topic string + brokerOffset *float64 + consumerOffset *float64 +} + +func parseConsumeQueues(body []byte) ([]consumeQueue, *float64) { + tps, _ := namedFloat(body, "consumeTps") + text := string(body) + var queues []consumeQueue + for i := 0; i < len(text); { + rel := strings.Index(text[i:], `"brokerName"`) + if rel < 0 { + break + } + at := i + rel + start := strings.LastIndex(text[:at], "{") + if start < 0 { + i = at + len(`"brokerName"`) + continue + } + keyEnd := matchBrace(text, start) + if keyEnd < 0 { + i = at + len(`"brokerName"`) + continue + } + rest := strings.TrimLeft(text[keyEnd+1:], " \t\r\n") + if !strings.HasPrefix(rest, ":") { + i = keyEnd + 1 + continue + } + rest = strings.TrimLeft(rest[1:], " \t\r\n") + if !strings.HasPrefix(rest, "{") { + i = keyEnd + 1 + continue + } + valueEnd := matchBrace(rest, 0) + if valueEnd < 0 { + break + } + key := text[start : keyEnd+1] + value := rest[:valueEnd+1] + topic := namedString(key, "topic") + brokerOffset, hasBroker := namedFloat([]byte(value), "brokerOffset") + consumerOffset, hasConsumer := namedFloat([]byte(value), "consumerOffset") + if topic != "" && (hasBroker || hasConsumer) { + queues = append(queues, consumeQueue{ + topic: topic, + brokerOffset: brokerOffset, + consumerOffset: consumerOffset, + }) + } + i = (len(text) - len(rest)) + valueEnd + 1 + } + return queues, tps +} + +func buildConsumeSamples(cluster, broker, group string, queues []consumeQueue, topicTPS map[string]*float64) ([]groupSample, []consumerSample) { + type agg struct { + diff float64 + hasDiff bool + offset float64 + hasOff bool + } + byTopic := map[string]*agg{} + var topics []string + for _, queue := range queues { + if queue.topic == "" { + continue + } + item := byTopic[queue.topic] + if item == nil { + item = &agg{} + byTopic[queue.topic] = item + topics = append(topics, queue.topic) + } + if queue.brokerOffset != nil && queue.consumerOffset != nil { + gap := *queue.brokerOffset - *queue.consumerOffset + if gap < 0 { + gap = 0 + } + item.diff += gap + item.hasDiff = true + } + if queue.consumerOffset != nil { + item.offset += *queue.consumerOffset + item.hasOff = true + } + } + var groups []groupSample + var consumers []consumerSample + for _, topic := range topics { + item := byTopic[topic] + if item.hasDiff { + diff := item.diff + groups = append(groups, groupSample{ + Cluster: cluster, + Group: group, + Topic: topic, + Diff: &diff, + }) + } + if !item.hasOff && topicTPS[topic] == nil { + continue + } + sample := consumerSample{Cluster: cluster, Broker: broker, Group: group, Topic: topic} + if item.hasOff { + offset := item.offset + sample.Offset = &offset + } + if tps := topicTPS[topic]; tps != nil { + sample.TPS = tps + } + consumers = append(consumers, sample) + } + return groups, consumers +} + +func consumeTopics(queues []consumeQueue) []string { + seen := map[string]struct{}{} + var topics []string + for _, queue := range queues { + if queue.topic == "" { + continue + } + if _, ok := seen[queue.topic]; ok { + continue + } + seen[queue.topic] = struct{}{} + topics = append(topics, queue.topic) + } + return topics +} + +func dlqGroupSample(cluster, group string, dlq float64) groupSample { + return groupSample{ + Cluster: cluster, + Group: group, + Topic: "%DLQ%" + group, + DLQ: &dlq, + } +} + +func matchBrace(text string, start int) int { + if start < 0 || start >= len(text) || text[start] != '{' { + return -1 + } + depth := 0 + for i := start; i < len(text); i++ { + switch text[i] { + case '{': + depth++ + case '}': + depth-- + if depth == 0 { + return i + } + } + } + return -1 +} + +func namedString(text, name string) string { + match := regexp.MustCompile(`"` + regexp.QuoteMeta(name) + `"\s*:\s*"([^"]*)"`).FindStringSubmatch(text) + if match == nil { + return "" + } + return match[1] +} + +func namedFloat(body []byte, name string) (*float64, bool) { + match := regexp.MustCompile(`"` + regexp.QuoteMeta(name) + `"\s*:\s*(-?\d+(?:\.\d+)?)`).FindSubmatch(body) + if match == nil { + return nil, false + } + value, err := strconv.ParseFloat(string(match[1]), 64) + if err != nil { + return nil, false + } + return &value, true +} + +func firstTPS(raw string) *float64 { + raw = strings.TrimSpace(raw) + if raw == "" { + return nil + } + return parseFloat(strings.Fields(raw)[0]) +} + +func parseFloat(raw string) *float64 { + raw = strings.TrimSpace(raw) + if raw == "" { + return nil + } + value, err := strconv.ParseFloat(raw, 64) + if err != nil { + return nil + } + return &value +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return value + } + } + return "" +} + +func joinAddrs(addrs []string) string { + return strings.Join(addrs, ",") +} + +// RocketMQ serializes integer map keys without quotes, which encoding/json rejects. +var unquotedKey = regexp.MustCompile(`([{\[,])(\d+):`) + +func relaxRocketMQJSON(body []byte) []byte { + return unquotedKey.ReplaceAll(body, []byte(`$1"$2":`)) +} diff --git a/plugins/inputs/bklite_rocketmq/metrics.go b/plugins/inputs/bklite_rocketmq/metrics.go new file mode 100644 index 0000000000000..dd00a97ee0fd1 --- /dev/null +++ b/plugins/inputs/bklite_rocketmq/metrics.go @@ -0,0 +1,141 @@ +package bklite_rocketmq + +import "github.com/influxdata/telegraf" + +// snapshot is one admin collection. Nil numeric fields are omitted. +// RocketMQ 4.x admin often returns a single backlog in Diff and does not +// split Ready and Inflight; those stay nil instead of being invented. +type snapshot struct { + Namesrv string + Brokers []brokerSample + Producers []producerSample + Consumers []consumerSample + Groups []groupSample +} + +type brokerSample struct { + Cluster string + Broker string + BrokerIP string + TPS *float64 + QPS *float64 + CommitLogDiskRatio *float64 +} + +type producerSample struct { + Cluster string + Broker string + Topic string + TPS *float64 + MessageSize *float64 + Offset *float64 +} + +type consumerSample struct { + Cluster string + Broker string + Group string + Topic string + TPS *float64 + MessageSize *float64 + Offset *float64 +} + +type groupSample struct { + Cluster string + Group string + Topic string + Diff *float64 + Ready *float64 + Inflight *float64 + LatencyMs *float64 + RetryDiff *float64 + DLQ *float64 +} + +func (r *RocketMQ) emit(acc telegraf.Accumulator, snap snapshot) { + namesrv := snap.Namesrv + if namesrv == "" { + namesrv = r.namesrvTag() + } + acc.AddGauge("bklite_rocketmq_up", map[string]interface{}{"up": int64(1)}, map[string]string{"namesrv": namesrv}) + + for _, broker := range snap.Brokers { + fields := map[string]interface{}{} + addOptional(fields, "tps", broker.TPS) + addOptional(fields, "qps", broker.QPS) + addOptional(fields, "commitlog_disk_ratio", broker.CommitLogDiskRatio) + if len(fields) == 0 { + continue + } + acc.AddGauge("bklite_rocketmq_broker", fields, map[string]string{ + "cluster": broker.Cluster, + "broker": broker.Broker, + "broker_ip": broker.BrokerIP, + }) + } + + for _, producer := range snap.Producers { + if !r.allowTopic(producer.Topic) { + continue + } + fields := map[string]interface{}{} + addOptional(fields, "tps", producer.TPS) + addOptional(fields, "message_size", producer.MessageSize) + addOptional(fields, "offset", producer.Offset) + if len(fields) == 0 { + continue + } + acc.AddGauge("bklite_rocketmq_producer", fields, map[string]string{ + "cluster": producer.Cluster, + "broker": producer.Broker, + "topic": producer.Topic, + }) + } + + for _, consumer := range snap.Consumers { + if !r.allowTopic(consumer.Topic) || !r.allowGroup(consumer.Group) { + continue + } + fields := map[string]interface{}{} + addOptional(fields, "tps", consumer.TPS) + addOptional(fields, "message_size", consumer.MessageSize) + addOptional(fields, "offset", consumer.Offset) + if len(fields) == 0 { + continue + } + acc.AddGauge("bklite_rocketmq_consumer", fields, map[string]string{ + "cluster": consumer.Cluster, + "broker": consumer.Broker, + "group": consumer.Group, + "topic": consumer.Topic, + }) + } + + for _, group := range snap.Groups { + if !r.allowTopic(group.Topic) || !r.allowGroup(group.Group) { + continue + } + fields := map[string]interface{}{} + addOptional(fields, "diff", group.Diff) + addOptional(fields, "ready", group.Ready) + addOptional(fields, "inflight", group.Inflight) + addOptional(fields, "latency_ms", group.LatencyMs) + addOptional(fields, "retry_diff", group.RetryDiff) + addOptional(fields, "dlq", group.DLQ) + if len(fields) == 0 { + continue + } + acc.AddGauge("bklite_rocketmq_group", fields, map[string]string{ + "cluster": group.Cluster, + "group": group.Group, + "topic": group.Topic, + }) + } +} + +func addOptional(fields map[string]interface{}, name string, value *float64) { + if value != nil { + fields[name] = *value + } +} diff --git a/plugins/inputs/bklite_rocketmq/rpc.go b/plugins/inputs/bklite_rocketmq/rpc.go new file mode 100644 index 0000000000000..34c7dfdb2e33f --- /dev/null +++ b/plugins/inputs/bklite_rocketmq/rpc.go @@ -0,0 +1,103 @@ +package bklite_rocketmq + +import ( + "context" + "encoding/binary" + "encoding/json" + "fmt" + "io" + "net" + "sync/atomic" + "time" +) + +const ( + codeGetBrokerRuntimeInfo = 28 + codeGetBrokerClusterInfo = 106 + codeGetAllSubscriptionGroup = 201 + codeGetTopicStatsInfo = 202 + codeGetAllTopicList = 206 + codeGetConsumeStats = 208 + remotingSuccess = 0 +) + +var rpcOpaque int32 + +type remotingHeader struct { + Code int `json:"code"` + Language string `json:"language"` + Version int `json:"version"` + Opaque int32 `json:"opaque"` + Flag int `json:"flag"` + Remark string `json:"remark,omitempty"` + ExtFields map[string]string `json:"extFields"` +} + +func invoke(ctx context.Context, addr string, code int, ext map[string]string) ([]byte, error) { + var dialer net.Dialer + conn, err := dialer.DialContext(ctx, "tcp", addr) + if err != nil { + return nil, err + } + defer conn.Close() + deadline, ok := ctx.Deadline() + if !ok { + deadline = time.Now().Add(10 * time.Second) + } + if err = conn.SetDeadline(deadline); err != nil { + return nil, err + } + + header := remotingHeader{ + Code: code, + Language: "GO", + Version: 317, + Opaque: atomic.AddInt32(&rpcOpaque, 1), + ExtFields: ext, + } + if header.ExtFields == nil { + header.ExtFields = map[string]string{} + } + headerBytes, err := json.Marshal(header) + if err != nil { + return nil, err + } + frameLen := 4 + len(headerBytes) + buf := make([]byte, 4+frameLen) + binary.BigEndian.PutUint32(buf[0:4], uint32(frameLen)) + buf[4] = 0 + buf[5] = byte((len(headerBytes) >> 16) & 0xFF) + buf[6] = byte((len(headerBytes) >> 8) & 0xFF) + buf[7] = byte(len(headerBytes) & 0xFF) + copy(buf[8:], headerBytes) + if _, err = conn.Write(buf); err != nil { + return nil, err + } + + var total uint32 + if err = binary.Read(conn, binary.BigEndian, &total); err != nil { + return nil, err + } + if total < 4 || total > 32*1024*1024 { + return nil, fmt.Errorf("rocketmq frame length %d", total) + } + payload := make([]byte, total) + if _, err = io.ReadFull(conn, payload); err != nil { + return nil, err + } + headerLen := binary.BigEndian.Uint32(payload[:4]) & 0xFFFFFF + if int(4+headerLen) > len(payload) { + return nil, fmt.Errorf("rocketmq header length %d", headerLen) + } + var resp remotingHeader + if err = json.Unmarshal(payload[4:4+headerLen], &resp); err != nil { + return nil, err + } + if resp.Code != remotingSuccess { + if resp.Remark == "" { + return nil, fmt.Errorf("rocketmq code %d", resp.Code) + } + return nil, fmt.Errorf("rocketmq code %d: %s", resp.Code, resp.Remark) + } + return relaxRocketMQJSON(payload[4+headerLen:]), nil +} diff --git a/plugins/inputs/bklite_rocketmq/sample.conf b/plugins/inputs/bklite_rocketmq/sample.conf new file mode 100644 index 0000000000000..9b75b92366c47 --- /dev/null +++ b/plugins/inputs/bklite_rocketmq/sample.conf @@ -0,0 +1,22 @@ +# RocketMQ 4.9.x admin collection. Nameserver protocol matches 4.4 through 4.9. +# RocketMQ 5.x native /metrics is not collected by this plugin. +[[inputs.bklite_rocketmq]] + ## NameServer addresses, host:port. + namesrv_addrs = ["127.0.0.1:9876"] + + ## Broker address to dial. Set this when the address registered in NameServer + ## is not reachable from the collector, for example a Kubernetes DNS name. + # broker_addr = "127.0.0.1:10911" + + ## Optional ACL. Leave empty when the cluster has ACL disabled. + # access_key = "" + # secret_key = "" + + ## Admin call timeout. + # response_timeout = "10s" + + ## Glob filters. Empty include means all. Used to bound topic and group cardinality. + # topic_include = [] + # topic_exclude = [] + # group_include = [] + # group_exclude = []