diff --git a/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types.go b/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types.go index 73b607bfc..e55ec4dc7 100644 --- a/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types.go +++ b/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types.go @@ -41,6 +41,8 @@ type FluentBitConfigSpec struct { OutputSelector metav1.LabelSelector `json:"outputSelector,omitempty"` // Select parser plugins ParserSelector metav1.LabelSelector `json:"parserSelector,omitempty"` + // Select multiline parser plugins + MultilineParserSelector metav1.LabelSelector `json:"multilineparserSelector,omitempty"` // If namespace is defined, then the configmap and secret for fluent-bit is in this namespace. // If it is not defined, it is in the namespace of the fluentd-operator Namespace *string `json:"namespace,omitempty"` @@ -101,6 +103,8 @@ type Service struct { LogLevel string `json:"logLevel,omitempty"` // Optional 'parsers' config file (can be multiple) ParsersFile string `json:"parsersFile,omitempty"` + // backward compatible + ParsersFiles []string `json:"parsersFiles,omitempty"` // Configure a global environment for the storage layer in Service. It is recommended to configure the volume and volumeMount separately for this storage. The hostPath type should be used for that Volume in Fluentbit daemon set. Storage *Storage `json:"storage,omitempty"` // Per-namespace re-emitter configuration @@ -184,6 +188,11 @@ func (s *Service) Params() *params.KVs { m.Insert("Parsers_File", s.ParsersFile) } } + if len(s.ParsersFiles) != 0 { + for _, parserFile := range s.ParsersFiles { + m.Insert("Parsers_File", parserFile) + } + } if s.Storage != nil { if s.Storage.Path != "" { m.Insert("storage.path", s.Storage.Path) @@ -210,8 +219,10 @@ func (s *Service) Params() *params.KVs { return m } -func (cfg ClusterFluentBitConfig) RenderMainConfig(sl plugins.SecretLoader, inputs ClusterInputList, filters ClusterFilterList, - outputs ClusterOutputList, nsFilterLists []FilterList, nsOutputLists []OutputList, rewriteTagConfigs []string) (string, error) { +func (cfg ClusterFluentBitConfig) RenderMainConfig( + sl plugins.SecretLoader, inputs ClusterInputList, filters ClusterFilterList, + outputs ClusterOutputList, nsFilterLists []FilterList, nsOutputLists []OutputList, rewriteTagConfigs []string, +) (string, error) { var buf bytes.Buffer // The Service defines the global behaviour of the Fluent Bit engine. @@ -289,8 +300,10 @@ func (cfg ClusterFluentBitConfig) RenderMainConfig(sl plugins.SecretLoader, inpu return buf.String(), nil } -func (cfg ClusterFluentBitConfig) RenderParserConfig(sl plugins.SecretLoader, parsers ClusterParserList, nsParserLists []ParserList, - nsClusterParserLists []ClusterParserList) (string, error) { +func (cfg ClusterFluentBitConfig) RenderParserConfig( + sl plugins.SecretLoader, parsers ClusterParserList, nsParserLists []ParserList, + nsClusterParserLists []ClusterParserList, +) (string, error) { var buf bytes.Buffer existingParsers := make(map[string]bool) parserSections, err := parsers.Load(sl, existingParsers) @@ -326,6 +339,45 @@ func (cfg ClusterFluentBitConfig) RenderParserConfig(sl plugins.SecretLoader, pa return buf.String(), nil } +func (cfg ClusterFluentBitConfig) RenderMultilineParserConfig( + sl plugins.SecretLoader, multilineParsers ClusterMultilineParserList, nsMultilineParserLists []MultilineParserList, + nsClusterMultilineParserLists []ClusterMultilineParserList, +) (string, error) { + var buf bytes.Buffer + + multilineParserSection, err := multilineParsers.Load(sl) + if err != nil { + return "", err + } + + buf.WriteString(multilineParserSection) + + for _, nsmp := range nsMultilineParserLists { + if len(nsmp.Items) == 0 { + continue + } + if nsmp.Items != nil { + ns := nsmp.Items[0].Namespace + namespacedSl := plugins.NewSecretLoader(sl.Client, ns) + nsmpSection, err := nsmp.Load(namespacedSl) + if err != nil { + return "", err + } + buf.WriteString(nsmpSection) + } + } + + for _, nscmp := range nsClusterMultilineParserLists { + nscmpSection, err := nscmp.Load(sl) + if err != nil { + return "", err + } + buf.WriteString(nscmpSection) + } + + return buf.String(), nil +} + // +kubebuilder:object:generate:=false type Script struct { @@ -342,7 +394,9 @@ func (a ByName) Len() int { return len(a) } func (a ByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a ByName) Less(i, j int) bool { return a[i].Name < a[j].Name } -func (cfg ClusterFluentBitConfig) RenderLuaScript(cl plugins.ConfigMapLoader, filters ClusterFilterList, namespace string) ([]Script, error) { +func (cfg ClusterFluentBitConfig) RenderLuaScript( + cl plugins.ConfigMapLoader, filters ClusterFilterList, namespace string, +) ([]Script, error) { scripts := make([]Script, 0) for _, f := range filters.Items { diff --git a/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types_test.go b/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types_test.go index 9a2fe9c93..344b6f5eb 100644 --- a/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types_test.go +++ b/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types_test.go @@ -1,6 +1,9 @@ package v1alpha2 import ( + "fmt" + "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins/multilineparser" + "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins/parser" "testing" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -144,6 +147,44 @@ var expectedK8s = `[Service] Write_Operation update ` +var expectedParsers = `[PARSER] + Name clusterparser0 + Format json + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S %z +[PARSER] + Name parser0-4087ca5ebba883e13a4369122e716be7 + Format regex + Regex .* + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S %z +[PARSER] + Name clusterparser1 + Format ltsv + Time_Key time + Time_Format [%d/%b/%Y:%H:%M:%S %z] + Types status:integer size:integer +` + +var expectedMultilineParsers = `[MULTILINE_PARSER] + Name clustermultilineparser0 + Type regex + Parser go + Key_Content log +[MULTILINE_PARSER] + Name multilineparser0 + Type regex + Flush_Timeout 1000 + Rule "start_state" "/(Dec \d+ \d+\:\d+\:\d+)(.*)/" "cont" + Rule "cont" "/^\s+at.*/" "cont" +[MULTILINE_PARSER] + Name clustermultilineparser1 + Type regex + Flush_Timeout 500 + Rule "start_state" "/^(\d+ \d+\:\d+\:\d+)(.*)/" "cont" + Rule "cont" "/^\s+at.*/" "cont" +` + var labels = map[string]string{ "label0": "lv0", "label1": "lv1", @@ -153,14 +194,16 @@ var labels = map[string]string{ } var cfg = ClusterFluentBitConfig{ - Spec: FluentBitConfigSpec{Service: &Service{ - Daemon: ptrBool(false), - FlushSeconds: ptrInt64(1), - GraceSeconds: ptrInt64(30), - HttpServer: ptrBool(true), - LogLevel: "info", - ParsersFile: "parsers.conf", - }}, + Spec: FluentBitConfigSpec{ + Service: &Service{ + Daemon: ptrBool(false), + FlushSeconds: ptrInt64(1), + GraceSeconds: ptrInt64(30), + HttpServer: ptrBool(true), + LogLevel: "info", + ParsersFile: "parsers.conf", + }, + }, } func Test_FluentBitConfig_RenderMainConfig(t *testing.T) { @@ -191,7 +234,8 @@ func Test_FluentBitConfig_RenderMainConfig(t *testing.T) { MemBufLimit: "5MB", RefreshIntervalSeconds: ptrInt64(10), DB: "/fluent-bit/tail/pos.db", - }}, + }, + }, } inputs := ClusterInputList{ @@ -417,7 +461,8 @@ func TestRenderMainConfigK8s(t *testing.T) { MemBufLimit: "5MB", RefreshIntervalSeconds: ptrInt64(10), DB: "/fluent-bit/tail/pos.db", - }}, + }, + }, } inputList := ClusterInputList{ Items: []ClusterInput{*inputObj}, @@ -504,10 +549,212 @@ func TestRenderMainConfigK8s(t *testing.T) { } var rewriteTagCfg []string - config, err := cfg.RenderMainConfig(sl, inputList, filterList, outputList, nsFilterList, nsOutputList, rewriteTagCfg) + config, err := cfg.RenderMainConfig( + sl, inputList, filterList, outputList, nsFilterList, nsOutputList, rewriteTagCfg, + ) g.Expect(err).NotTo(HaveOccurred()) g.Expect(config).To(Equal(expectedK8s)) +} + +func TestClusterFluentBitConfig_RenderMainConfig_WithParsersFiles(t *testing.T) { + g := NewGomegaWithT(t) + sl := plugins.NewSecretLoader(nil, "testnamespace") + + cfbc := ClusterFluentBitConfig{ + Spec: FluentBitConfigSpec{ + Service: &Service{ + Daemon: ptrBool(false), + FlushSeconds: ptrInt64(1), + GraceSeconds: ptrInt64(30), + HttpServer: ptrBool(true), + LogLevel: "info", + ParsersFiles: []string{"parsers.conf", "parsers_multiline.conf"}, + }, + }, + } + + config, err := cfbc.RenderMainConfig( + sl, ClusterInputList{}, ClusterFilterList{}, ClusterOutputList{}, nil, nil, nil, + ) + g.Expect(err).NotTo(HaveOccurred()) + fmt.Println(config) +} + +func TestClusterFluentBitConfig_RenderParserConfig(t *testing.T) { + g := NewGomegaWithT(t) + sl := plugins.NewSecretLoader(nil, "testnamespace") + clusterParser := &ClusterParser{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "fluentbit.fluent.io/v1alpha2", + Kind: "ClusterParser", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "clusterparser0", + Labels: labels, + }, + Spec: ParserSpec{ + JSON: &parser.JSON{ + TimeKey: "time", + TimeFormat: "%Y-%m-%dT%H:%M:%S %z", + }, + }, + } + clusterParsers := ClusterParserList{ + Items: []ClusterParser{*clusterParser}, + } + + nsParser := &Parser{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "fluentbit.fluent.io/v1alpha2", + Kind: "Parser", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "parser0", + Namespace: "testnamespace", + Labels: labels, + }, + Spec: ParserSpec{ + Regex: &parser.Regex{ + Regex: ".*", + TimeKey: "time", + TimeFormat: "%Y-%m-%dT%H:%M:%S %z", + }, + }, + } + nsParsers := []ParserList{ + { + Items: []Parser{*nsParser}, + }, + } + + nsClusterParser := &ClusterParser{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "fluentbit.fluent.io/v1alpha2", + Kind: "ClusterParser", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "clusterparser1", + Labels: labels, + }, + Spec: ParserSpec{ + LTSV: &parser.LSTV{ + TimeKey: "time", + TimeFormat: "[%d/%b/%Y:%H:%M:%S %z]", + Types: "status:integer size:integer", + }, + }, + } + nsClusterParsers := []ClusterParserList{ + { + Items: []ClusterParser{*nsClusterParser}, + }, + } + + config, err := cfg.RenderParserConfig(sl, clusterParsers, nsParsers, nsClusterParsers) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(config).To(Equal(expectedParsers)) +} + +func TestClusterFluentBitConfig_RenderMultilineParserConfig(t *testing.T) { + g := NewGomegaWithT(t) + sl := plugins.NewSecretLoader(nil, "testnamespace") + + clusterMultilineParser := &ClusterMultilineParser{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "fluentbit.fluent.io/v1alpha2", + Kind: "ClusterMultilineParser", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "clustermultilineparser0", + Labels: labels, + }, + Spec: MultilineParserSpec{ + MultilineParser: &multilineparser.MultilineParser{ + Type: "regex", + Parser: "go", + KeyContent: "log", + }, + }, + } + clusterMultilineParsers := ClusterMultilineParserList{ + Items: []ClusterMultilineParser{*clusterMultilineParser}, + } + + nsMultilineParser := &MultilineParser{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "fluentbit.fluent.io/v1alpha2", + Kind: "MultilineParser", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "multilineparser0", + Namespace: "testnamespace", + Labels: labels, + }, + Spec: MultilineParserSpec{ + MultilineParser: &multilineparser.MultilineParser{ + Type: "regex", + FlushTimeout: 1000, + Rules: []multilineparser.Rule{ + { + Start: "start_state", + Regex: `/(Dec \d+ \d+\:\d+\:\d+)(.*)/`, + Next: "cont", + }, + { + Start: "cont", + Regex: `/^\s+at.*/`, + Next: "cont", + }, + }, + }, + }, + } + nsMultilineParsers := []MultilineParserList{ + { + Items: []MultilineParser{*nsMultilineParser}, + }, + } + + nsClusterMultilineParser := &ClusterMultilineParser{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "fluentbit.fluent.io/v1alpha2", + Kind: "ClusterMultilineParser", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "clustermultilineparser1", + Labels: labels, + }, + Spec: MultilineParserSpec{ + MultilineParser: &multilineparser.MultilineParser{ + Type: "regex", + FlushTimeout: 500, + Rules: []multilineparser.Rule{ + { + Start: "start_state", + Regex: `/^(\d+ \d+\:\d+\:\d+)(.*)/`, + Next: "cont", + }, + { + Start: "cont", + Regex: `/^\s+at.*/`, + Next: "cont", + }, + }, + }, + }, + } + nsClusterMultilineParsers := []ClusterMultilineParserList{ + { + Items: []ClusterMultilineParser{*nsClusterMultilineParser}, + }, + } + + config, err := cfg.RenderMultilineParserConfig( + sl, clusterMultilineParsers, nsMultilineParsers, nsClusterMultilineParsers, + ) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(config).To(Equal(expectedMultilineParsers)) } func ptrBool(v bool) *bool { diff --git a/apis/fluentbit/v1alpha2/clustermultilineparser_types.go b/apis/fluentbit/v1alpha2/clustermultilineparser_types.go new file mode 100644 index 000000000..0d31a70d0 --- /dev/null +++ b/apis/fluentbit/v1alpha2/clustermultilineparser_types.go @@ -0,0 +1,110 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha2 + +import ( + "bytes" + "fmt" + "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins" + "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins/multilineparser" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "reflect" + "sort" +) + +// +kubebuilder:object:generate:=true + +type MultilineParserSpec struct { + MultilineParser *multilineparser.MultilineParser `json:",inline"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:shortName=cfbmp,scope=Cluster +// +genclient +// +genclient:nonNamespaced + +// ClusterMultilineParser is the Schema for the cluster-level multiline parser API +type ClusterMultilineParser struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec MultilineParserSpec `json:"spec,omitempty"` +} + +// +kubebuilder:object:root=true + +// ClusterMultilineParserList contains a list of ClusterMultilineParser +type ClusterMultilineParserList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ClusterMultilineParser `json:"items"` +} + +// +kubebuilder:object:generate:=false + +// ClusterMultilineParserByName implements sort.Interface for []ClusterParser based on the Name field. +type ClusterMultilineParserByName []ClusterMultilineParser + +func (a ClusterMultilineParserByName) Len() int { + return len(a) +} + +func (a ClusterMultilineParserByName) Swap(i, j int) { + a[i], a[j] = a[j], a[i] +} + +func (a ClusterMultilineParserByName) Less(i, j int) bool { + return a[i].Name < a[j].Name +} + +func (list ClusterMultilineParserList) Load(sl plugins.SecretLoader) (string, error) { + var buf bytes.Buffer + + sort.Sort(ClusterMultilineParserByName(list.Items)) + + for _, item := range list.Items { + merge := func(p plugins.Plugin) error { + if p == nil || reflect.ValueOf(p).IsNil() { + return nil + } + + buf.WriteString("[MULTILINE_PARSER]\n") + buf.WriteString(fmt.Sprintf(" Name %s\n", item.Name)) + + kvs, err := p.Params(sl) + if err != nil { + return err + } + buf.WriteString(kvs.String()) + + return nil + } + + for i := 0; i < reflect.ValueOf(item.Spec).NumField(); i++ { + p, _ := reflect.ValueOf(item.Spec).Field(i).Interface().(plugins.Plugin) + if err := merge(p); err != nil { + return "", err + } + } + } + + return buf.String(), nil +} + +func init() { + SchemeBuilder.Register(&ClusterMultilineParser{}, &ClusterMultilineParserList{}) +} diff --git a/apis/fluentbit/v1alpha2/clustermultilineparser_types_test.go b/apis/fluentbit/v1alpha2/clustermultilineparser_types_test.go new file mode 100644 index 000000000..dc5aa883b --- /dev/null +++ b/apis/fluentbit/v1alpha2/clustermultilineparser_types_test.go @@ -0,0 +1,85 @@ +package v1alpha2 + +import ( + "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins" + "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins/multilineparser" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "testing" +) + +var clusterMultilineParserExpected = `[MULTILINE_PARSER] + Name clustermultilineparser_test0 + Type regex + Parser go + Key_Content log +[MULTILINE_PARSER] + Name clustermultilineparser_test1 + Type regex + Rule "start_state" "/([a-zA-Z]+ \d+ \d+\:\d+\:\d+)(.*)/" "cont" + Rule "cont" "/^\s+at.*/" "cont" +` + +func TestClusterMultilineParserList_Load(t *testing.T) { + g := NewGomegaWithT(t) + + sl := plugins.NewSecretLoader(nil, "testnamespace") + + labels := map[string]string{ + "label0": "lv0", + } + + goMultilineParser := &ClusterMultilineParser{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "fluentbit.fluent.io/v1alpha2", + Kind: "ClusterMultilineParser", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "clustermultilineparser_test0", + Labels: labels, + }, + Spec: MultilineParserSpec{ + MultilineParser: &multilineparser.MultilineParser{ + Type: "regex", + Parser: "go", + KeyContent: "log", + }, + }, + } + + customMultilineParser := &ClusterMultilineParser{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "fluentbit.fluent.io/v1alpha2", + Kind: "ClusterMultilineParser", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "clustermultilineparser_test1", + Labels: labels, + }, + Spec: MultilineParserSpec{ + MultilineParser: &multilineparser.MultilineParser{ + Type: "regex", + Rules: []multilineparser.Rule{ + { + Start: "start_state", + Regex: `/([a-zA-Z]+ \d+ \d+\:\d+\:\d+)(.*)/`, + Next: "cont", + }, + { + Start: "cont", + Regex: `/^\s+at.*/`, + Next: "cont", + }, + }, + }, + }, + } + + clustermultilineparsers := ClusterMultilineParserList{ + Items: []ClusterMultilineParser{*goMultilineParser, *customMultilineParser}, + } + + cmp, err := clustermultilineparsers.Load(sl) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(cmp).To(Equal(clusterMultilineParserExpected)) +} diff --git a/apis/fluentbit/v1alpha2/fluentbitconfig_types.go b/apis/fluentbit/v1alpha2/fluentbitconfig_types.go index 29b13b9d8..1087f8f38 100644 --- a/apis/fluentbit/v1alpha2/fluentbitconfig_types.go +++ b/apis/fluentbit/v1alpha2/fluentbitconfig_types.go @@ -35,6 +35,10 @@ type NamespacedFluentBitCfgSpec struct { Service *Service `json:"service,omitempty"` // Select cluster level parser config ClusterParserSelector metav1.LabelSelector `json:"clusterParserSelector,omitempty"` + // Select multiline parser plugins + MultilineParserSelector metav1.LabelSelector `json:"multilineParserSelector,omitempty"` + // Select cluster level multiline parser config + ClusterMultilineParserSelector metav1.LabelSelector `json:"clusterMultilineParserSelector,omitempty"` } // +kubebuilder:object:root=true diff --git a/apis/fluentbit/v1alpha2/multilineparser_types.go b/apis/fluentbit/v1alpha2/multilineparser_types.go new file mode 100644 index 000000000..ec662e6b1 --- /dev/null +++ b/apis/fluentbit/v1alpha2/multilineparser_types.go @@ -0,0 +1,101 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha2 + +import ( + "bytes" + "fmt" + "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "reflect" + "sort" +) + +// +kubebuilder:object:root=true +// +kubebuilder:resource:shortName=fbmp +// +genclient + +// MultilineParser is the Schema of namespace-level multiline parser API +type MultilineParser struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec MultilineParserSpec `json:"spec,omitempty"` +} + +// +kubebuilder:object:root=true + +// MultilineParserList contains a list of MultilineParser +type MultilineParserList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []MultilineParser `json:"items"` +} + +// +kubebuilder:object:generate:=false + +type MultilineParserByName []MultilineParser + +func (a MultilineParserByName) Len() int { + return len(a) +} + +func (a MultilineParserByName) Swap(i, j int) { + a[i], a[j] = a[j], a[i] +} + +func (a MultilineParserByName) Less(i, j int) bool { + return a[i].Name < a[j].Name +} + +func (list MultilineParserList) Load(sl plugins.SecretLoader) (string, error) { + var buf bytes.Buffer + + sort.Sort(MultilineParserByName(list.Items)) + + for _, item := range list.Items { + merge := func(p plugins.Plugin) error { + if p == nil || reflect.ValueOf(p).IsNil() { + return nil + } + + buf.WriteString("[MULTILINE_PARSER]\n") + buf.WriteString(fmt.Sprintf(" Name %s\n", item.Name)) + + kvs, err := p.Params(sl) + if err != nil { + return err + } + buf.WriteString(kvs.String()) + + return nil + } + + for i := 0; i < reflect.ValueOf(item.Spec).NumField(); i++ { + p, _ := reflect.ValueOf(item.Spec).Field(i).Interface().(plugins.Plugin) + if err := merge(p); err != nil { + return "", err + } + } + } + + return buf.String(), nil +} + +func init() { + SchemeBuilder.Register(&MultilineParser{}, &MultilineParserList{}) +} diff --git a/apis/fluentbit/v1alpha2/multilineparser_types_test.go b/apis/fluentbit/v1alpha2/multilineparser_types_test.go new file mode 100644 index 000000000..6e637057b --- /dev/null +++ b/apis/fluentbit/v1alpha2/multilineparser_types_test.go @@ -0,0 +1,85 @@ +package v1alpha2 + +import ( + "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins" + "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins/multilineparser" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "testing" +) + +var multilineParserExpected = `[MULTILINE_PARSER] + Name multilineparser_test0 + Type regex + Parser go + Key_Content log +[MULTILINE_PARSER] + Name multilineparser_test1 + Type regex + Rule "start_state" "/([a-zA-Z]+ \d+ \d+\:\d+\:\d+)(.*)/" "cont" + Rule "cont" "/^\s+at.*/" "cont" +` + +func TestMultilineParserList_Load(t *testing.T) { + g := NewGomegaWithT(t) + + sl := plugins.NewSecretLoader(nil, "testnamespace") + + labels := map[string]string{ + "label0": "lv0", + } + + goMultilineParser := &MultilineParser{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "fluentbit.fluent.io/v1alpha2", + Kind: "MultilineParser", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "multilineparser_test0", + Labels: labels, + }, + Spec: MultilineParserSpec{ + MultilineParser: &multilineparser.MultilineParser{ + Type: "regex", + Parser: "go", + KeyContent: "log", + }, + }, + } + + customMultilineParser := &MultilineParser{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "fluentbit.fluent.io/v1alpha2", + Kind: "MultilineParser", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "multilineparser_test1", + Labels: labels, + }, + Spec: MultilineParserSpec{ + MultilineParser: &multilineparser.MultilineParser{ + Type: "regex", + Rules: []multilineparser.Rule{ + { + Start: "start_state", + Regex: `/([a-zA-Z]+ \d+ \d+\:\d+\:\d+)(.*)/`, + Next: "cont", + }, + { + Start: "cont", + Regex: `/^\s+at.*/`, + Next: "cont", + }, + }, + }, + }, + } + + multilineparsers := MultilineParserList{ + Items: []MultilineParser{*goMultilineParser, *customMultilineParser}, + } + + mp, err := multilineparsers.Load(sl) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(mp).To(Equal(multilineParserExpected)) +} diff --git a/apis/fluentbit/v1alpha2/plugins/filter/multiline_types.go b/apis/fluentbit/v1alpha2/plugins/filter/multiline_types.go index ad9b28cc1..f1b0f6f2c 100644 --- a/apis/fluentbit/v1alpha2/plugins/filter/multiline_types.go +++ b/apis/fluentbit/v1alpha2/plugins/filter/multiline_types.go @@ -1,6 +1,7 @@ package filter import ( + "fmt" "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins" "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins/params" ) @@ -22,9 +23,21 @@ type Multi struct { //Key name that holds the content to process. //Note that a Multiline Parser definition can already specify the key_content to use, but this option allows to overwrite that value for the purpose of the filter. KeyContent string `json:"keyContent,omitempty"` + // +kubebuilder:validation:Enum:=parser;partial_message + Mode string `json:"mode,omitempty"` + // +kubebuilder:default:=false + Buffer bool `json:"buffer,omitempty"` + // +kubebuilder:default:=2000 + FlushMS int `json:"flushMs,omitempty"` + // Name for the emitter input instance which re-emits the completed records at the beginning of the pipeline. EmitterName string `json:"emitterName,omitempty"` - EmitterMemBufLimit string `json:"emitterMemBufLimit,omitempty"` + // The storage type for the emitter input instance. This option supports the values memory (default) and filesystem. + // +kubebuilder:validation:Enum:=memory;filesystem + // +kubebuilder:default:=memory EmitterStorageType string `json:"emitterStorageType,omitempty"` + // Set a limit on the amount of memory the emitter can consume if the outputs provide backpressure. The default for this limit is 10M. The pipeline will pause once the buffer exceeds the value of this setting. For example, if the value is set to 10M then the pipeline will pause if the buffer exceeds 10M. The pipeline will remain paused until the output drains the buffer below the 10M limit. + // +kubebuilder:default:=10 + EmitterMemBufLimit int `json:"emitterMemBufLimit,omitempty"` } func (_ *Multiline) Name() string { @@ -44,15 +57,24 @@ func (m *Multiline) Params(_ plugins.SecretLoader) (*params.KVs, error) { if m.Multi.KeyContent != "" { kvs.Insert("multiline.key_content", m.Multi.KeyContent) } - if m.Multi.EmitterName != "" { - kvs.Insert("Emitter_Name", m.Multi.EmitterName) - } - if m.Multi.EmitterMemBufLimit != "" { - kvs.Insert("Emitter_Mem_Buf_Limit", m.Multi.EmitterMemBufLimit) - } - if m.Multi.EmitterStorageType != "" { - kvs.Insert("Emitter_Storage.type", m.Multi.EmitterStorageType) - } + if m.Multi.Mode != "" { + kvs.Insert("mode", m.Multi.Mode) + } + if m.Multi.Buffer != false { + kvs.Insert("buffer", fmt.Sprint(m.Multi.Buffer)) + } + if m.Multi.FlushMS != 0 { + kvs.Insert("flush_ms", fmt.Sprint(m.Multi.FlushMS)) + } + if m.Multi.EmitterName != "" { + kvs.Insert("emitter_name", m.Multi.EmitterName) + } + if m.Multi.EmitterStorageType != "" { + kvs.Insert("Emitter_Storage.type", m.Multi.EmitterStorageType) + } + if m.Multi.EmitterMemBufLimit != 0 { + kvs.Insert("emitter_mem_buf_limit", fmt.Sprint(m.Multi.EmitterMemBufLimit)) + } } return kvs, nil } diff --git a/apis/fluentbit/v1alpha2/plugins/multilineparser/multilineparser_types.go b/apis/fluentbit/v1alpha2/plugins/multilineparser/multilineparser_types.go new file mode 100644 index 000000000..285ac9ad6 --- /dev/null +++ b/apis/fluentbit/v1alpha2/plugins/multilineparser/multilineparser_types.go @@ -0,0 +1,60 @@ +package multilineparser + +import ( + "fmt" + "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins" + "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins/params" +) + +// +kubebuilder:object:generate:=true + +// **For full documentation, refer to https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/multiline-parsing** +type MultilineParser struct { + // Set the multiline mode, for now, we support the type regex. + // +kubebuilder:validation:Enum:=regex + // +kubebuilder:default:=regex + Type string `json:"type,omitempty"` + // Name of a pre-defined parser that must be applied to the incoming content before applying the regex rule. If no parser is defined, it's assumed that's a raw text and not a structured message. + Parser string `json:"parser,omitempty"` + // For an incoming structured message, specify the key that contains the data that should be processed by the regular expression and possibly concatenated. + KeyContent string `json:"keyContent,omitempty"` + // Timeout in milliseconds to flush a non-terminated multiline buffer. Default is set to 5 seconds. + // +kubebuilder:default:=5000 + FlushTimeout int `json:"flushTimeout,omitempty"` + // Configure a rule to match a multiline pattern. The rule has a specific format described below. Multiple rules can be defined. + Rules []Rule `json:"rules,omitempty"` +} + +type Rule struct { + Start string `json:"start"` + Regex string `json:"regex"` + Next string `json:"next"` +} + +func (_ *MultilineParser) Name() string { + return "multilineparser" +} + +func (m *MultilineParser) Params(_ plugins.SecretLoader) (*params.KVs, error) { + kvs := params.NewKVs() + + if m.Type != "" { + kvs.Insert("Type", m.Type) + } + if m.Parser != "" { + kvs.Insert("Parser", m.Parser) + } + if m.KeyContent != "" { + kvs.Insert("Key_Content", m.KeyContent) + } + if m.FlushTimeout != 0 { + kvs.Insert("Flush_Timeout", fmt.Sprint(m.FlushTimeout)) + } + if len(m.Rules) != 0 { + for _, rule := range m.Rules { + // add quotes and don't try to escape characters + kvs.Insert("Rule", fmt.Sprintf("\"%s\" \"%s\" \"%s\"", rule.Start, rule.Regex, rule.Next)) + } + } + return kvs, nil +} diff --git a/apis/fluentbit/v1alpha2/plugins/multilineparser/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/plugins/multilineparser/zz_generated.deepcopy.go new file mode 100644 index 000000000..9590dd2bc --- /dev/null +++ b/apis/fluentbit/v1alpha2/plugins/multilineparser/zz_generated.deepcopy.go @@ -0,0 +1,44 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package multilineparser + +import () + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MultilineParser) DeepCopyInto(out *MultilineParser) { + *out = *in + if in.Rules != nil { + in, out := &in.Rules, &out.Rules + *out = make([]Rule, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MultilineParser. +func (in *MultilineParser) DeepCopy() *MultilineParser { + if in == nil { + return nil + } + out := new(MultilineParser) + in.DeepCopyInto(out) + return out +} diff --git a/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go index c699b3eb9..724c175cc 100644 --- a/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go +++ b/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go @@ -25,6 +25,7 @@ import ( "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins/custom" "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins/filter" "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins/input" + "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins/multilineparser" "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins/output" "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins/parser" "k8s.io/api/core/v1" @@ -206,6 +207,64 @@ func (in *ClusterInputList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterMultilineParser) DeepCopyInto(out *ClusterMultilineParser) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterMultilineParser. +func (in *ClusterMultilineParser) DeepCopy() *ClusterMultilineParser { + if in == nil { + return nil + } + out := new(ClusterMultilineParser) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterMultilineParser) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterMultilineParserList) DeepCopyInto(out *ClusterMultilineParserList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ClusterMultilineParser, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterMultilineParserList. +func (in *ClusterMultilineParserList) DeepCopy() *ClusterMultilineParserList { + if in == nil { + return nil + } + out := new(ClusterMultilineParserList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterMultilineParserList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterOutput) DeepCopyInto(out *ClusterOutput) { *out = *in @@ -798,6 +857,7 @@ func (in *FluentBitConfigSpec) DeepCopyInto(out *FluentBitConfigSpec) { in.FilterSelector.DeepCopyInto(&out.FilterSelector) in.OutputSelector.DeepCopyInto(&out.OutputSelector) in.ParserSelector.DeepCopyInto(&out.ParserSelector) + in.MultilineParserSelector.DeepCopyInto(&out.MultilineParserSelector) if in.Namespace != nil { in, out := &in.Namespace, &out.Namespace *out = new(string) @@ -1130,6 +1190,84 @@ func (in *InputSpec) DeepCopy() *InputSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MultilineParser) DeepCopyInto(out *MultilineParser) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MultilineParser. +func (in *MultilineParser) DeepCopy() *MultilineParser { + if in == nil { + return nil + } + out := new(MultilineParser) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *MultilineParser) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MultilineParserList) DeepCopyInto(out *MultilineParserList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]MultilineParser, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MultilineParserList. +func (in *MultilineParserList) DeepCopy() *MultilineParserList { + if in == nil { + return nil + } + out := new(MultilineParserList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *MultilineParserList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MultilineParserSpec) DeepCopyInto(out *MultilineParserSpec) { + *out = *in + if in.MultilineParser != nil { + in, out := &in.MultilineParser, &out.MultilineParser + *out = new(multilineparser.MultilineParser) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MultilineParserSpec. +func (in *MultilineParserSpec) DeepCopy() *MultilineParserSpec { + if in == nil { + return nil + } + out := new(MultilineParserSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in NSFilterByName) DeepCopyInto(out *NSFilterByName) { { @@ -1200,6 +1338,8 @@ func (in *NamespacedFluentBitCfgSpec) DeepCopyInto(out *NamespacedFluentBitCfgSp in.OutputSelector.DeepCopyInto(&out.OutputSelector) in.ParserSelector.DeepCopyInto(&out.ParserSelector) in.ClusterParserSelector.DeepCopyInto(&out.ClusterParserSelector) + in.MultilineParserSelector.DeepCopyInto(&out.MultilineParserSelector) + in.ClusterMultilineParserSelector.DeepCopyInto(&out.ClusterMultilineParserSelector) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NamespacedFluentBitCfgSpec. @@ -1561,6 +1701,11 @@ func (in *Service) DeepCopyInto(out *Service) { *out = new(bool) **out = **in } + if in.ParsersFiles != nil { + in, out := &in.ParsersFiles, &out.ParsersFiles + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.Storage != nil { in, out := &in.Storage, &out.Storage *out = new(Storage) diff --git a/apis/generated/clientset/versioned/typed/fluentbit/v1alpha2/clustermultilineparser.go b/apis/generated/clientset/versioned/typed/fluentbit/v1alpha2/clustermultilineparser.go new file mode 100644 index 000000000..6b2da9614 --- /dev/null +++ b/apis/generated/clientset/versioned/typed/fluentbit/v1alpha2/clustermultilineparser.go @@ -0,0 +1,167 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha2 + +import ( + "context" + "time" + + v1alpha2 "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2" + scheme "github.com/fluent/fluent-operator/v2/apis/generated/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// ClusterMultilineParsersGetter has a method to return a ClusterMultilineParserInterface. +// A group's client should implement this interface. +type ClusterMultilineParsersGetter interface { + ClusterMultilineParsers() ClusterMultilineParserInterface +} + +// ClusterMultilineParserInterface has methods to work with ClusterMultilineParser resources. +type ClusterMultilineParserInterface interface { + Create(ctx context.Context, clusterMultilineParser *v1alpha2.ClusterMultilineParser, opts v1.CreateOptions) (*v1alpha2.ClusterMultilineParser, error) + Update(ctx context.Context, clusterMultilineParser *v1alpha2.ClusterMultilineParser, opts v1.UpdateOptions) (*v1alpha2.ClusterMultilineParser, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha2.ClusterMultilineParser, error) + List(ctx context.Context, opts v1.ListOptions) (*v1alpha2.ClusterMultilineParserList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha2.ClusterMultilineParser, err error) + ClusterMultilineParserExpansion +} + +// clusterMultilineParsers implements ClusterMultilineParserInterface +type clusterMultilineParsers struct { + client rest.Interface +} + +// newClusterMultilineParsers returns a ClusterMultilineParsers +func newClusterMultilineParsers(c *FluentbitV1alpha2Client) *clusterMultilineParsers { + return &clusterMultilineParsers{ + client: c.RESTClient(), + } +} + +// Get takes name of the clusterMultilineParser, and returns the corresponding clusterMultilineParser object, and an error if there is any. +func (c *clusterMultilineParsers) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha2.ClusterMultilineParser, err error) { + result = &v1alpha2.ClusterMultilineParser{} + err = c.client.Get(). + Resource("clustermultilineparsers"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of ClusterMultilineParsers that match those selectors. +func (c *clusterMultilineParsers) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha2.ClusterMultilineParserList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + result = &v1alpha2.ClusterMultilineParserList{} + err = c.client.Get(). + Resource("clustermultilineparsers"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Do(ctx). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested clusterMultilineParsers. +func (c *clusterMultilineParsers) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + opts.Watch = true + return c.client.Get(). + Resource("clustermultilineparsers"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Watch(ctx) +} + +// Create takes the representation of a clusterMultilineParser and creates it. Returns the server's representation of the clusterMultilineParser, and an error, if there is any. +func (c *clusterMultilineParsers) Create(ctx context.Context, clusterMultilineParser *v1alpha2.ClusterMultilineParser, opts v1.CreateOptions) (result *v1alpha2.ClusterMultilineParser, err error) { + result = &v1alpha2.ClusterMultilineParser{} + err = c.client.Post(). + Resource("clustermultilineparsers"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(clusterMultilineParser). + Do(ctx). + Into(result) + return +} + +// Update takes the representation of a clusterMultilineParser and updates it. Returns the server's representation of the clusterMultilineParser, and an error, if there is any. +func (c *clusterMultilineParsers) Update(ctx context.Context, clusterMultilineParser *v1alpha2.ClusterMultilineParser, opts v1.UpdateOptions) (result *v1alpha2.ClusterMultilineParser, err error) { + result = &v1alpha2.ClusterMultilineParser{} + err = c.client.Put(). + Resource("clustermultilineparsers"). + Name(clusterMultilineParser.Name). + VersionedParams(&opts, scheme.ParameterCodec). + Body(clusterMultilineParser). + Do(ctx). + Into(result) + return +} + +// Delete takes name of the clusterMultilineParser and deletes it. Returns an error if one occurs. +func (c *clusterMultilineParsers) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + return c.client.Delete(). + Resource("clustermultilineparsers"). + Name(name). + Body(&opts). + Do(ctx). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *clusterMultilineParsers) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + var timeout time.Duration + if listOpts.TimeoutSeconds != nil { + timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second + } + return c.client.Delete(). + Resource("clustermultilineparsers"). + VersionedParams(&listOpts, scheme.ParameterCodec). + Timeout(timeout). + Body(&opts). + Do(ctx). + Error() +} + +// Patch applies the patch and returns the patched clusterMultilineParser. +func (c *clusterMultilineParsers) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha2.ClusterMultilineParser, err error) { + result = &v1alpha2.ClusterMultilineParser{} + err = c.client.Patch(pt). + Resource("clustermultilineparsers"). + Name(name). + SubResource(subresources...). + VersionedParams(&opts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} diff --git a/apis/generated/clientset/versioned/typed/fluentbit/v1alpha2/fake/fake_clustermultilineparser.go b/apis/generated/clientset/versioned/typed/fluentbit/v1alpha2/fake/fake_clustermultilineparser.go new file mode 100644 index 000000000..c1026af11 --- /dev/null +++ b/apis/generated/clientset/versioned/typed/fluentbit/v1alpha2/fake/fake_clustermultilineparser.go @@ -0,0 +1,121 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1alpha2 "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeClusterMultilineParsers implements ClusterMultilineParserInterface +type FakeClusterMultilineParsers struct { + Fake *FakeFluentbitV1alpha2 +} + +var clustermultilineparsersResource = schema.GroupVersionResource{Group: "fluentbit.fluent.io", Version: "v1alpha2", Resource: "clustermultilineparsers"} + +var clustermultilineparsersKind = schema.GroupVersionKind{Group: "fluentbit.fluent.io", Version: "v1alpha2", Kind: "ClusterMultilineParser"} + +// Get takes name of the clusterMultilineParser, and returns the corresponding clusterMultilineParser object, and an error if there is any. +func (c *FakeClusterMultilineParsers) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha2.ClusterMultilineParser, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootGetAction(clustermultilineparsersResource, name), &v1alpha2.ClusterMultilineParser{}) + if obj == nil { + return nil, err + } + return obj.(*v1alpha2.ClusterMultilineParser), err +} + +// List takes label and field selectors, and returns the list of ClusterMultilineParsers that match those selectors. +func (c *FakeClusterMultilineParsers) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha2.ClusterMultilineParserList, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootListAction(clustermultilineparsersResource, clustermultilineparsersKind, opts), &v1alpha2.ClusterMultilineParserList{}) + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha2.ClusterMultilineParserList{ListMeta: obj.(*v1alpha2.ClusterMultilineParserList).ListMeta} + for _, item := range obj.(*v1alpha2.ClusterMultilineParserList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested clusterMultilineParsers. +func (c *FakeClusterMultilineParsers) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewRootWatchAction(clustermultilineparsersResource, opts)) +} + +// Create takes the representation of a clusterMultilineParser and creates it. Returns the server's representation of the clusterMultilineParser, and an error, if there is any. +func (c *FakeClusterMultilineParsers) Create(ctx context.Context, clusterMultilineParser *v1alpha2.ClusterMultilineParser, opts v1.CreateOptions) (result *v1alpha2.ClusterMultilineParser, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootCreateAction(clustermultilineparsersResource, clusterMultilineParser), &v1alpha2.ClusterMultilineParser{}) + if obj == nil { + return nil, err + } + return obj.(*v1alpha2.ClusterMultilineParser), err +} + +// Update takes the representation of a clusterMultilineParser and updates it. Returns the server's representation of the clusterMultilineParser, and an error, if there is any. +func (c *FakeClusterMultilineParsers) Update(ctx context.Context, clusterMultilineParser *v1alpha2.ClusterMultilineParser, opts v1.UpdateOptions) (result *v1alpha2.ClusterMultilineParser, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateAction(clustermultilineparsersResource, clusterMultilineParser), &v1alpha2.ClusterMultilineParser{}) + if obj == nil { + return nil, err + } + return obj.(*v1alpha2.ClusterMultilineParser), err +} + +// Delete takes name of the clusterMultilineParser and deletes it. Returns an error if one occurs. +func (c *FakeClusterMultilineParsers) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewRootDeleteActionWithOptions(clustermultilineparsersResource, name, opts), &v1alpha2.ClusterMultilineParser{}) + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeClusterMultilineParsers) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewRootDeleteCollectionAction(clustermultilineparsersResource, listOpts) + + _, err := c.Fake.Invokes(action, &v1alpha2.ClusterMultilineParserList{}) + return err +} + +// Patch applies the patch and returns the patched clusterMultilineParser. +func (c *FakeClusterMultilineParsers) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha2.ClusterMultilineParser, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootPatchSubresourceAction(clustermultilineparsersResource, name, pt, data, subresources...), &v1alpha2.ClusterMultilineParser{}) + if obj == nil { + return nil, err + } + return obj.(*v1alpha2.ClusterMultilineParser), err +} diff --git a/apis/generated/clientset/versioned/typed/fluentbit/v1alpha2/fake/fake_fluentbit_client.go b/apis/generated/clientset/versioned/typed/fluentbit/v1alpha2/fake/fake_fluentbit_client.go index 5da1a2d8b..e2944b092 100644 --- a/apis/generated/clientset/versioned/typed/fluentbit/v1alpha2/fake/fake_fluentbit_client.go +++ b/apis/generated/clientset/versioned/typed/fluentbit/v1alpha2/fake/fake_fluentbit_client.go @@ -39,6 +39,10 @@ func (c *FakeFluentbitV1alpha2) ClusterInputs() v1alpha2.ClusterInputInterface { return &FakeClusterInputs{c} } +func (c *FakeFluentbitV1alpha2) ClusterMultilineParsers() v1alpha2.ClusterMultilineParserInterface { + return &FakeClusterMultilineParsers{c} +} + func (c *FakeFluentbitV1alpha2) ClusterOutputs() v1alpha2.ClusterOutputInterface { return &FakeClusterOutputs{c} } @@ -63,6 +67,10 @@ func (c *FakeFluentbitV1alpha2) FluentBitConfigs(namespace string) v1alpha2.Flue return &FakeFluentBitConfigs{c, namespace} } +func (c *FakeFluentbitV1alpha2) MultilineParsers(namespace string) v1alpha2.MultilineParserInterface { + return &FakeMultilineParsers{c, namespace} +} + func (c *FakeFluentbitV1alpha2) Outputs(namespace string) v1alpha2.OutputInterface { return &FakeOutputs{c, namespace} } diff --git a/apis/generated/clientset/versioned/typed/fluentbit/v1alpha2/fake/fake_multilineparser.go b/apis/generated/clientset/versioned/typed/fluentbit/v1alpha2/fake/fake_multilineparser.go new file mode 100644 index 000000000..005083269 --- /dev/null +++ b/apis/generated/clientset/versioned/typed/fluentbit/v1alpha2/fake/fake_multilineparser.go @@ -0,0 +1,129 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1alpha2 "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeMultilineParsers implements MultilineParserInterface +type FakeMultilineParsers struct { + Fake *FakeFluentbitV1alpha2 + ns string +} + +var multilineparsersResource = schema.GroupVersionResource{Group: "fluentbit.fluent.io", Version: "v1alpha2", Resource: "multilineparsers"} + +var multilineparsersKind = schema.GroupVersionKind{Group: "fluentbit.fluent.io", Version: "v1alpha2", Kind: "MultilineParser"} + +// Get takes name of the multilineParser, and returns the corresponding multilineParser object, and an error if there is any. +func (c *FakeMultilineParsers) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha2.MultilineParser, err error) { + obj, err := c.Fake. + Invokes(testing.NewGetAction(multilineparsersResource, c.ns, name), &v1alpha2.MultilineParser{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha2.MultilineParser), err +} + +// List takes label and field selectors, and returns the list of MultilineParsers that match those selectors. +func (c *FakeMultilineParsers) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha2.MultilineParserList, err error) { + obj, err := c.Fake. + Invokes(testing.NewListAction(multilineparsersResource, multilineparsersKind, c.ns, opts), &v1alpha2.MultilineParserList{}) + + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha2.MultilineParserList{ListMeta: obj.(*v1alpha2.MultilineParserList).ListMeta} + for _, item := range obj.(*v1alpha2.MultilineParserList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested multilineParsers. +func (c *FakeMultilineParsers) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchAction(multilineparsersResource, c.ns, opts)) + +} + +// Create takes the representation of a multilineParser and creates it. Returns the server's representation of the multilineParser, and an error, if there is any. +func (c *FakeMultilineParsers) Create(ctx context.Context, multilineParser *v1alpha2.MultilineParser, opts v1.CreateOptions) (result *v1alpha2.MultilineParser, err error) { + obj, err := c.Fake. + Invokes(testing.NewCreateAction(multilineparsersResource, c.ns, multilineParser), &v1alpha2.MultilineParser{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha2.MultilineParser), err +} + +// Update takes the representation of a multilineParser and updates it. Returns the server's representation of the multilineParser, and an error, if there is any. +func (c *FakeMultilineParsers) Update(ctx context.Context, multilineParser *v1alpha2.MultilineParser, opts v1.UpdateOptions) (result *v1alpha2.MultilineParser, err error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateAction(multilineparsersResource, c.ns, multilineParser), &v1alpha2.MultilineParser{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha2.MultilineParser), err +} + +// Delete takes name of the multilineParser and deletes it. Returns an error if one occurs. +func (c *FakeMultilineParsers) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteActionWithOptions(multilineparsersResource, c.ns, name, opts), &v1alpha2.MultilineParser{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeMultilineParsers) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionAction(multilineparsersResource, c.ns, listOpts) + + _, err := c.Fake.Invokes(action, &v1alpha2.MultilineParserList{}) + return err +} + +// Patch applies the patch and returns the patched multilineParser. +func (c *FakeMultilineParsers) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha2.MultilineParser, err error) { + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceAction(multilineparsersResource, c.ns, name, pt, data, subresources...), &v1alpha2.MultilineParser{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha2.MultilineParser), err +} diff --git a/apis/generated/clientset/versioned/typed/fluentbit/v1alpha2/fluentbit_client.go b/apis/generated/clientset/versioned/typed/fluentbit/v1alpha2/fluentbit_client.go index 771446eb3..bf6aa04d6 100644 --- a/apis/generated/clientset/versioned/typed/fluentbit/v1alpha2/fluentbit_client.go +++ b/apis/generated/clientset/versioned/typed/fluentbit/v1alpha2/fluentbit_client.go @@ -30,12 +30,14 @@ type FluentbitV1alpha2Interface interface { ClusterFiltersGetter ClusterFluentBitConfigsGetter ClusterInputsGetter + ClusterMultilineParsersGetter ClusterOutputsGetter ClusterParsersGetter CollectorsGetter FiltersGetter FluentBitsGetter FluentBitConfigsGetter + MultilineParsersGetter OutputsGetter ParsersGetter } @@ -57,6 +59,10 @@ func (c *FluentbitV1alpha2Client) ClusterInputs() ClusterInputInterface { return newClusterInputs(c) } +func (c *FluentbitV1alpha2Client) ClusterMultilineParsers() ClusterMultilineParserInterface { + return newClusterMultilineParsers(c) +} + func (c *FluentbitV1alpha2Client) ClusterOutputs() ClusterOutputInterface { return newClusterOutputs(c) } @@ -81,6 +87,10 @@ func (c *FluentbitV1alpha2Client) FluentBitConfigs(namespace string) FluentBitCo return newFluentBitConfigs(c, namespace) } +func (c *FluentbitV1alpha2Client) MultilineParsers(namespace string) MultilineParserInterface { + return newMultilineParsers(c, namespace) +} + func (c *FluentbitV1alpha2Client) Outputs(namespace string) OutputInterface { return newOutputs(c, namespace) } diff --git a/apis/generated/clientset/versioned/typed/fluentbit/v1alpha2/generated_expansion.go b/apis/generated/clientset/versioned/typed/fluentbit/v1alpha2/generated_expansion.go index b496a4c03..5b9855d62 100644 --- a/apis/generated/clientset/versioned/typed/fluentbit/v1alpha2/generated_expansion.go +++ b/apis/generated/clientset/versioned/typed/fluentbit/v1alpha2/generated_expansion.go @@ -23,6 +23,8 @@ type ClusterFluentBitConfigExpansion interface{} type ClusterInputExpansion interface{} +type ClusterMultilineParserExpansion interface{} + type ClusterOutputExpansion interface{} type ClusterParserExpansion interface{} @@ -35,6 +37,8 @@ type FluentBitExpansion interface{} type FluentBitConfigExpansion interface{} +type MultilineParserExpansion interface{} + type OutputExpansion interface{} type ParserExpansion interface{} diff --git a/apis/generated/clientset/versioned/typed/fluentbit/v1alpha2/multilineparser.go b/apis/generated/clientset/versioned/typed/fluentbit/v1alpha2/multilineparser.go new file mode 100644 index 000000000..de5f42d43 --- /dev/null +++ b/apis/generated/clientset/versioned/typed/fluentbit/v1alpha2/multilineparser.go @@ -0,0 +1,177 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha2 + +import ( + "context" + "time" + + v1alpha2 "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2" + scheme "github.com/fluent/fluent-operator/v2/apis/generated/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// MultilineParsersGetter has a method to return a MultilineParserInterface. +// A group's client should implement this interface. +type MultilineParsersGetter interface { + MultilineParsers(namespace string) MultilineParserInterface +} + +// MultilineParserInterface has methods to work with MultilineParser resources. +type MultilineParserInterface interface { + Create(ctx context.Context, multilineParser *v1alpha2.MultilineParser, opts v1.CreateOptions) (*v1alpha2.MultilineParser, error) + Update(ctx context.Context, multilineParser *v1alpha2.MultilineParser, opts v1.UpdateOptions) (*v1alpha2.MultilineParser, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha2.MultilineParser, error) + List(ctx context.Context, opts v1.ListOptions) (*v1alpha2.MultilineParserList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha2.MultilineParser, err error) + MultilineParserExpansion +} + +// multilineParsers implements MultilineParserInterface +type multilineParsers struct { + client rest.Interface + ns string +} + +// newMultilineParsers returns a MultilineParsers +func newMultilineParsers(c *FluentbitV1alpha2Client, namespace string) *multilineParsers { + return &multilineParsers{ + client: c.RESTClient(), + ns: namespace, + } +} + +// Get takes name of the multilineParser, and returns the corresponding multilineParser object, and an error if there is any. +func (c *multilineParsers) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha2.MultilineParser, err error) { + result = &v1alpha2.MultilineParser{} + err = c.client.Get(). + Namespace(c.ns). + Resource("multilineparsers"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of MultilineParsers that match those selectors. +func (c *multilineParsers) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha2.MultilineParserList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + result = &v1alpha2.MultilineParserList{} + err = c.client.Get(). + Namespace(c.ns). + Resource("multilineparsers"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Do(ctx). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested multilineParsers. +func (c *multilineParsers) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + opts.Watch = true + return c.client.Get(). + Namespace(c.ns). + Resource("multilineparsers"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Watch(ctx) +} + +// Create takes the representation of a multilineParser and creates it. Returns the server's representation of the multilineParser, and an error, if there is any. +func (c *multilineParsers) Create(ctx context.Context, multilineParser *v1alpha2.MultilineParser, opts v1.CreateOptions) (result *v1alpha2.MultilineParser, err error) { + result = &v1alpha2.MultilineParser{} + err = c.client.Post(). + Namespace(c.ns). + Resource("multilineparsers"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(multilineParser). + Do(ctx). + Into(result) + return +} + +// Update takes the representation of a multilineParser and updates it. Returns the server's representation of the multilineParser, and an error, if there is any. +func (c *multilineParsers) Update(ctx context.Context, multilineParser *v1alpha2.MultilineParser, opts v1.UpdateOptions) (result *v1alpha2.MultilineParser, err error) { + result = &v1alpha2.MultilineParser{} + err = c.client.Put(). + Namespace(c.ns). + Resource("multilineparsers"). + Name(multilineParser.Name). + VersionedParams(&opts, scheme.ParameterCodec). + Body(multilineParser). + Do(ctx). + Into(result) + return +} + +// Delete takes name of the multilineParser and deletes it. Returns an error if one occurs. +func (c *multilineParsers) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + return c.client.Delete(). + Namespace(c.ns). + Resource("multilineparsers"). + Name(name). + Body(&opts). + Do(ctx). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *multilineParsers) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + var timeout time.Duration + if listOpts.TimeoutSeconds != nil { + timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second + } + return c.client.Delete(). + Namespace(c.ns). + Resource("multilineparsers"). + VersionedParams(&listOpts, scheme.ParameterCodec). + Timeout(timeout). + Body(&opts). + Do(ctx). + Error() +} + +// Patch applies the patch and returns the patched multilineParser. +func (c *multilineParsers) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha2.MultilineParser, err error) { + result = &v1alpha2.MultilineParser{} + err = c.client.Patch(pt). + Namespace(c.ns). + Resource("multilineparsers"). + Name(name). + SubResource(subresources...). + VersionedParams(&opts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml index 54745c7a4..9dcf04f35 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml @@ -486,31 +486,45 @@ spec: alias: description: Alias for the plugin type: string + buffer: + default: false + type: boolean + emitterMemBufLimit: + default: 10 + description: Set a limit on the amount of memory the emitter + can consume if the outputs provide backpressure. The default + for this limit is 10M. The pipeline will pause once the + buffer exceeds the value of this setting. For example, + if the value is set to 10M then the pipeline will pause + if the buffer exceeds 10M. The pipeline will remain paused + until the output drains the buffer below the 10M limit. + type: integer emitterName: - description: When the filter emits a record under the new - Tag, there is an internal emitter plugin that takes care - of the job. Since this emitter expose metrics as any other - component of the pipeline, you can use this property to - configure an optional name for it. + description: Name for the emitter input instance which re-emits + the completed records at the beginning of the pipeline. type: string emitterStorageType: - description: Specify the emitter buffering mechanism to use. It can be - memory or filesystem + default: memory + description: The storage type for the emitter input instance. + This option supports the values memory (default) and filesystem. enum: - - filesystem - memory + - filesystem type: string - emitterMemBufLimit: - description: Set a limit of memory that Emitter plugin can use when - appending data to the Engine. If the limit is reach, it will - be paused; when the data is flushed it resumes. - type: string + flushMs: + default: 2000 + type: integer keyContent: description: Key name that holds the content to process. Note that a Multiline Parser definition can already specify the key_content to use, but this option allows to overwrite that value for the purpose of the filter. type: string + mode: + enum: + - parser + - partial_message + type: string parser: description: Specify one or multiple Multiline Parsing definitions to apply to the content. You can specify multiple multiline diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfluentbitconfigs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfluentbitconfigs.yaml index 31d611b98..1b7d5d17d 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfluentbitconfigs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfluentbitconfigs.yaml @@ -128,6 +128,51 @@ spec: type: object type: object x-kubernetes-map-type: atomic + multilineparserSelector: + description: Select multiline parser plugins + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic namespace: description: If namespace is defined, then the configmap and secret for fluent-bit is in this namespace. If it is not defined, it is @@ -297,6 +342,11 @@ spec: parsersFile: description: Optional 'parsers' config file (can be multiple) type: string + parsersFiles: + description: backward compatible + items: + type: string + type: array storage: description: Configure a global environment for the storage layer in Service. It is recommended to configure the volume and volumeMount diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clustermultilineparsers.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clustermultilineparsers.yaml new file mode 100644 index 000000000..71eefcd51 --- /dev/null +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clustermultilineparsers.yaml @@ -0,0 +1,83 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.3 + creationTimestamp: null + name: clustermultilineparsers.fluentbit.fluent.io +spec: + group: fluentbit.fluent.io + names: + kind: ClusterMultilineParser + listKind: ClusterMultilineParserList + plural: clustermultilineparsers + shortNames: + - cfbmp + singular: clustermultilineparser + scope: Cluster + versions: + - name: v1alpha2 + schema: + openAPIV3Schema: + description: ClusterMultilineParser is the Schema for the cluster-level multiline + parser API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + flushTimeout: + default: 5000 + description: Timeout in milliseconds to flush a non-terminated multiline + buffer. Default is set to 5 seconds. + type: integer + keyContent: + description: For an incoming structured message, specify the key that + contains the data that should be processed by the regular expression + and possibly concatenated. + type: string + parser: + description: Name of a pre-defined parser that must be applied to + the incoming content before applying the regex rule. If no parser + is defined, it's assumed that's a raw text and not a structured + message. + type: string + rules: + description: Configure a rule to match a multiline pattern. The rule + has a specific format described below. Multiple rules can be defined. + items: + properties: + next: + type: string + regex: + type: string + start: + type: string + required: + - next + - regex + - start + type: object + type: array + type: + default: regex + description: Set the multiline mode, for now, we support the type + regex. + enum: + - regex + type: string + type: object + type: object + served: true + storage: true diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml index 814ae6bbb..a48bf1fcf 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml @@ -488,12 +488,45 @@ spec: alias: description: Alias for the plugin type: string + buffer: + default: false + type: boolean + emitterMemBufLimit: + default: 10 + description: Set a limit on the amount of memory the emitter + can consume if the outputs provide backpressure. The default + for this limit is 10M. The pipeline will pause once the + buffer exceeds the value of this setting. For example, + if the value is set to 10M then the pipeline will pause + if the buffer exceeds 10M. The pipeline will remain paused + until the output drains the buffer below the 10M limit. + type: integer + emitterName: + description: Name for the emitter input instance which re-emits + the completed records at the beginning of the pipeline. + type: string + emitterType: + default: memory + description: The storage type for the emitter input instance. + This option supports the values memory (default) and filesystem. + enum: + - memory + - filesystem + type: string + flushMs: + default: 2000 + type: integer keyContent: description: Key name that holds the content to process. Note that a Multiline Parser definition can already specify the key_content to use, but this option allows to overwrite that value for the purpose of the filter. type: string + mode: + enum: + - parser + - partial_message + type: string parser: description: Specify one or multiple Multiline Parsing definitions to apply to the content. You can specify multiple multiline diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_fluentbitconfigs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_fluentbitconfigs.yaml index 40468ec61..11078d43c 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_fluentbitconfigs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_fluentbitconfigs.yaml @@ -37,6 +37,51 @@ spec: spec: description: NamespacedFluentBitCfgSpec defines the desired state of FluentBit properties: + clusterMultilineParserSelector: + description: Select cluster level multiline parser config + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic clusterParserSelector: description: Select cluster level parser config properties: @@ -127,6 +172,51 @@ spec: type: object type: object x-kubernetes-map-type: atomic + multilineParserSelector: + description: Select multiline parser plugins + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic outputSelector: description: Select output plugins properties: diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_multilineparsers.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_multilineparsers.yaml new file mode 100644 index 000000000..d3081e841 --- /dev/null +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_multilineparsers.yaml @@ -0,0 +1,83 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.3 + creationTimestamp: null + name: multilineparsers.fluentbit.fluent.io +spec: + group: fluentbit.fluent.io + names: + kind: MultilineParser + listKind: MultilineParserList + plural: multilineparsers + shortNames: + - fbmp + singular: multilineparser + scope: Namespaced + versions: + - name: v1alpha2 + schema: + openAPIV3Schema: + description: MultilineParser is the Schema of namespace-level multiline parser + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + flushTimeout: + default: 5000 + description: Timeout in milliseconds to flush a non-terminated multiline + buffer. Default is set to 5 seconds. + type: integer + keyContent: + description: For an incoming structured message, specify the key that + contains the data that should be processed by the regular expression + and possibly concatenated. + type: string + parser: + description: Name of a pre-defined parser that must be applied to + the incoming content before applying the regex rule. If no parser + is defined, it's assumed that's a raw text and not a structured + message. + type: string + rules: + description: Configure a rule to match a multiline pattern. The rule + has a specific format described below. Multiple rules can be defined. + items: + properties: + next: + type: string + regex: + type: string + start: + type: string + required: + - next + - regex + - start + type: object + type: array + type: + default: regex + description: Set the multiline mode, for now, we support the type + regex. + enum: + - regex + type: string + type: object + type: object + served: true + storage: true diff --git a/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml b/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml index 38064b4a6..e4c621a41 100644 --- a/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml +++ b/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml @@ -486,12 +486,45 @@ spec: alias: description: Alias for the plugin type: string + buffer: + default: false + type: boolean + emitterMemBufLimit: + default: 10 + description: Set a limit on the amount of memory the emitter + can consume if the outputs provide backpressure. The default + for this limit is 10M. The pipeline will pause once the + buffer exceeds the value of this setting. For example, + if the value is set to 10M then the pipeline will pause + if the buffer exceeds 10M. The pipeline will remain paused + until the output drains the buffer below the 10M limit. + type: integer + emitterName: + description: Name for the emitter input instance which re-emits + the completed records at the beginning of the pipeline. + type: string + emitterType: + default: memory + description: The storage type for the emitter input instance. + This option supports the values memory (default) and filesystem. + enum: + - memory + - filesystem + type: string + flushMs: + default: 2000 + type: integer keyContent: description: Key name that holds the content to process. Note that a Multiline Parser definition can already specify the key_content to use, but this option allows to overwrite that value for the purpose of the filter. type: string + mode: + enum: + - parser + - partial_message + type: string parser: description: Specify one or multiple Multiline Parsing definitions to apply to the content. You can specify multiple multiline diff --git a/config/crd/bases/fluentbit.fluent.io_clusterfluentbitconfigs.yaml b/config/crd/bases/fluentbit.fluent.io_clusterfluentbitconfigs.yaml index 31d611b98..1b7d5d17d 100644 --- a/config/crd/bases/fluentbit.fluent.io_clusterfluentbitconfigs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_clusterfluentbitconfigs.yaml @@ -128,6 +128,51 @@ spec: type: object type: object x-kubernetes-map-type: atomic + multilineparserSelector: + description: Select multiline parser plugins + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic namespace: description: If namespace is defined, then the configmap and secret for fluent-bit is in this namespace. If it is not defined, it is @@ -297,6 +342,11 @@ spec: parsersFile: description: Optional 'parsers' config file (can be multiple) type: string + parsersFiles: + description: backward compatible + items: + type: string + type: array storage: description: Configure a global environment for the storage layer in Service. It is recommended to configure the volume and volumeMount diff --git a/config/crd/bases/fluentbit.fluent.io_clustermultilineparsers.yaml b/config/crd/bases/fluentbit.fluent.io_clustermultilineparsers.yaml new file mode 100644 index 000000000..71eefcd51 --- /dev/null +++ b/config/crd/bases/fluentbit.fluent.io_clustermultilineparsers.yaml @@ -0,0 +1,83 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.3 + creationTimestamp: null + name: clustermultilineparsers.fluentbit.fluent.io +spec: + group: fluentbit.fluent.io + names: + kind: ClusterMultilineParser + listKind: ClusterMultilineParserList + plural: clustermultilineparsers + shortNames: + - cfbmp + singular: clustermultilineparser + scope: Cluster + versions: + - name: v1alpha2 + schema: + openAPIV3Schema: + description: ClusterMultilineParser is the Schema for the cluster-level multiline + parser API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + flushTimeout: + default: 5000 + description: Timeout in milliseconds to flush a non-terminated multiline + buffer. Default is set to 5 seconds. + type: integer + keyContent: + description: For an incoming structured message, specify the key that + contains the data that should be processed by the regular expression + and possibly concatenated. + type: string + parser: + description: Name of a pre-defined parser that must be applied to + the incoming content before applying the regex rule. If no parser + is defined, it's assumed that's a raw text and not a structured + message. + type: string + rules: + description: Configure a rule to match a multiline pattern. The rule + has a specific format described below. Multiple rules can be defined. + items: + properties: + next: + type: string + regex: + type: string + start: + type: string + required: + - next + - regex + - start + type: object + type: array + type: + default: regex + description: Set the multiline mode, for now, we support the type + regex. + enum: + - regex + type: string + type: object + type: object + served: true + storage: true diff --git a/config/crd/bases/fluentbit.fluent.io_filters.yaml b/config/crd/bases/fluentbit.fluent.io_filters.yaml index f84c8b4d2..a505d3242 100644 --- a/config/crd/bases/fluentbit.fluent.io_filters.yaml +++ b/config/crd/bases/fluentbit.fluent.io_filters.yaml @@ -486,12 +486,45 @@ spec: alias: description: Alias for the plugin type: string + buffer: + default: false + type: boolean + emitterMemBufLimit: + default: 10 + description: Set a limit on the amount of memory the emitter + can consume if the outputs provide backpressure. The default + for this limit is 10M. The pipeline will pause once the + buffer exceeds the value of this setting. For example, + if the value is set to 10M then the pipeline will pause + if the buffer exceeds 10M. The pipeline will remain paused + until the output drains the buffer below the 10M limit. + type: integer + emitterName: + description: Name for the emitter input instance which re-emits + the completed records at the beginning of the pipeline. + type: string + emitterType: + default: memory + description: The storage type for the emitter input instance. + This option supports the values memory (default) and filesystem. + enum: + - memory + - filesystem + type: string + flushMs: + default: 2000 + type: integer keyContent: description: Key name that holds the content to process. Note that a Multiline Parser definition can already specify the key_content to use, but this option allows to overwrite that value for the purpose of the filter. type: string + mode: + enum: + - parser + - partial_message + type: string parser: description: Specify one or multiple Multiline Parsing definitions to apply to the content. You can specify multiple multiline diff --git a/config/crd/bases/fluentbit.fluent.io_fluentbitconfigs.yaml b/config/crd/bases/fluentbit.fluent.io_fluentbitconfigs.yaml index 73a7e7ede..f724c02bd 100644 --- a/config/crd/bases/fluentbit.fluent.io_fluentbitconfigs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_fluentbitconfigs.yaml @@ -37,6 +37,51 @@ spec: spec: description: NamespacedFluentBitCfgSpec defines the desired state of FluentBit properties: + clusterMultilineParserSelector: + description: Select cluster level multiline parser config + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic clusterParserSelector: description: Select cluster level parser config properties: @@ -127,6 +172,51 @@ spec: type: object type: object x-kubernetes-map-type: atomic + multilineParserSelector: + description: Select multiline parser plugins + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic outputSelector: description: Select output plugins properties: diff --git a/config/crd/bases/fluentbit.fluent.io_multilineparsers.yaml b/config/crd/bases/fluentbit.fluent.io_multilineparsers.yaml new file mode 100644 index 000000000..d3081e841 --- /dev/null +++ b/config/crd/bases/fluentbit.fluent.io_multilineparsers.yaml @@ -0,0 +1,83 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.3 + creationTimestamp: null + name: multilineparsers.fluentbit.fluent.io +spec: + group: fluentbit.fluent.io + names: + kind: MultilineParser + listKind: MultilineParserList + plural: multilineparsers + shortNames: + - fbmp + singular: multilineparser + scope: Namespaced + versions: + - name: v1alpha2 + schema: + openAPIV3Schema: + description: MultilineParser is the Schema of namespace-level multiline parser + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + flushTimeout: + default: 5000 + description: Timeout in milliseconds to flush a non-terminated multiline + buffer. Default is set to 5 seconds. + type: integer + keyContent: + description: For an incoming structured message, specify the key that + contains the data that should be processed by the regular expression + and possibly concatenated. + type: string + parser: + description: Name of a pre-defined parser that must be applied to + the incoming content before applying the regex rule. If no parser + is defined, it's assumed that's a raw text and not a structured + message. + type: string + rules: + description: Configure a rule to match a multiline pattern. The rule + has a specific format described below. Multiple rules can be defined. + items: + properties: + next: + type: string + regex: + type: string + start: + type: string + required: + - next + - regex + - start + type: object + type: array + type: + default: regex + description: Set the multiline mode, for now, we support the type + regex. + enum: + - regex + type: string + type: object + type: object + served: true + storage: true diff --git a/controllers/fluentbitconfig_controller.go b/controllers/fluentbitconfig_controller.go index c27da6c2f..da414c18b 100644 --- a/controllers/fluentbitconfig_controller.go +++ b/controllers/fluentbitconfig_controller.go @@ -52,8 +52,8 @@ var storeNamespaces map[string]bool // +kubebuilder:rbac:groups=fluentbit.fluent.io,resources=clusterfluentbitconfigs,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=fluentbit.fluent.io,resources=fluentbitconfigs,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=fluentbit.fluent.io,resources=clusterinputs;clusterfilters;clusteroutputs;clusterparsers,verbs=list -// +kubebuilder:rbac:groups=fluentbit.fluent.io,resources=filters;outputs;parsers,verbs=list +// +kubebuilder:rbac:groups=fluentbit.fluent.io,resources=clusterinputs;clusterfilters;clusteroutputs;clusterparsers;clustermultilineparsers,verbs=list +// +kubebuilder:rbac:groups=fluentbit.fluent.io,resources=filters;outputs;parsers;multilineparsers,verbs=list // +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete // Reconcile is part of the main kubernetes reconciliation loop which aims to @@ -134,8 +134,22 @@ func (r *FluentBitConfigReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, err } + // List all multiline parsers matching the label selector. + var multilineParsers fluentbitv1alpha2.ClusterMultilineParserList + selector, err = metav1.LabelSelectorAsSelector(&cfg.Spec.MultilineParserSelector) + if err != nil { + return ctrl.Result{}, err + } + if err = r.List( + ctx, &multilineParsers, client.MatchingLabelsSelector{Selector: selector}, + ); err != nil { + return ctrl.Result{}, err + } + // List all the namespace level resources if they exist and generate configs to mutate tags - nsFilterLists, nsOutputLists, nsParserLists, nsClusterParserLists, rewriteTagConfigs, err := r.processNamespacedFluentBitCfgs(ctx, fb, inputs) + nsFilterLists, nsOutputLists, nsParserLists, nsClusterParserLists, nsMultilineParserLists, nsClusterMultilineParserLists, rewriteTagConfigs, err := r.processNamespacedFluentBitCfgs( + ctx, fb, inputs, + ) if err != nil { return ctrl.Result{}, err @@ -148,7 +162,9 @@ func (r *FluentBitConfigReconciler) Reconcile(ctx context.Context, req ctrl.Requ } // Inject config data into Secret sl := plugins.NewSecretLoader(r.Client, ns) - mainAppCfg, err := cfg.RenderMainConfig(sl, inputs, filters, outputs, nsFilterLists, nsOutputLists, rewriteTagConfigs) + mainAppCfg, err := cfg.RenderMainConfig( + sl, inputs, filters, outputs, nsFilterLists, nsOutputLists, rewriteTagConfigs, + ) if err != nil { return ctrl.Result{}, err } @@ -156,6 +172,12 @@ func (r *FluentBitConfigReconciler) Reconcile(ctx context.Context, req ctrl.Requ if err != nil { return ctrl.Result{}, err } + multilineParserCfg, err := cfg.RenderMultilineParserConfig( + sl, multilineParsers, nsMultilineParserLists, nsClusterMultilineParserLists, + ) + if err != nil { + return ctrl.Result{}, err + } cl := plugins.NewConfigMapLoader(r.Client, ns) scripts, err := cfg.RenderLuaScript(cl, filters, ns) @@ -175,58 +197,75 @@ func (r *FluentBitConfigReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, err } - if _, err := controllerutil.CreateOrPatch(ctx, r.Client, sec, func() error { - sec.Data = map[string][]byte{ - "fluent-bit.conf": []byte(mainAppCfg), - "parsers.conf": []byte(parserCfg), - } - for _, s := range scripts { - sec.Data[s.Name] = []byte(s.Content) - } - sec.SetOwnerReferences(nil) - if err := ctrl.SetControllerReference(&cfg, sec, r.Scheme); err != nil { - return err - } - return nil - }); err != nil { + if _, err := controllerutil.CreateOrPatch( + ctx, r.Client, sec, func() error { + sec.Data = map[string][]byte{ + "fluent-bit.conf": []byte(mainAppCfg), + "parsers.conf": []byte(parserCfg), + "parsers_multiline.conf": []byte(multilineParserCfg), + } + for _, s := range scripts { + sec.Data[s.Name] = []byte(s.Content) + } + sec.SetOwnerReferences(nil) + if err := ctrl.SetControllerReference(&cfg, sec, r.Scheme); err != nil { + return err + } + return nil + }, + ); err != nil { return ctrl.Result{}, err } - r.Log.Info("Fluent Bit main configuration has updated", "logging-control-plane", ns, "fluentbitconfig", cfg.Name, "secret", sec.Name) + r.Log.Info( + "Fluent Bit main configuration has updated", "logging-control-plane", ns, "fluentbitconfig", cfg.Name, + "secret", sec.Name, + ) } } return ctrl.Result{}, nil } -func (r *FluentBitConfigReconciler) processNamespacedFluentBitCfgs(ctx context.Context, fb fluentbitv1alpha2.FluentBit, inputs fluentbitv1alpha2.ClusterInputList) ([]fluentbitv1alpha2.FilterList, []fluentbitv1alpha2.OutputList, - []fluentbitv1alpha2.ParserList, []fluentbitv1alpha2.ClusterParserList, []string, error) { +func (r *FluentBitConfigReconciler) processNamespacedFluentBitCfgs( + ctx context.Context, fb fluentbitv1alpha2.FluentBit, inputs fluentbitv1alpha2.ClusterInputList, +) ( + []fluentbitv1alpha2.FilterList, []fluentbitv1alpha2.OutputList, + []fluentbitv1alpha2.ParserList, []fluentbitv1alpha2.ClusterParserList, + []fluentbitv1alpha2.MultilineParserList, []fluentbitv1alpha2.ClusterMultilineParserList, []string, error, +) { var nsCfgs fluentbitv1alpha2.FluentBitConfigList var filters []fluentbitv1alpha2.FilterList var outputs []fluentbitv1alpha2.OutputList var parsers []fluentbitv1alpha2.ParserList var clusterParsers []fluentbitv1alpha2.ClusterParserList + var multilineParsers []fluentbitv1alpha2.MultilineParserList + var clusterMultilineParsers []fluentbitv1alpha2.ClusterMultilineParserList var rewriteTagConfigs []string // set of rewrite_tag plugin configs to mutate tags for log records coming out of a namespace selector, err := metav1.LabelSelectorAsSelector(&fb.Spec.NamespacedFluentBitCfgSelector) if err != nil { - return filters, outputs, parsers, clusterParsers, nil, err + return filters, outputs, parsers, clusterParsers, multilineParsers, clusterMultilineParsers, nil, err } if err := r.List(ctx, &nsCfgs, client.MatchingLabelsSelector{Selector: selector}); err != nil { - return filters, outputs, parsers, clusterParsers, nil, err + return filters, outputs, parsers, clusterParsers, multilineParsers, clusterMultilineParsers, nil, err } // Form a slice of list of resources per namespace for _, cfg := range nsCfgs.Items { - filterList, outputList, parserList, clusterParserList, err := r.ListNamespacedResources(ctx, cfg) + filterList, outputList, parserList, clusterParserList, multilineParsersList, clusterMultilineParsersList, err := r.ListNamespacedResources( + ctx, cfg, + ) if err != nil { - return filters, outputs, parsers, clusterParsers, nil, err + return filters, outputs, parsers, clusterParsers, multilineParsers, clusterMultilineParsers, nil, err } filters = append(filters, filterList) outputs = append(outputs, outputList) parsers = append(parsers, parserList) clusterParsers = append(clusterParsers, clusterParserList) + multilineParsers = append(multilineParsers, multilineParsersList) + clusterMultilineParsers = append(clusterMultilineParsers, clusterMultilineParsersList) if _, ok := storeNamespaces[cfg.Namespace]; !ok { rewriteTagConfig := r.generateRewriteTagConfig(cfg, inputs) @@ -237,58 +276,103 @@ func (r *FluentBitConfigReconciler) processNamespacedFluentBitCfgs(ctx context.C } } - return filters, outputs, parsers, clusterParsers, rewriteTagConfigs, nil + return filters, outputs, parsers, clusterParsers, multilineParsers, clusterMultilineParsers, rewriteTagConfigs, nil } -func (r *FluentBitConfigReconciler) ListNamespacedResources(ctx context.Context, cfg fluentbitv1alpha2.FluentBitConfig) (fluentbitv1alpha2.FilterList, - fluentbitv1alpha2.OutputList, fluentbitv1alpha2.ParserList, fluentbitv1alpha2.ClusterParserList, error) { +func (r *FluentBitConfigReconciler) ListNamespacedResources( + ctx context.Context, cfg fluentbitv1alpha2.FluentBitConfig, +) ( + fluentbitv1alpha2.FilterList, + fluentbitv1alpha2.OutputList, fluentbitv1alpha2.ParserList, fluentbitv1alpha2.ClusterParserList, + fluentbitv1alpha2.MultilineParserList, fluentbitv1alpha2.ClusterMultilineParserList, error, +) { var filters fluentbitv1alpha2.FilterList var outputs fluentbitv1alpha2.OutputList var parsers fluentbitv1alpha2.ParserList var clusterParsers fluentbitv1alpha2.ClusterParserList + var multipleParsers fluentbitv1alpha2.MultilineParserList + var clusterMultipleParsers fluentbitv1alpha2.ClusterMultilineParserList + selector, err := metav1.LabelSelectorAsSelector(&cfg.Spec.FilterSelector) if err != nil { - return filters, outputs, parsers, clusterParsers, err + return filters, outputs, parsers, clusterParsers, multipleParsers, clusterMultipleParsers, err } - if err := r.List(ctx, &filters, client.InNamespace(cfg.Namespace), client.MatchingLabelsSelector{Selector: selector}); err != nil { - return filters, outputs, parsers, clusterParsers, err + if err := r.List( + ctx, &filters, client.InNamespace(cfg.Namespace), client.MatchingLabelsSelector{Selector: selector}, + ); err != nil { + return filters, outputs, parsers, clusterParsers, multipleParsers, clusterMultipleParsers, err } selector, err = metav1.LabelSelectorAsSelector(&cfg.Spec.OutputSelector) if err != nil { - return filters, outputs, parsers, clusterParsers, err + return filters, outputs, parsers, clusterParsers, multipleParsers, clusterMultipleParsers, err } - if err := r.List(ctx, &outputs, client.InNamespace(cfg.Namespace), client.MatchingLabelsSelector{Selector: selector}); err != nil { - return filters, outputs, parsers, clusterParsers, err + if err := r.List( + ctx, &outputs, client.InNamespace(cfg.Namespace), client.MatchingLabelsSelector{Selector: selector}, + ); err != nil { + return filters, outputs, parsers, clusterParsers, multipleParsers, clusterMultipleParsers, err } selector, err = metav1.LabelSelectorAsSelector(&cfg.Spec.ParserSelector) if err != nil { - return filters, outputs, parsers, clusterParsers, err + return filters, outputs, parsers, clusterParsers, multipleParsers, clusterMultipleParsers, err } - if err := r.List(ctx, &parsers, client.InNamespace(cfg.Namespace), client.MatchingLabelsSelector{Selector: selector}); err != nil { - return filters, outputs, parsers, clusterParsers, err + if err := r.List( + ctx, &parsers, client.InNamespace(cfg.Namespace), client.MatchingLabelsSelector{Selector: selector}, + ); err != nil { + return filters, outputs, parsers, clusterParsers, multipleParsers, clusterMultipleParsers, err } selector, err = metav1.LabelSelectorAsSelector(&cfg.Spec.ClusterParserSelector) if err != nil { - return filters, outputs, parsers, clusterParsers, err + return filters, outputs, parsers, clusterParsers, multipleParsers, clusterMultipleParsers, err } if err := r.List(ctx, &clusterParsers, client.MatchingLabelsSelector{Selector: selector}); err != nil { - return filters, outputs, parsers, clusterParsers, err + return filters, outputs, parsers, clusterParsers, multipleParsers, clusterMultipleParsers, err + } + + selector, err = metav1.LabelSelectorAsSelector(&cfg.Spec.MultilineParserSelector) + if err != nil { + return filters, outputs, parsers, clusterParsers, multipleParsers, clusterMultipleParsers, err + } + if err := r.List( + ctx, &multipleParsers, client.InNamespace(cfg.Namespace), client.MatchingLabelsSelector{Selector: selector}, + ); err != nil { + return filters, outputs, parsers, clusterParsers, multipleParsers, clusterMultipleParsers, err + } + + selector, err = metav1.LabelSelectorAsSelector(&cfg.Spec.ClusterMultilineParserSelector) + if err != nil { + return filters, outputs, parsers, clusterParsers, multipleParsers, clusterMultipleParsers, err + } + if err := r.List( + ctx, &clusterMultipleParsers, client.InNamespace(cfg.Namespace), + client.MatchingLabelsSelector{Selector: selector}, + ); err != nil { + return filters, outputs, parsers, clusterParsers, multipleParsers, clusterMultipleParsers, err } // Update the name of the local copies of cluster level parsers. // The intention is to have each namespace use their own copy // of the cluster parser. for i := range clusterParsers.Items { - clusterParsers.Items[i].Name = fmt.Sprintf("%s-%x", clusterParsers.Items[i].Name, md5.Sum([]byte(cfg.Namespace))) + clusterParsers.Items[i].Name = fmt.Sprintf( + "%s-%x", clusterParsers.Items[i].Name, md5.Sum([]byte(cfg.Namespace)), + ) + } + + for i := range clusterMultipleParsers.Items { + clusterMultipleParsers.Items[i].Name = fmt.Sprintf( + "%s-%x", clusterMultipleParsers.Items[i].Name, md5.Sum([]byte(cfg.Namespace)), + ) } - return filters, outputs, parsers, clusterParsers, nil + return filters, outputs, parsers, clusterParsers, multipleParsers, clusterMultipleParsers, nil } -func (r *FluentBitConfigReconciler) generateRewriteTagConfig(cfg fluentbitv1alpha2.FluentBitConfig, inputs fluentbitv1alpha2.ClusterInputList) string { +func (r *FluentBitConfigReconciler) generateRewriteTagConfig( + cfg fluentbitv1alpha2.FluentBitConfig, inputs fluentbitv1alpha2.ClusterInputList, +) string { var tag string for _, input := range inputs.Items { if input.Spec.Tail == nil || !strings.Contains(input.Spec.Tail.Path, "/var/log/containers") { @@ -309,7 +393,12 @@ func (r *FluentBitConfigReconciler) generateRewriteTagConfig(cfg fluentbitv1alph buf.WriteString(fmt.Sprintf("[Filter]\n")) buf.WriteString(fmt.Sprintf(" Name rewrite_tag\n")) buf.WriteString(fmt.Sprintf(" Match %s\n", tag)) - buf.WriteString(fmt.Sprintf(" Rule $kubernetes['namespace_name'] ^(%s)$ %x.$TAG false\n", cfg.Namespace, md5.Sum([]byte(cfg.Namespace)))) + buf.WriteString( + fmt.Sprintf( + " Rule $kubernetes['namespace_name'] ^(%s)$ %x.$TAG false\n", cfg.Namespace, + md5.Sum([]byte(cfg.Namespace)), + ), + ) if cfg.Spec.Service.EmitterName != "" { buf.WriteString(fmt.Sprintf(" Emitter_Name %s\n", cfg.Spec.Service.EmitterName)) } else { @@ -325,19 +414,21 @@ func (r *FluentBitConfigReconciler) generateRewriteTagConfig(cfg fluentbitv1alph } func (r *FluentBitConfigReconciler) SetupWithManager(mgr ctrl.Manager) error { - if err := mgr.GetFieldIndexer().IndexField(context.Background(), &corev1.Secret{}, fluentbitOwnerKey, func(rawObj client.Object) []string { - // Grab the job object, extract the owner. - sec := rawObj.(*corev1.Secret) - owner := metav1.GetControllerOf(sec) - if owner == nil { - return nil - } - // Make sure it's a FluentBitConfig. If so, return it. - if owner.APIVersion != fluentbitApiGVStr || owner.Kind != "FluentBitConfig" { - return nil - } - return []string{owner.Name} - }); err != nil { + if err := mgr.GetFieldIndexer().IndexField( + context.Background(), &corev1.Secret{}, fluentbitOwnerKey, func(rawObj client.Object) []string { + // Grab the job object, extract the owner. + sec := rawObj.(*corev1.Secret) + owner := metav1.GetControllerOf(sec) + if owner == nil { + return nil + } + // Make sure it's a FluentBitConfig. If so, return it. + if owner.APIVersion != fluentbitApiGVStr || owner.Kind != "FluentBitConfig" { + return nil + } + return []string{owner.Name} + }, + ); err != nil { return err } diff --git a/manifests/setup/fluent-operator-crd.yaml b/manifests/setup/fluent-operator-crd.yaml index b3ae9afb5..baf47e532 100644 --- a/manifests/setup/fluent-operator-crd.yaml +++ b/manifests/setup/fluent-operator-crd.yaml @@ -484,12 +484,45 @@ spec: alias: description: Alias for the plugin type: string + buffer: + default: false + type: boolean + emitterMemBufLimit: + default: 10 + description: Set a limit on the amount of memory the emitter + can consume if the outputs provide backpressure. The default + for this limit is 10M. The pipeline will pause once the + buffer exceeds the value of this setting. For example, + if the value is set to 10M then the pipeline will pause + if the buffer exceeds 10M. The pipeline will remain paused + until the output drains the buffer below the 10M limit. + type: integer + emitterName: + description: Name for the emitter input instance which re-emits + the completed records at the beginning of the pipeline. + type: string + emitterType: + default: memory + description: The storage type for the emitter input instance. + This option supports the values memory (default) and filesystem. + enum: + - memory + - filesystem + type: string + flushMs: + default: 2000 + type: integer keyContent: description: Key name that holds the content to process. Note that a Multiline Parser definition can already specify the key_content to use, but this option allows to overwrite that value for the purpose of the filter. type: string + mode: + enum: + - parser + - partial_message + type: string parser: description: Specify one or multiple Multiline Parsing definitions to apply to the content. You can specify multiple multiline @@ -1340,6 +1373,51 @@ spec: type: object type: object x-kubernetes-map-type: atomic + multilineparserSelector: + description: Select multiline parser plugins + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic namespace: description: If namespace is defined, then the configmap and secret for fluent-bit is in this namespace. If it is not defined, it is @@ -1509,6 +1587,11 @@ spec: parsersFile: description: Optional 'parsers' config file (can be multiple) type: string + parsersFiles: + description: backward compatible + items: + type: string + type: array storage: description: Configure a global environment for the storage layer in Service. It is recommended to configure the volume and volumeMount @@ -11473,12 +11556,45 @@ spec: alias: description: Alias for the plugin type: string + buffer: + default: false + type: boolean + emitterMemBufLimit: + default: 10 + description: Set a limit on the amount of memory the emitter + can consume if the outputs provide backpressure. The default + for this limit is 10M. The pipeline will pause once the + buffer exceeds the value of this setting. For example, + if the value is set to 10M then the pipeline will pause + if the buffer exceeds 10M. The pipeline will remain paused + until the output drains the buffer below the 10M limit. + type: integer + emitterName: + description: Name for the emitter input instance which re-emits + the completed records at the beginning of the pipeline. + type: string + emitterType: + default: memory + description: The storage type for the emitter input instance. + This option supports the values memory (default) and filesystem. + enum: + - memory + - filesystem + type: string + flushMs: + default: 2000 + type: integer keyContent: description: Key name that holds the content to process. Note that a Multiline Parser definition can already specify the key_content to use, but this option allows to overwrite that value for the purpose of the filter. type: string + mode: + enum: + - parser + - partial_message + type: string parser: description: Specify one or multiple Multiline Parsing definitions to apply to the content. You can specify multiple multiline @@ -12238,6 +12354,51 @@ spec: spec: description: NamespacedFluentBitCfgSpec defines the desired state of FluentBit properties: + clusterMultilineParserSelector: + description: Select cluster level multiline parser config + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic clusterParserSelector: description: Select cluster level parser config properties: @@ -12328,6 +12489,51 @@ spec: type: object type: object x-kubernetes-map-type: atomic + multilineParserSelector: + description: Select multiline parser plugins + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic outputSelector: description: Select output plugins properties: diff --git a/manifests/setup/setup.yaml b/manifests/setup/setup.yaml index 64769f8fd..516227456 100644 --- a/manifests/setup/setup.yaml +++ b/manifests/setup/setup.yaml @@ -484,12 +484,45 @@ spec: alias: description: Alias for the plugin type: string + buffer: + default: false + type: boolean + emitterMemBufLimit: + default: 10 + description: Set a limit on the amount of memory the emitter + can consume if the outputs provide backpressure. The default + for this limit is 10M. The pipeline will pause once the + buffer exceeds the value of this setting. For example, + if the value is set to 10M then the pipeline will pause + if the buffer exceeds 10M. The pipeline will remain paused + until the output drains the buffer below the 10M limit. + type: integer + emitterName: + description: Name for the emitter input instance which re-emits + the completed records at the beginning of the pipeline. + type: string + emitterType: + default: memory + description: The storage type for the emitter input instance. + This option supports the values memory (default) and filesystem. + enum: + - memory + - filesystem + type: string + flushMs: + default: 2000 + type: integer keyContent: description: Key name that holds the content to process. Note that a Multiline Parser definition can already specify the key_content to use, but this option allows to overwrite that value for the purpose of the filter. type: string + mode: + enum: + - parser + - partial_message + type: string parser: description: Specify one or multiple Multiline Parsing definitions to apply to the content. You can specify multiple multiline @@ -1340,6 +1373,51 @@ spec: type: object type: object x-kubernetes-map-type: atomic + multilineparserSelector: + description: Select multiline parser plugins + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic namespace: description: If namespace is defined, then the configmap and secret for fluent-bit is in this namespace. If it is not defined, it is @@ -1509,6 +1587,11 @@ spec: parsersFile: description: Optional 'parsers' config file (can be multiple) type: string + parsersFiles: + description: backward compatible + items: + type: string + type: array storage: description: Configure a global environment for the storage layer in Service. It is recommended to configure the volume and volumeMount @@ -11473,12 +11556,45 @@ spec: alias: description: Alias for the plugin type: string + buffer: + default: false + type: boolean + emitterMemBufLimit: + default: 10 + description: Set a limit on the amount of memory the emitter + can consume if the outputs provide backpressure. The default + for this limit is 10M. The pipeline will pause once the + buffer exceeds the value of this setting. For example, + if the value is set to 10M then the pipeline will pause + if the buffer exceeds 10M. The pipeline will remain paused + until the output drains the buffer below the 10M limit. + type: integer + emitterName: + description: Name for the emitter input instance which re-emits + the completed records at the beginning of the pipeline. + type: string + emitterType: + default: memory + description: The storage type for the emitter input instance. + This option supports the values memory (default) and filesystem. + enum: + - memory + - filesystem + type: string + flushMs: + default: 2000 + type: integer keyContent: description: Key name that holds the content to process. Note that a Multiline Parser definition can already specify the key_content to use, but this option allows to overwrite that value for the purpose of the filter. type: string + mode: + enum: + - parser + - partial_message + type: string parser: description: Specify one or multiple Multiline Parsing definitions to apply to the content. You can specify multiple multiline @@ -12238,6 +12354,51 @@ spec: spec: description: NamespacedFluentBitCfgSpec defines the desired state of FluentBit properties: + clusterMultilineParserSelector: + description: Select cluster level multiline parser config + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic clusterParserSelector: description: Select cluster level parser config properties: @@ -12328,6 +12489,51 @@ spec: type: object type: object x-kubernetes-map-type: atomic + multilineParserSelector: + description: Select multiline parser plugins + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic outputSelector: description: Select output plugins properties: